[PATCH xserver] randr: Account for panning and transforms when constraining the cursor

[Matt Turner]

From: Chris Wilson <chris at chris-wilson.co.uk>

commit 56c90e29f04727c903bd0f084d23bf44eb1a0a11 [1.10.99.901]
Author: Adam Jackson <ajax at redhat.com>
Date:   Mon Nov 15 14:29:14 2010 -0500

    randr: Add RRConstrainCursorHarder

introduced a regression as it ignored the effect of panning and
transforms upon the crtc bounds. The result was that the cursor would be
constrained to the visible area even though the panning arena was much
bigger, or the cursor was constrained to a region that did not even
match the visible area when the output was transformed or reflected.

This supercedes the hack introduced by
commit 1bf81af4a6be1113bcc3b940ab264d5c9e0f0c5d [1.12.99.904]
Author: Rui Matos <tiagomatos at gmail.com>
Date:   Mon Jul 30 14:32:12 2012 -0400

    xf86RandR12: Don't call ConstrainCursorHarder() if panning is enabled
which disabled the cursor constraints if a panning mode was active, but
did not fix the regression with arbitrary output transforms.

Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=39949
Cc: Adam Jackson <ajax at redhat.com>
Cc: Rui Matos <tiagomatos at gmail.com>
---
The bug's up to 78 comments and I'm getting pings from Gentoo users.

 randr/rrcrtc.c | 60 +++++++++++++++++++++++++++++++++-------------------------
 1 file changed, 34 insertions(+), 26 deletions(-)

diff --git a/randr/rrcrtc.c b/randr/rrcrtc.c
index 338064f9f..d7d937e80 100644
--- a/randr/rrcrtc.c
+++ b/randr/rrcrtc.c
@@ -272,25 +272,34 @@ RRCrtcPendingProperties(RRCrtcPtr crtc)
     return FALSE;
 }
 
-static void
-crtc_bounds(RRCrtcPtr crtc, int *left, int *right, int *top, int *bottom)
+static Bool
+cursor_bounds(RRCrtcPtr crtc, int *left, int *right, int *top, int *bottom)
 {
-    *left = crtc->x;
-    *top = crtc->y;
+    rrScrPriv(crtc->pScreen);
+    BoxRec bounds;
 
-    switch (crtc->rotation & 0xf) {
-    case RR_Rotate_0:
-    case RR_Rotate_180:
-    default:
-        *right = crtc->x + crtc->mode->mode.width;
-        *bottom = crtc->y + crtc->mode->mode.height;
-        return;
-    case RR_Rotate_90:
-    case RR_Rotate_270:
-        *right = crtc->x + crtc->mode->mode.height;
-        *bottom = crtc->y + crtc->mode->mode.width;
-        return;
+    if (crtc->mode == NULL)
+	return FALSE;
+
+    memset(&bounds, 0, sizeof(bounds));
+    if (pScrPriv->rrGetPanning)
+	pScrPriv->rrGetPanning(crtc->pScreen, crtc, NULL, &bounds, NULL);
+
+    if (bounds.y2 <= bounds.y1 || bounds.x2 <= bounds.x1) {
+	bounds.x1 = 0;
+	bounds.y1 = 0;
+	bounds.x2 = crtc->mode->mode.width;
+	bounds.y2 = crtc->mode->mode.height;
     }
+
+    pixman_f_transform_bounds(&crtc->f_transform, &bounds);
+
+    *left = bounds.x1;
+    *right = bounds.x2;
+    *top = bounds.y1;
+    *bottom = bounds.y2;
+
+    return TRUE;
 }
 
 /* overlapping counts as adjacent */
@@ -302,8 +311,10 @@ crtcs_adjacent(const RRCrtcPtr a, const RRCrtcPtr b)
     int bl, br, bt, bb;
     int cl, cr, ct, cb;         /* the overlap, if any */
 
-    crtc_bounds(a, &al, &ar, &at, &ab);
-    crtc_bounds(b, &bl, &br, &bt, &bb);
+    if (!cursor_bounds(a, &al, &ar, &at, &ab))
+	    return FALSE;
+    if (!cursor_bounds(b, &bl, &br, &bt, &bb))
+	    return FALSE;
 
     cl = max(al, bl);
     cr = min(ar, br);
@@ -321,7 +332,7 @@ mark_crtcs(rrScrPrivPtr pScrPriv, int *reachable, int cur)
 
     reachable[cur] = TRUE;
     for (i = 0; i < pScrPriv->numCrtcs; ++i) {
-        if (reachable[i] || !pScrPriv->crtcs[i]->mode)
+        if (reachable[i])
             continue;
         if (crtcs_adjacent(pScrPriv->crtcs[cur], pScrPriv->crtcs[i]))
             mark_crtcs(pScrPriv, reachable, i);
@@ -1811,10 +1822,8 @@ check_all_screen_crtcs(ScreenPtr pScreen, int *x, int *y)
 
         int left, right, top, bottom;
 
-        if (!crtc->mode)
-            continue;
-
-        crtc_bounds(crtc, &left, &right, &top, &bottom);
+        if (!cursor_bounds(crtc, &left, &right, &top, &bottom))
+	    continue;
 
         if ((*x >= left) && (*x < right) && (*y >= top) && (*y < bottom))
             return TRUE;
@@ -1834,10 +1843,9 @@ constrain_all_screen_crtcs(DeviceIntPtr pDev, ScreenPtr pScreen, int *x, int *y)
         int nx, ny;
         int left, right, top, bottom;
 
-        if (!crtc->mode)
-            continue;
+        if (!cursor_bounds(crtc, &left, &right, &top, &bottom))
+	    continue;
 
-        crtc_bounds(crtc, &left, &right, &top, &bottom);
         miPointerGetPosition(pDev, &nx, &ny);
 
         if ((nx >= left) && (nx < right) && (ny >= top) && (ny < bottom)) {
-- 
2.16.1