[PATCH xserver 3/3] modesetting: allow switching from software to hardware cursors (v5).

[Michael Thayer]

Currently if modesetting ever fails to set a hardware cursor it will switch
to using a software cursor and never go back.  Change this to only
permanently switch to a software cursor if -ENXIO is returned (which means
hardware cursors not supported), and to otherwise still try a hardware
cursor first every time a new one is set.  This is needed because hardware
may be able to handle some cursors in hardware and others not, or virtual
hardware may be able to handle hardware cursors at some times and not
others.

Changes since v1, v2 and v3:
 * take into account the switch to load_cursor_argb_check
 * keep the permanent software cursor fall-back if -ENXIO is returned
 * move parts of v3 into separate patches

Signed-off-by: Michael Thayer <michael.thayer at oracle.com>
---
 hw/xfree86/drivers/modesetting/drmmode_display.c | 30 ++++++++++--------------
 hw/xfree86/drivers/modesetting/drmmode_display.h |  1 -
 2 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/hw/xfree86/drivers/modesetting/drmmode_display.c b/hw/xfree86/drivers/modesetting/drmmode_display.c
index 8e68c0e..bbe1a5f 100644
--- a/hw/xfree86/drivers/modesetting/drmmode_display.c
+++ b/hw/xfree86/drivers/modesetting/drmmode_display.c
@@ -756,37 +756,33 @@ drmmode_set_cursor(xf86CrtcPtr crtc)
     drmmode_ptr drmmode = drmmode_crtc->drmmode;
     uint32_t handle = drmmode_crtc->cursor_bo->handle;
     modesettingPtr ms = modesettingPTR(crtc->scrn);
+    CursorPtr cursor = xf86CurrentCursor(crtc->scrn->pScreen);
     int ret = -EINVAL;
 
-    if (!drmmode_crtc->set_cursor2_failed) {
-        CursorPtr cursor = xf86CurrentCursor(crtc->scrn->pScreen);
-
-        ret =
-            drmModeSetCursor2(drmmode->fd, drmmode_crtc->mode_crtc->crtc_id,
-                              handle, ms->cursor_width, ms->cursor_height,
-                              cursor->bits->xhot, cursor->bits->yhot);
-        if (!ret)
-            return TRUE;
-
-        /* -EINVAL can mean that an old kernel supports drmModeSetCursor but
-         * not drmModeSetCursor2, though it can mean other things too. */
-        if (ret == -EINVAL)
-            drmmode_crtc->set_cursor2_failed = TRUE;
-    }
+    ret = drmModeSetCursor2(drmmode->fd, drmmode_crtc->mode_crtc->crtc_id,
+                            handle, ms->cursor_width, ms->cursor_height,
+                            cursor->bits->xhot, cursor->bits->yhot);
 
+    /* -EINVAL can mean that an old kernel supports drmModeSetCursor but
+     * not drmModeSetCursor2, though it can mean other things too. */
     if (ret == -EINVAL)
         ret = drmModeSetCursor(drmmode->fd, drmmode_crtc->mode_crtc->crtc_id,
                                handle, ms->cursor_width, ms->cursor_height);
 
-    if (ret) {
+    /* -ENXIO normally means that the current drm driver supports neither
+     * cursor_set nor cursor_set2.  Disable hardware cursor support for
+     * the rest of the session in that case. */
+    if (ret == -ENXIO) {
         xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(crtc->scrn);
         xf86CursorInfoPtr cursor_info = xf86_config->cursor_info;
 
         cursor_info->MaxWidth = cursor_info->MaxHeight = 0;
         drmmode_crtc->drmmode->sw_cursor = TRUE;
+    }
+
+    if (ret)
         /* fallback to swcursor */
         return FALSE;
-    }
     return TRUE;
 }
 
diff --git a/hw/xfree86/drivers/modesetting/drmmode_display.h b/hw/xfree86/drivers/modesetting/drmmode_display.h
index 3b0eb2b..f774250 100644
--- a/hw/xfree86/drivers/modesetting/drmmode_display.h
+++ b/hw/xfree86/drivers/modesetting/drmmode_display.h
@@ -92,7 +92,6 @@ typedef struct {
     int dpms_mode;
     struct dumb_bo *cursor_bo;
     Bool cursor_up;
-    Bool set_cursor2_failed;
     uint16_t lut_r[256], lut_g[256], lut_b[256];
 
     drmmode_bo rotate_bo;
-- 
2.9.3