[PATCH xserver] modesetting: allow switching from software to hardware cursors (v4).

[Michael Thayer]

Currently if modesetting ever fails to set a hardware cursor it will switch
to using a software cursor and never go back.  Change this to only
permanently switch to a software cursor if -ENXIO is returned (which means
hardware cursors not supported), and to otherwise still try a hardware
cursor first every time a new one is set.  This is needed because hardware
may be able to handle some cursors in hardware and others not, or virtual
hardware may be able to handle hardware cursors at some times and not
others.

Changes since v1, v2 and v3:
 * take into account the switch to load_cursor_argb_check
 * keep the permanent software cursor fall-back if -ENXIO is returned
 * move parts of v3 into separate patches

Signed-off-by: Michael Thayer <michael.thayer at oracle.com>
---
Tested switching from a hardware to a software cursor and back.

 hw/xfree86/drivers/modesetting/drmmode_display.c | 30 ++++++++++--------------
 hw/xfree86/drivers/modesetting/drmmode_display.h |  1 -
 2 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/hw/xfree86/drivers/modesetting/drmmode_display.c b/hw/xfree86/drivers/modesetting/drmmode_display.c
index 7d171a3..7b66795 100644
--- a/hw/xfree86/drivers/modesetting/drmmode_display.c
+++ b/hw/xfree86/drivers/modesetting/drmmode_display.c
@@ -756,37 +756,33 @@ drmmode_set_cursor(xf86CrtcPtr crtc)
     drmmode_ptr drmmode = drmmode_crtc->drmmode;
     uint32_t handle = drmmode_crtc->cursor_bo->handle;
     modesettingPtr ms = modesettingPTR(crtc->scrn);
+    CursorPtr cursor = xf86CurrentCursor(crtc->scrn->pScreen);
     int ret;
 
-    if (!drmmode_crtc->set_cursor2_failed) {
-        CursorPtr cursor = xf86CurrentCursor(crtc->scrn->pScreen);
-
-        ret =
-            drmModeSetCursor2(drmmode->fd, drmmode_crtc->mode_crtc->crtc_id,
-                              handle, ms->cursor_width, ms->cursor_height,
-                              cursor->bits->xhot, cursor->bits->yhot);
-        if (!ret)
-            return TRUE;
-
-        /* -EINVAL can mean that an old kernel supports drmModeSetCursor but
-         * not drmModeSetCursor2, though it can mean other things too. */
-        if (ret == -EINVAL)
-            drmmode_crtc->set_cursor2_failed = TRUE;
-    }
+    ret = drmModeSetCursor2(drmmode->fd, drmmode_crtc->mode_crtc->crtc_id,
+                            handle, ms->cursor_width, ms->cursor_height,
+                            cursor->bits->xhot, cursor->bits->yhot);
 
+    /* -EINVAL can mean that an old kernel supports drmModeSetCursor but
+     * not drmModeSetCursor2, though it can mean other things too. */
     if (ret == -EINVAL)
         ret = drmModeSetCursor(drmmode->fd, drmmode_crtc->mode_crtc->crtc_id,
                                handle, ms->cursor_width, ms->cursor_height);
 
-    if (ret) {
+    /* -ENXIO normally means that the current drm driver supports neither
+     * cursor_set nor cursor_set2.  Disable hardware cursor support for
+     * the rest of the session in that case. */
+    if (ret == -ENXIO) {
         xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(crtc->scrn);
         xf86CursorInfoPtr cursor_info = xf86_config->cursor_info;
 
         cursor_info->MaxWidth = cursor_info->MaxHeight = 0;
         drmmode_crtc->drmmode->sw_cursor = TRUE;
+    }
+
+    if (ret)
         /* fallback to swcursor */
         return FALSE;
-    }
     return TRUE;
 }
 
diff --git a/hw/xfree86/drivers/modesetting/drmmode_display.h b/hw/xfree86/drivers/modesetting/drmmode_display.h
index bd82968..f979b99 100644
--- a/hw/xfree86/drivers/modesetting/drmmode_display.h
+++ b/hw/xfree86/drivers/modesetting/drmmode_display.h
@@ -91,7 +91,6 @@ typedef struct {
     uint32_t vblank_pipe;
     int dpms_mode;
     struct dumb_bo *cursor_bo;
-    Bool set_cursor2_failed;
     uint16_t lut_r[256], lut_g[256], lut_b[256];
 
     drmmode_bo rotate_bo;
-- 
2.9.3