[PATCH xserver 5/7] modesetting: Fix server crash on failing modeset

[Daniel Martin]

This fixes a potential:
    (EE) modeset(0): failed to set mode: Invalid argument
crashing the server on EnterVT.

To reproduce the bug you need a laptop (i.e. Lenovo X270), a docking
station using MST to connect the outputs (i.e. ThinkPad Ultra Dock) and
have at least one active output at the dock. Then switch to another VT,
undock the laptop and switch back to the server.

drmmode_set_mode_major() will fail on EnterVT as it tries to
re-configure a disconnected output. Skip disconnected outputs and skip
drmModeSetCrtc() if no outputs have to be configured (output_count == 0)
as this would fail too.

Additionally, log the crtc id if drmModeSetCrtc() fails anyway. That id
plus KMS debug logs may help to solve potential other problems.

Signed-off-by: Daniel Martin <consume.noise at gmail.com>
---
 hw/xfree86/drivers/modesetting/drmmode_display.c | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/hw/xfree86/drivers/modesetting/drmmode_display.c b/hw/xfree86/drivers/modesetting/drmmode_display.c
index 66cc9c06a..a6f10ca91 100644
--- a/hw/xfree86/drivers/modesetting/drmmode_display.c
+++ b/hw/xfree86/drivers/modesetting/drmmode_display.c
@@ -592,7 +592,7 @@ drmmode_set_mode_major(xf86CrtcPtr crtc, DisplayModePtr mode,
     DisplayModeRec saved_mode;
     uint32_t *output_ids = NULL;
     int output_count = 0;
-    Bool ret = TRUE;
+    Bool ret = TRUE, rewind = FALSE;
     int i;
     uint32_t fb_id = 0;
     drmModeModeInfo kmode;
@@ -620,7 +620,8 @@ drmmode_set_mode_major(xf86CrtcPtr crtc, DisplayModePtr mode,
         xf86OutputPtr output = xf86_config->output[i];
         drmmode_output_private_ptr drmmode_output;
 
-        if (output->crtc != crtc)
+        if (output->crtc != crtc ||
+                output->status != XF86OutputStatusConnected)
             continue;
 
         drmmode_output = output->driver_private;
@@ -630,6 +631,16 @@ drmmode_set_mode_major(xf86CrtcPtr crtc, DisplayModePtr mode,
         output_count++;
     }
 
+    if (!output_count) {
+        xf86DrvMsg(pScrn->scrnIndex, X_DEBUG,
+                "Skip mode set of crtc[%d], no outputs\n",
+                drmmode_crtc->mode_crtc->crtc_id);
+        crtc->active = FALSE;
+        ret = TRUE;
+        rewind = TRUE;
+        goto done;
+    }
+
     if (!xf86CrtcRotate(crtc)) {
         goto done;
     }
@@ -672,7 +683,9 @@ drmmode_set_mode_major(xf86CrtcPtr crtc, DisplayModePtr mode,
     if (drmModeSetCrtc(drmmode->fd, drmmode_crtc->mode_crtc->crtc_id,
                        fb_id, x, y, output_ids, output_count, &kmode)) {
         xf86DrvMsg(crtc->scrn->scrnIndex, X_ERROR,
-                   "failed to set mode: %s\n", strerror(errno));
+                   "failed to set mode on crtc[%d]: %s\n",
+                   drmmode_crtc->mode_crtc->crtc_id,
+                   strerror(errno));
         ret = FALSE;
         goto done;
     } else
@@ -702,7 +715,7 @@ drmmode_set_mode_major(xf86CrtcPtr crtc, DisplayModePtr mode,
     }
 
  done:
-    if (!ret) {
+    if (!ret || rewind) {
         crtc->x = saved_x;
         crtc->y = saved_y;
         crtc->rotation = saved_rotation;
-- 
2.13.6