[PATCH xserver 1/5] glamor-xv: Check that FBO is usable

[Olivier Fourdan]

On some hardware/driver such as nouveau with nv30, using GL_ALPHA for
8-bit depth pixmaps will raise an incomplete attachment error when
binding the FBO, which will cause a segfault later on when trying to
access the FBO from the glamor pixmap in glamor_xv_put_image().

Check early in glamor_xv_core_init() that FBO binding works and return
TRUE if so, or FALSE otherwise so that callers can tell early if
glamor-xv is usable.

Signed-off-by: Olivier Fourdan <ofourdan at redhat.com>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=100710
---
 glamor/glamor_priv.h |  2 +-
 glamor/glamor_xv.c   | 23 ++++++++++++++++++++++-
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/glamor/glamor_priv.h b/glamor/glamor_priv.h
index 7b92f35..66a870f 100644
--- a/glamor/glamor_priv.h
+++ b/glamor/glamor_priv.h
@@ -907,7 +907,7 @@ int glamor_xv_put_image(glamor_port_private *port_priv,
                         short height,
                         Bool sync,
                         RegionPtr clipBoxes);
-void glamor_xv_core_init(ScreenPtr screen);
+Bool glamor_xv_core_init(ScreenPtr screen);
 void glamor_xv_render(glamor_port_private *port_priv);
 
 #include "glamor_utils.h"
diff --git a/glamor/glamor_xv.c b/glamor/glamor_xv.c
index 3bcf909..20495c3 100644
--- a/glamor/glamor_xv.c
+++ b/glamor/glamor_xv.c
@@ -522,13 +522,34 @@ glamor_xv_init_port(glamor_port_private *port_priv)
     REGION_NULL(pScreen, &port_priv->clip);
 }
 
-void
+Bool
 glamor_xv_core_init(ScreenPtr screen)
 {
+    PixmapPtr pixmap;
+    glamor_pixmap_private *pixmap_priv;
+    Bool ret = TRUE;
+
+    /* Create a pixmap with depth 8 to see if this is supported by the
+     * driver/hardware or if that causes an incomplete attachment when
+     * creating the fbo...
+     */
+    pixmap = glamor_create_pixmap(screen, 1, 1, 8, 0);
+    pixmap_priv = glamor_get_pixmap_private(pixmap);
+
+    if (pixmap_priv == NULL || pixmap_priv->fbo == NULL) {
+        /* Something is wrong with pixmaps */
+        ret = FALSE;
+        goto done;
+    }
+
     glamorBrightness = MAKE_ATOM("XV_BRIGHTNESS");
     glamorContrast = MAKE_ATOM("XV_CONTRAST");
     glamorSaturation = MAKE_ATOM("XV_SATURATION");
     glamorHue = MAKE_ATOM("XV_HUE");
     glamorGamma = MAKE_ATOM("XV_GAMMA");
     glamorColorspace = MAKE_ATOM("XV_COLORSPACE");
+
+done:
+    glamor_destroy_pixmap(pixmap);
+    return ret;
 }
-- 
2.9.3