[PATCH 3/5] glamor: Fallback to system memory when fail to allocate one big fbo.

[Michel Dänzer]

From: Zhigang Gong <zhigang.gong at intel.com>

Even when create a pixmap which smaller than the max_fbo_size,
it may fail due to some low level driver limitation. If that is
the case, we don't need to crash the xserver. We just need to
fallback to system memory.

See the related bug at:
https://bugs.freedesktop.org/show_bug.cgi?id=71190

Signed-off-by: Zhigang Gong <zhigang.gong at intel.com>
Reviewed-by: Michel Dänzer <michel.daenzer at amd.com>
Tested-by: Kai Wasserbach <kai at dev.carbon-project.org>
Tested-by: Erich Seifert <eseifert at error-reports.org>
Signed-off-by: Michel Dänzer <michel.daenzer at amd.com>
---
 glamor/glamor.c     | 17 +++++++++--------
 glamor/glamor_fbo.c | 21 ++++++++++++++-------
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/glamor/glamor.c b/glamor/glamor.c
index 29f8874..80679ab 100644
--- a/glamor/glamor.c
+++ b/glamor/glamor.c
@@ -146,7 +146,7 @@ glamor_create_pixmap(ScreenPtr screen, int w, int h, int depth,
     glamor_pixmap_type_t type = GLAMOR_TEXTURE_ONLY;
     glamor_pixmap_private *pixmap_priv;
     glamor_screen_private *glamor_priv = glamor_get_screen_private(screen);
-    glamor_pixmap_fbo *fbo;
+    glamor_pixmap_fbo *fbo = NULL;
     int pitch;
     GLenum format;
 
@@ -199,13 +199,12 @@ glamor_create_pixmap(ScreenPtr screen, int w, int h, int depth,
         pixmap_priv->base.box.x2 = w;
         pixmap_priv->base.box.y2 = h;
         fbo = glamor_create_fbo(glamor_priv, w, h, format, usage);
-    }
-    else {
-        DEBUGF("Create LARGE pixmap %p width %d height %d\n", pixmap, w, h);
+    } else {
+        int tile_size = glamor_priv->max_fbo_size;
+        DEBUGF("Create LARGE pixmap %p width %d height %d, tile size %d\n", pixmap, w, h, tile_size);
         pixmap_priv->type = GLAMOR_TEXTURE_LARGE;
         fbo = glamor_create_fbo_array(glamor_priv, w, h, format, usage,
-                                      glamor_priv->max_fbo_size,
-                                      glamor_priv->max_fbo_size, pixmap_priv);
+                                      tile_size, tile_size, pixmap_priv);
     }
 
     if (fbo == NULL) {
@@ -660,7 +659,8 @@ glamor_fd_from_pixmap(ScreenPtr screen,
     switch (pixmap_priv->type) {
     case GLAMOR_TEXTURE_DRM:
     case GLAMOR_TEXTURE_ONLY:
-        glamor_pixmap_ensure_fbo(pixmap, GL_RGBA, 0);
+        if (!glamor_pixmap_ensure_fbo(pixmap, GL_RGBA, 0))
+            return -1;
         return glamor_egl_dri3_fd_name_from_tex(screen,
                                                 pixmap,
                                                 pixmap_priv->base.fbo->tex,
@@ -684,7 +684,8 @@ glamor_name_from_pixmap(PixmapPtr pixmap, CARD16 *stride, CARD32 *size)
     switch (pixmap_priv->type) {
     case GLAMOR_TEXTURE_DRM:
     case GLAMOR_TEXTURE_ONLY:
-        glamor_pixmap_ensure_fbo(pixmap, GL_RGBA, 0);
+        if (!glamor_pixmap_ensure_fbo(pixmap, GL_RGBA, 0))
+            return -1;
         return glamor_egl_dri3_fd_name_from_tex(pixmap->drawable.pScreen,
                                                 pixmap,
                                                 pixmap_priv->base.fbo->tex,
diff --git a/glamor/glamor_fbo.c b/glamor/glamor_fbo.c
index ed4e348..5521683 100644
--- a/glamor/glamor_fbo.c
+++ b/glamor/glamor_fbo.c
@@ -174,10 +174,10 @@ glamor_pixmap_fbo_cache_put(glamor_pixmap_fbo *fbo)
 #endif
 }
 
-static void
+static int
 glamor_pixmap_ensure_fb(glamor_pixmap_fbo *fbo)
 {
-    int status;
+    int status, err = 0;
 
     glamor_make_current(fbo->glamor_priv);
 
@@ -215,9 +215,11 @@ glamor_pixmap_ensure_fb(glamor_pixmap_fbo *fbo)
             break;
         }
 
-        FatalError("destination is framebuffer incomplete: %s [%x]\n",
-                   str, status);
+        glamor_fallback("glamor: Failed to create fbo, %s\n", str);
+        err = -1;
     }
+
+    return err;
 }
 
 glamor_pixmap_fbo *
@@ -244,8 +246,12 @@ glamor_create_fbo_from_tex(glamor_screen_private *glamor_priv,
         goto done;
     }
 
-    if (flag != GLAMOR_CREATE_FBO_NO_FBO)
-        glamor_pixmap_ensure_fb(fbo);
+    if (flag != GLAMOR_CREATE_FBO_NO_FBO) {
+        if (glamor_pixmap_ensure_fb(fbo) != 0) {
+            glamor_purge_fbo(fbo);
+            fbo = NULL;
+        }
+    }
 
  done:
     return fbo;
@@ -562,7 +568,8 @@ glamor_pixmap_ensure_fbo(PixmapPtr pixmap, GLenum format, int flag)
                                    pixmap->drawable.height, format);
 
         if (flag != GLAMOR_CREATE_FBO_NO_FBO && pixmap_priv->base.fbo->fb == 0)
-            glamor_pixmap_ensure_fb(pixmap_priv->base.fbo);
+            if (glamor_pixmap_ensure_fb(pixmap_priv->base.fbo) != 0)
+                return FALSE;
     }
 
     return TRUE;
-- 
1.9.2