[PATCH xserver 1/5] glamor: Handle GL_OUT_OF_MEMORY when allocating texture images.

[Eric Anholt]

The spec allows general undefined behavior when GL_OOM is thrown.  But
if the driver happens to throw the error at this point, it probably
means the pixmap was just too big, so we should delete that texture
and have this pixmap fall back to software.

Signed-off-by: Eric Anholt <eric at anholt.net>
---
 glamor/glamor.c      |  7 +++++++
 glamor/glamor_fbo.c  | 18 ++++++++++++++++++
 glamor/glamor_priv.h |  3 +++
 3 files changed, 28 insertions(+)

diff --git a/glamor/glamor.c b/glamor/glamor.c
index d4a0236..e69f83d 100644
--- a/glamor/glamor.c
+++ b/glamor/glamor.c
@@ -379,6 +379,13 @@ glamor_debug_output_callback(GLenum source,
                              const void *userParam)
 {
     ScreenPtr screen = (void *)userParam;
+    glamor_screen_private *glamor_priv = glamor_get_screen_private(screen);
+
+    if (glamor_priv->suppress_gl_out_of_memory_logging &&
+        source == GL_DEBUG_SOURCE_API && type == GL_DEBUG_TYPE_ERROR) {
+        return;
+    }
+
     LogMessageVerb(X_ERROR, 0, "glamor%d: GL error: %*s\n",
                screen->myNum, length, message);
 }
diff --git a/glamor/glamor_fbo.c b/glamor/glamor_fbo.c
index 262033f..545f89f 100644
--- a/glamor/glamor_fbo.c
+++ b/glamor/glamor_fbo.c
@@ -347,9 +347,25 @@ _glamor_create_tex(glamor_screen_private *glamor_priv,
         glBindTexture(GL_TEXTURE_2D, tex);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+
+        glamor_priv->suppress_gl_out_of_memory_logging = true;
         glTexImage2D(GL_TEXTURE_2D, 0, format, w, h, 0,
                      format, GL_UNSIGNED_BYTE, NULL);
+        glamor_priv->suppress_gl_out_of_memory_logging = false;
     }
+
+    if (glGetError() == GL_OUT_OF_MEMORY) {
+        if (!glamor_priv->logged_any_fbo_allocation_failure) {
+            LogMessageVerb(X_WARNING, 0, "glamor: Failed to allocate %dx%d "
+                           "FBO due to GL_OUT_OF_MEMORY.\n", w, h);
+            LogMessageVerb(X_WARNING, 0,
+                           "glamor: Expect reduced performance.\n");
+            glamor_priv->logged_any_fbo_allocation_failure = true;
+        }
+        glDeleteTextures(1, &tex);
+        return 0;
+    }
+
     return tex;
 }
 
@@ -368,6 +384,8 @@ glamor_create_fbo(glamor_screen_private *glamor_priv,
         return fbo;
  new_fbo:
     tex = _glamor_create_tex(glamor_priv, w, h, format, flag == CREATE_PIXMAP_USAGE_SHARED);
+    if (!tex)
+        return NULL;
     fbo = glamor_create_fbo_from_tex(glamor_priv, w, h, format, tex, flag);
 
     return fbo;
diff --git a/glamor/glamor_priv.h b/glamor/glamor_priv.h
index f3950f1..058edae 100644
--- a/glamor/glamor_priv.h
+++ b/glamor/glamor_priv.h
@@ -292,6 +292,9 @@ typedef struct glamor_screen_private {
     ScreenPtr screen;
     int dri3_enabled;
 
+    Bool suppress_gl_out_of_memory_logging;
+    Bool logged_any_fbo_allocation_failure;
+
     /* xv */
     GLint xv_prog;
 
-- 
2.6.2