[PATCH xserver 1/4] glamor: Flush BO cache on GL_OUT_OF_MEMORY

[Max Staudt]

If we run out of memory, we have most likely hogged it with our huge
cache. DRI drivers flush their caches in OOM situations, so we should
do so, too.

NOTE: This may still not free the memory we need as the DRI drivers
typically have a time delay until they actually release the buffers
that GLAMOR just handed back - and thus, until there is a large
contiguous area of memory available to allocate the new buffer from.

In practice this means that the current allocation will probably still
fail, but after two or three seconds the next ones will work.

This could be improved upon by using GL_APPLE_object_purgeable, as
implemented by the i915/i965 driver.

Signed-off-by: Max Staudt <mstaudt at suse.de>
---
 glamor/glamor_fbo.c | 36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/glamor/glamor_fbo.c b/glamor/glamor_fbo.c
index c6ba095..f80c20d 100644
--- a/glamor/glamor_fbo.c
+++ b/glamor/glamor_fbo.c
@@ -143,6 +143,13 @@ glamor_purge_fbo(glamor_screen_private *glamor_priv,
 }
 
 static void
+glamor_fbo_flush_cache(glamor_screen_private *glamor_priv)
+{
+    glamor_priv->tick += GLAMOR_CACHE_EXPIRE_MAX;
+    glamor_fbo_expire(glamor_priv);
+}
+
+static void
 glamor_pixmap_fbo_cache_put(glamor_screen_private *glamor_priv,
                             glamor_pixmap_fbo *fbo)
 {
@@ -157,8 +164,7 @@ glamor_pixmap_fbo_cache_put(glamor_screen_private *glamor_priv,
 
     if (fbo->fb == 0 || fbo->external || n_format == -1
         || glamor_priv->fbo_cache_watermark >= FBO_CACHE_THRESHOLD) {
-        glamor_priv->tick += GLAMOR_CACHE_EXPIRE_MAX;
-        glamor_fbo_expire(glamor_priv);
+        glamor_fbo_flush_cache(glamor_priv);
         glamor_purge_fbo(glamor_priv, fbo);
         return;
     }
@@ -344,18 +350,32 @@ _glamor_create_tex(glamor_screen_private *glamor_priv,
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_ZERO);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_A, GL_RED);
     }
+
+    /* Allocate the texture. */
     glamor_priv->suppress_gl_out_of_memory_logging = true;
     glTexImage2D(GL_TEXTURE_2D, 0, format, w, h, 0,
                  format, GL_UNSIGNED_BYTE, NULL);
     glamor_priv->suppress_gl_out_of_memory_logging = false;
 
     if (glGetError() == GL_OUT_OF_MEMORY) {
-        if (!glamor_priv->logged_any_fbo_allocation_failure) {
-            LogMessageVerb(X_WARNING, 0, "glamor: Failed to allocate %dx%d "
-                           "FBO due to GL_OUT_OF_MEMORY.\n", w, h);
-            LogMessageVerb(X_WARNING, 0,
-                           "glamor: Expect reduced performance.\n");
-            glamor_priv->logged_any_fbo_allocation_failure = true;
+        LogMessageVerb(X_WARNING, 0,
+                       "glamor: glTexImage2D() failed, flushing caches and trying again...\n");
+        glamor_fbo_flush_cache(glamor_priv);
+
+        /* Retry allocating the texture. */
+        glamor_priv->suppress_gl_out_of_memory_logging = true;
+        glTexImage2D(GL_TEXTURE_2D, 0, format, w, h, 0,
+                     format, GL_UNSIGNED_BYTE, NULL);
+        glamor_priv->suppress_gl_out_of_memory_logging = false;
+
+        if (glGetError() == GL_OUT_OF_MEMORY) {
+            if (!glamor_priv->logged_any_fbo_allocation_failure) {
+                LogMessageVerb(X_WARNING, 0, "glamor: Failed to allocate %dx%d "
+                               "FBO due to GL_OUT_OF_MEMORY.\n", w, h);
+                LogMessageVerb(X_WARNING, 0,
+                               "glamor: Expect reduced performance.\n");
+                glamor_priv->logged_any_fbo_allocation_failure = true;
+            }
         }
         glDeleteTextures(1, &tex);
         return 0;
-- 
2.6.6