[PATCH 01/12] glamor: Don't try to do rendering with unsupported formats.

[Eric Anholt]

I'm amazed we've made it as far as we have without these checks: if
you made an unusual format picture that wasn't the normal a8r8g8b8 or
x8r8g8b8 or a8, we'd go ahead and try to render with it, ignoring that
the sampler would fetch totally wrong bits.

Fixes 260 tests in rendercheck -t blend -o src -f a8r8g8b8,x2r10g10b10

Signed-off-by: Eric Anholt <eric at anholt.net>
---
 glamor/glamor_render.c | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/glamor/glamor_render.c b/glamor/glamor_render.c
index 488d1a7..3048cd8 100644
--- a/glamor/glamor_render.c
+++ b/glamor/glamor_render.c
@@ -753,6 +753,29 @@ glamor_set_normalize_tcoords_generic(PixmapPtr pixmap,
                                                             texcoords, stride);
 }
 
+/**
+ * Returns whether the general composite path supports this picture
+ * format for a pixmap that is permanently stored in an FBO (as
+ * opposed to the GLAMOR_PIXMAP_DYNAMIC_UPLOAD path).
+ *
+ * We could support many more formats by using GL_ARB_texture_view to
+ * parse the same bits as different formats.  For now, we only support
+ * tweaking whether we sample the alpha bits of an a8r8g8b8, or just
+ * force them to 1.
+ */
+static Bool
+glamor_render_format_is_supported(PictFormatShort format)
+{
+    switch (format) {
+    case PICT_a8r8g8b8:
+    case PICT_x8r8g8b8:
+    case PICT_a8:
+        return TRUE;
+    default:
+        return FALSE;
+    }
+}
+
 static Bool
 glamor_composite_choose_shader(CARD8 op,
                                PicturePtr source,
@@ -783,6 +806,11 @@ glamor_composite_choose_shader(CARD8 op,
         goto fail;
     }
 
+    if (!glamor_render_format_is_supported(dest->format)) {
+        glamor_fallback("Unsupported dest picture format.\n");
+        goto fail;
+    }
+
     memset(&key, 0, sizeof(key));
     if (!source) {
         key.source = SHADER_SOURCE_SOLID;
@@ -951,6 +979,11 @@ glamor_composite_choose_shader(CARD8 op,
                 glamor_fallback("Failed to upload source texture.\n");
                 goto fail;
             }
+        } else {
+            if (!glamor_render_format_is_supported(source->format)) {
+                glamor_fallback("Unsupported source picture format.\n");
+                goto fail;
+            }
         }
 
         if (mask_status == GLAMOR_UPLOAD_PENDING) {
@@ -959,6 +992,11 @@ glamor_composite_choose_shader(CARD8 op,
                 glamor_fallback("Failed to upload mask texture.\n");
                 goto fail;
             }
+        } else if (mask) {
+            if (!glamor_render_format_is_supported(mask->format)) {
+                glamor_fallback("Unsupported mask picture format.\n");
+                goto fail;
+            }
         }
     }
 #endif
-- 
2.1.4