[PATCH xserver 2/3] squash: glamor: Fix rendering to a8 textures in core profile.

[Eric Anholt]

Signed-off-by: Eric Anholt <eric at anholt.net>
---
 glamor/glamor_priv.h   | 10 +++++++++-
 glamor/glamor_render.c | 50 +++++++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/glamor/glamor_priv.h b/glamor/glamor_priv.h
index 558ed63..f1eed5b 100644
--- a/glamor/glamor_priv.h
+++ b/glamor/glamor_priv.h
@@ -116,10 +116,17 @@ enum shader_in {
     SHADER_IN_COUNT,
 };
 
+enum shader_dest_swizzle {
+    SHADER_DEST_SWIZZLE_DEFAULT,
+    SHADER_DEST_SWIZZLE_ALPHA_TO_RED,
+    SHADER_DEST_SWIZZLE_COUNT,
+};
+
 struct shader_key {
     enum shader_source source;
     enum shader_mask mask;
     enum shader_in in;
+    enum shader_dest_swizzle dest_swizzle;
 };
 
 struct blendinfo {
@@ -285,7 +292,8 @@ typedef struct glamor_screen_private {
     int render_nr_quads;
     glamor_composite_shader composite_shader[SHADER_SOURCE_COUNT]
         [SHADER_MASK_COUNT]
-        [SHADER_IN_COUNT];
+        [SHADER_IN_COUNT]
+        [SHADER_DEST_SWIZZLE_COUNT];
 
     /* shaders to restore a texture to another texture. */
     GLint finish_access_prog[2];
diff --git a/glamor/glamor_render.c b/glamor/glamor_render.c
index 7a50fe9..c36b345 100644
--- a/glamor/glamor_render.c
+++ b/glamor/glamor_render.c
@@ -177,25 +177,38 @@ glamor_create_composite_fs(struct shader_key *key)
         "		return rel_sampler(mask_sampler, mask_texture,\n"
         "				   mask_wh, mask_repeat_mode, 1);\n"
         "}\n";
+
+    const char *dest_swizzle_default =
+        "vec4 dest_swizzle(vec4 color)\n"
+        "{"
+        "	return color;"
+        "}";
+    const char *dest_swizzle_alpha_to_red =
+        "vec4 dest_swizzle(vec4 color)\n"
+        "{"
+        "	float undef;\n"
+        "	return vec4(color.a, undef, undef, undef);"
+        "}";
+
     const char *in_source_only =
         "void main()\n"
         "{\n"
-        "	gl_FragColor = get_source();\n"
+        "	gl_FragColor = dest_swizzle(get_source());\n"
         "}\n";
     const char *in_normal =
         "void main()\n"
         "{\n"
-        "	gl_FragColor = get_source() * get_mask().a;\n"
+        "	gl_FragColor = dest_swizzle(get_source() * get_mask().a);\n"
         "}\n";
     const char *in_ca_source =
         "void main()\n"
         "{\n"
-        "	gl_FragColor = get_source() * get_mask();\n"
+        "	gl_FragColor = dest_swizzle(get_source() * get_mask());\n"
         "}\n";
     const char *in_ca_alpha =
         "void main()\n"
         "{\n"
-        "	gl_FragColor = get_source().a * get_mask();\n"
+        "	gl_FragColor = dest_swizzle(get_source().a * get_mask());\n"
         "}\n";
     const char *in_ca_dual_blend =
         "out vec4 color0;\n"
@@ -214,6 +227,7 @@ glamor_create_composite_fs(struct shader_key *key)
     const char *in;
     const char *header;
     const char *header_norm = "";
+    const char *dest_swizzle;
     GLuint prog;
 
     switch (key->source) {
@@ -246,6 +260,21 @@ glamor_create_composite_fs(struct shader_key *key)
         FatalError("Bad composite shader mask");
     }
 
+    /* If we're storing to an a8 texture but our texture format is
+     * GL_RED because of a core context, then we need to make sure to
+     * put the alpha into the red channel.
+     */
+    switch (key->dest_swizzle) {
+    case SHADER_DEST_SWIZZLE_DEFAULT:
+        dest_swizzle = dest_swizzle_default;
+        break;
+    case SHADER_DEST_SWIZZLE_ALPHA_TO_RED:
+        dest_swizzle = dest_swizzle_alpha_to_red;
+        break;
+    default:
+        FatalError("Bad composite shader dest swizzle");
+    }
+
     header = header_norm;
     switch (key->in) {
     case SHADER_IN_SOURCE_ONLY:
@@ -271,8 +300,8 @@ glamor_create_composite_fs(struct shader_key *key)
     XNFasprintf(&source,
                 "%s"
                 GLAMOR_DEFAULT_PRECISION
-                "%s%s%s%s%s%s", header, repeat_define, relocate_texture,
-                rel_sampler, source_fetch, mask_fetch, in);
+                "%s%s%s%s%s%s%s", header, repeat_define, relocate_texture,
+                rel_sampler, source_fetch, mask_fetch, dest_swizzle, in);
 
     prog = glamor_compile_glsl_prog(GL_FRAGMENT_SHADER, source);
     free(source);
@@ -386,7 +415,7 @@ glamor_lookup_composite_shader(ScreenPtr screen, struct
     glamor_screen_private *glamor_priv = glamor_get_screen_private(screen);
     glamor_composite_shader *shader;
 
-    shader = &glamor_priv->composite_shader[key->source][key->mask][key->in];
+    shader = &glamor_priv->composite_shader[key->source][key->mask][key->in][key->dest_swizzle];
     if (shader->prog == 0)
         glamor_create_composite_shader(screen, key, shader);
 
@@ -841,6 +870,13 @@ glamor_composite_choose_shader(CARD8 op,
         key.in = SHADER_IN_SOURCE_ONLY;
     }
 
+    if (dest_pixmap->drawable.bitsPerPixel <= 8 &&
+        glamor_priv->one_channel_format == GL_RED) {
+        key.dest_swizzle = SHADER_DEST_SWIZZLE_ALPHA_TO_RED;
+    } else {
+        key.dest_swizzle = SHADER_DEST_SWIZZLE_DEFAULT;
+    }
+
     if (source && source->alphaMap) {
         glamor_fallback("source alphaMap\n");
         goto fail;
-- 
2.6.4