[PATCH xserver] glamor: Only force alpha to 1.0 inside RGBx source pictures

[Michel Dänzer]

From: Michel Dänzer <michel.daenzer at amd.com>

We were forcing the alpha channel of RGBx source pictures to 1.0 for
all source coordinates. However, RENDER requires that sampling outside
of source picture boundaries generates alpha 0.0, regardless of whether
the source picture has an alpha channel or not.

May fix
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=94514

Signed-off-by: Michel Dänzer <michel.daenzer at amd.com>
---
 glamor/glamor_render.c | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/glamor/glamor_render.c b/glamor/glamor_render.c
index 65f7059..aef0758 100644
--- a/glamor/glamor_render.c
+++ b/glamor/glamor_render.c
@@ -105,7 +105,7 @@ glamor_create_composite_fs(struct shader_key *key)
     /* The texture and the pixmap size is not match eaxctly, so can't sample it directly.
      * rel_sampler will recalculate the texture coords.*/
     const char *rel_sampler =
-        " vec4 rel_sampler(sampler2D tex_image, vec2 tex, vec4 wh, int repeat)\n"
+        " vec4 rel_sampler_rgba(sampler2D tex_image, vec2 tex, vec4 wh, int repeat)\n"
         "{\n"
         "	if (repeat >= RepeatFix) {\n"
         "		tex = rel_tex_coord(tex, wh, repeat);\n"
@@ -117,6 +117,19 @@ glamor_create_composite_fs(struct shader_key *key)
         "		}\n"
         "	}\n"
         "	return texture2D(tex_image, tex);\n"
+        "}\n"
+        " vec4 rel_sampler_rgbx(sampler2D tex_image, vec2 tex, vec4 wh, int repeat)\n"
+        "{\n"
+        "	if (repeat >= RepeatFix) {\n"
+        "		tex = rel_tex_coord(tex, wh, repeat);\n"
+        "		if (repeat == RepeatFix + RepeatNone) {\n"
+        "			if (tex.x < 0.0 || tex.x >= 1.0 || \n"
+        "			    tex.y < 0.0 || tex.y >= 1.0)\n"
+        "				return vec4(0.0, 0.0, 0.0, 0.0);\n"
+        "			tex = (fract(tex) / wh.xy);\n"
+        "		}\n"
+        "	}\n"
+        "	return vec4(texture2D(tex_image, tex).rgb, 1.0);\n"
         "}\n";
 
     const char *source_solid_fetch =
@@ -131,8 +144,8 @@ glamor_create_composite_fs(struct shader_key *key)
         "uniform vec4 source_wh;"
         "vec4 get_source()\n"
         "{\n"
-        "	return rel_sampler(source_sampler, source_texture,\n"
-        "			   source_wh, source_repeat_mode);\n"
+        "	return rel_sampler_rgba(source_sampler, source_texture,\n"
+        "			        source_wh, source_repeat_mode);\n"
         "}\n";
     const char *source_pixmap_fetch =
         "varying vec2 source_texture;\n"
@@ -140,9 +153,8 @@ glamor_create_composite_fs(struct shader_key *key)
         "uniform vec4 source_wh;\n"
         "vec4 get_source()\n"
         "{\n"
-        "	return vec4(rel_sampler(source_sampler, source_texture,\n"
-        "				source_wh, source_repeat_mode).rgb,\n"
-        "				1.0);\n"
+        "	return rel_sampler_rgbx(source_sampler, source_texture,\n"
+        "				source_wh, source_repeat_mode);\n"
         "}\n";
     const char *mask_none =
         "vec4 get_mask()\n"
@@ -161,8 +173,8 @@ glamor_create_composite_fs(struct shader_key *key)
         "uniform vec4 mask_wh;\n"
         "vec4 get_mask()\n"
         "{\n"
-        "	return rel_sampler(mask_sampler, mask_texture,\n"
-        "			   mask_wh, mask_repeat_mode);\n"
+        "	return rel_sampler_rgba(mask_sampler, mask_texture,\n"
+        "			        mask_wh, mask_repeat_mode);\n"
         "}\n";
     const char *mask_pixmap_fetch =
         "varying vec2 mask_texture;\n"
@@ -170,8 +182,8 @@ glamor_create_composite_fs(struct shader_key *key)
         "uniform vec4 mask_wh;\n"
         "vec4 get_mask()\n"
         "{\n"
-        "	return vec4(rel_sampler(mask_sampler, mask_texture,\n"
-        "				mask_wh, mask_repeat_mode).rgb, 1.0);\n"
+        "	return rel_sampler_rgbx(mask_sampler, mask_texture,\n"
+        "				mask_wh, mask_repeat_mode);\n"
         "}\n";
 
     const char *dest_swizzle_default =
-- 
2.8.0.rc3