[PATCH xserver] glamor: Fix sampling outside of RGBx source/mask pictures

[Michel Dänzer]

From: Michel Dänzer <michel.daenzer at amd.com>

RENDER requires that sampling outside of any source/mask picture results
in alpha == 0.0.

The OpenGL border colour cannot set alpha = 0.0 if the texture format
doesn't have an alpha channel, so we have to use the RepeatFix handling
in that case.

Also, only force alpha = 1.0 when sampling inside of RGBx source/mask
pictures.

Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=94514
Signed-off-by: Michel Dänzer <michel.daenzer at amd.com>
---
 glamor/glamor_render.c | 36 ++++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/glamor/glamor_render.c b/glamor/glamor_render.c
index 65f7059..cc03136 100644
--- a/glamor/glamor_render.c
+++ b/glamor/glamor_render.c
@@ -105,7 +105,7 @@ glamor_create_composite_fs(struct shader_key *key)
     /* The texture and the pixmap size is not match eaxctly, so can't sample it directly.
      * rel_sampler will recalculate the texture coords.*/
     const char *rel_sampler =
-        " vec4 rel_sampler(sampler2D tex_image, vec2 tex, vec4 wh, int repeat)\n"
+        " vec4 rel_sampler_rgba(sampler2D tex_image, vec2 tex, vec4 wh, int repeat)\n"
         "{\n"
         "	if (repeat >= RepeatFix) {\n"
         "		tex = rel_tex_coord(tex, wh, repeat);\n"
@@ -117,6 +117,19 @@ glamor_create_composite_fs(struct shader_key *key)
         "		}\n"
         "	}\n"
         "	return texture2D(tex_image, tex);\n"
+        "}\n"
+        " vec4 rel_sampler_rgbx(sampler2D tex_image, vec2 tex, vec4 wh, int repeat)\n"
+        "{\n"
+        "	if (repeat >= RepeatFix) {\n"
+        "		tex = rel_tex_coord(tex, wh, repeat);\n"
+        "		if (repeat == RepeatFix + RepeatNone) {\n"
+        "			if (tex.x < 0.0 || tex.x >= 1.0 || \n"
+        "			    tex.y < 0.0 || tex.y >= 1.0)\n"
+        "				return vec4(0.0, 0.0, 0.0, 0.0);\n"
+        "			tex = (fract(tex) / wh.xy);\n"
+        "		}\n"
+        "	}\n"
+        "	return vec4(texture2D(tex_image, tex).rgb, 1.0);\n"
         "}\n";
 
     const char *source_solid_fetch =
@@ -131,8 +144,8 @@ glamor_create_composite_fs(struct shader_key *key)
         "uniform vec4 source_wh;"
         "vec4 get_source()\n"
         "{\n"
-        "	return rel_sampler(source_sampler, source_texture,\n"
-        "			   source_wh, source_repeat_mode);\n"
+        "	return rel_sampler_rgba(source_sampler, source_texture,\n"
+        "			        source_wh, source_repeat_mode);\n"
         "}\n";
     const char *source_pixmap_fetch =
         "varying vec2 source_texture;\n"
@@ -140,9 +153,8 @@ glamor_create_composite_fs(struct shader_key *key)
         "uniform vec4 source_wh;\n"
         "vec4 get_source()\n"
         "{\n"
-        "	return vec4(rel_sampler(source_sampler, source_texture,\n"
-        "				source_wh, source_repeat_mode).rgb,\n"
-        "				1.0);\n"
+        "	return rel_sampler_rgbx(source_sampler, source_texture,\n"
+        "				source_wh, source_repeat_mode);\n"
         "}\n";
     const char *mask_none =
         "vec4 get_mask()\n"
@@ -161,8 +173,8 @@ glamor_create_composite_fs(struct shader_key *key)
         "uniform vec4 mask_wh;\n"
         "vec4 get_mask()\n"
         "{\n"
-        "	return rel_sampler(mask_sampler, mask_texture,\n"
-        "			   mask_wh, mask_repeat_mode);\n"
+        "	return rel_sampler_rgba(mask_sampler, mask_texture,\n"
+        "			        mask_wh, mask_repeat_mode);\n"
         "}\n";
     const char *mask_pixmap_fetch =
         "varying vec2 mask_texture;\n"
@@ -170,8 +182,8 @@ glamor_create_composite_fs(struct shader_key *key)
         "uniform vec4 mask_wh;\n"
         "vec4 get_mask()\n"
         "{\n"
-        "	return vec4(rel_sampler(mask_sampler, mask_texture,\n"
-        "				mask_wh, mask_repeat_mode).rgb, 1.0);\n"
+        "	return rel_sampler_rgbx(mask_sampler, mask_texture,\n"
+        "				mask_wh, mask_repeat_mode);\n"
         "}\n";
 
     const char *dest_swizzle_default =
@@ -557,8 +569,8 @@ glamor_set_composite_texture(glamor_screen_private *glamor_priv, int unit,
      *
      **/
     if (glamor_pixmap_priv_is_large(pixmap_priv) ||
-        (glamor_priv->gl_flavor == GLAMOR_GL_ES2 && repeat_type == RepeatNone &&
-         picture->transform)) {
+        ((!PICT_FORMAT_A(picture->format) || glamor_priv->gl_flavor == GLAMOR_GL_ES2) &&
+         repeat_type == RepeatNone && picture->transform)) {
         glamor_pixmap_fbo_fix_wh_ratio(wh, pixmap, pixmap_priv);
         glUniform4fv(wh_location, 1, wh);
 
-- 
2.8.1