[PATCH xserver] glamor: Fix sampling outside of RGBx source/mask pictures

[Alex Deucher]

On Tue, May 24, 2016 at 5:12 AM, Michel Dänzer <michel at daenzer.net> wrote:
> From: Michel Dänzer <michel.daenzer at amd.com>
>
> RENDER requires that sampling outside of any source/mask picture results
> in alpha == 0.0.
>
> The OpenGL border colour cannot set alpha = 0.0 if the texture format
> doesn't have an alpha channel, so we have to use the RepeatFix handling
> in that case.
>
> Also, only force alpha = 1.0 when sampling inside of RGBx source/mask
> pictures.
>
> Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=94514
> Signed-off-by: Michel Dänzer <michel.daenzer at amd.com>

Reviewed-by: Alex Deucher <alexander.deucher at amd.com>

> ---
>  glamor/glamor_render.c | 36 ++++++++++++++++++++++++------------
>  1 file changed, 24 insertions(+), 12 deletions(-)
>
> diff --git a/glamor/glamor_render.c b/glamor/glamor_render.c
> index 65f7059..cc03136 100644
> --- a/glamor/glamor_render.c
> +++ b/glamor/glamor_render.c
> @@ -105,7 +105,7 @@ glamor_create_composite_fs(struct shader_key *key)
>      /* The texture and the pixmap size is not match eaxctly, so can't sample it directly.
>       * rel_sampler will recalculate the texture coords.*/
>      const char *rel_sampler =
> -        " vec4 rel_sampler(sampler2D tex_image, vec2 tex, vec4 wh, int repeat)\n"
> +        " vec4 rel_sampler_rgba(sampler2D tex_image, vec2 tex, vec4 wh, int repeat)\n"
>          "{\n"
>          "      if (repeat >= RepeatFix) {\n"
>          "              tex = rel_tex_coord(tex, wh, repeat);\n"
> @@ -117,6 +117,19 @@ glamor_create_composite_fs(struct shader_key *key)
>          "              }\n"
>          "      }\n"
>          "      return texture2D(tex_image, tex);\n"
> +        "}\n"
> +        " vec4 rel_sampler_rgbx(sampler2D tex_image, vec2 tex, vec4 wh, int repeat)\n"
> +        "{\n"
> +        "      if (repeat >= RepeatFix) {\n"
> +        "              tex = rel_tex_coord(tex, wh, repeat);\n"
> +        "              if (repeat == RepeatFix + RepeatNone) {\n"
> +        "                      if (tex.x < 0.0 || tex.x >= 1.0 || \n"
> +        "                          tex.y < 0.0 || tex.y >= 1.0)\n"
> +        "                              return vec4(0.0, 0.0, 0.0, 0.0);\n"
> +        "                      tex = (fract(tex) / wh.xy);\n"
> +        "              }\n"
> +        "      }\n"
> +        "      return vec4(texture2D(tex_image, tex).rgb, 1.0);\n"
>          "}\n";
>
>      const char *source_solid_fetch =
> @@ -131,8 +144,8 @@ glamor_create_composite_fs(struct shader_key *key)
>          "uniform vec4 source_wh;"
>          "vec4 get_source()\n"
>          "{\n"
> -        "      return rel_sampler(source_sampler, source_texture,\n"
> -        "                         source_wh, source_repeat_mode);\n"
> +        "      return rel_sampler_rgba(source_sampler, source_texture,\n"
> +        "                              source_wh, source_repeat_mode);\n"
>          "}\n";
>      const char *source_pixmap_fetch =
>          "varying vec2 source_texture;\n"
> @@ -140,9 +153,8 @@ glamor_create_composite_fs(struct shader_key *key)
>          "uniform vec4 source_wh;\n"
>          "vec4 get_source()\n"
>          "{\n"
> -        "      return vec4(rel_sampler(source_sampler, source_texture,\n"
> -        "                              source_wh, source_repeat_mode).rgb,\n"
> -        "                              1.0);\n"
> +        "      return rel_sampler_rgbx(source_sampler, source_texture,\n"
> +        "                              source_wh, source_repeat_mode);\n"
>          "}\n";
>      const char *mask_none =
>          "vec4 get_mask()\n"
> @@ -161,8 +173,8 @@ glamor_create_composite_fs(struct shader_key *key)
>          "uniform vec4 mask_wh;\n"
>          "vec4 get_mask()\n"
>          "{\n"
> -        "      return rel_sampler(mask_sampler, mask_texture,\n"
> -        "                         mask_wh, mask_repeat_mode);\n"
> +        "      return rel_sampler_rgba(mask_sampler, mask_texture,\n"
> +        "                              mask_wh, mask_repeat_mode);\n"
>          "}\n";
>      const char *mask_pixmap_fetch =
>          "varying vec2 mask_texture;\n"
> @@ -170,8 +182,8 @@ glamor_create_composite_fs(struct shader_key *key)
>          "uniform vec4 mask_wh;\n"
>          "vec4 get_mask()\n"
>          "{\n"
> -        "      return vec4(rel_sampler(mask_sampler, mask_texture,\n"
> -        "                              mask_wh, mask_repeat_mode).rgb, 1.0);\n"
> +        "      return rel_sampler_rgbx(mask_sampler, mask_texture,\n"
> +        "                              mask_wh, mask_repeat_mode);\n"
>          "}\n";
>
>      const char *dest_swizzle_default =
> @@ -557,8 +569,8 @@ glamor_set_composite_texture(glamor_screen_private *glamor_priv, int unit,
>       *
>       **/
>      if (glamor_pixmap_priv_is_large(pixmap_priv) ||
> -        (glamor_priv->gl_flavor == GLAMOR_GL_ES2 && repeat_type == RepeatNone &&
> -         picture->transform)) {
> +        ((!PICT_FORMAT_A(picture->format) || glamor_priv->gl_flavor == GLAMOR_GL_ES2) &&
> +         repeat_type == RepeatNone && picture->transform)) {
>          glamor_pixmap_fbo_fix_wh_ratio(wh, pixmap, pixmap_priv);
>          glUniform4fv(wh_location, 1, wh);
>
> --
> 2.8.1
>
> _______________________________________________
> xorg-devel at lists.x.org: X.Org development
> Archives: http://lists.x.org/archives/xorg-devel
> Info: https://lists.x.org/mailman/listinfo/xorg-devel