[PATCH xserver] xfree86: fix gamma compute when palette_size > 256

[Qiang Yu]

palette_(red|green|blue)_size > crtc->gamma_size (=256)
this may happen when screen has per RGB chanel > 8bit,
i.e. 30bit depth screen 10bit per RGB.

Signed-off-by: Qiang Yu <Qiang.Yu at amd.com>
---
 hw/xfree86/modes/xf86RandR12.c | 96 ++++++++++++++++++++++++++++++------------
 1 file changed, 69 insertions(+), 27 deletions(-)

diff --git a/hw/xfree86/modes/xf86RandR12.c b/hw/xfree86/modes/xf86RandR12.c
index fe8770d..881c8a6 100644
--- a/hw/xfree86/modes/xf86RandR12.c
+++ b/hw/xfree86/modes/xf86RandR12.c
@@ -1258,40 +1258,82 @@ xf86RandR12CrtcComputeGamma(xf86CrtcPtr crtc, LOCO *palette,
 
     for (shift = 0; (gamma_size << shift) < (1 << 16); shift++);
 
-    gamma_slots = crtc->gamma_size / palette_red_size;
-    for (i = 0; i < palette_red_size; i++) {
-        value = palette[i].red;
-        if (gamma_red)
-            value = gamma_red[value];
-        else
-            value <<= shift;
+    if (crtc->gamma_size >= palette_red_size) {
+        gamma_slots = crtc->gamma_size / palette_red_size;
+        for (i = 0; i < palette_red_size; i++) {
+            value = palette[i].red;
+            if (gamma_red)
+                value = gamma_red[value];
+            else
+                value <<= shift;
 
-        for (j = 0; j < gamma_slots; j++)
-            crtc->gamma_red[i * gamma_slots + j] = value;
+            for (j = 0; j < gamma_slots; j++)
+                crtc->gamma_red[i * gamma_slots + j] = value;
+        }
     }
+    else {
+        gamma_slots = palette_red_size / crtc->gamma_size;
+        for (i = 0; i < crtc->gamma_size; i++) {
+            value = palette[i * gamma_slots].red;
+            if (gamma_red)
+                value = gamma_red[value];
+            else
+                value <<= shift;
 
-    gamma_slots = crtc->gamma_size / palette_green_size;
-    for (i = 0; i < palette_green_size; i++) {
-        value = palette[i].green;
-        if (gamma_green)
-            value = gamma_green[value];
-        else
-            value <<= shift;
+            crtc->gamma_red[i] = value;
+        }
+    }
+
+    if (crtc->gamma_size >= palette_green_size) {
+        gamma_slots = crtc->gamma_size / palette_green_size;
+        for (i = 0; i < palette_green_size; i++) {
+            value = palette[i].green;
+            if (gamma_green)
+                value = gamma_green[value];
+            else
+                value <<= shift;
+
+            for (j = 0; j < gamma_slots; j++)
+                crtc->gamma_green[i * gamma_slots + j] = value;
+        }
+    }
+    else {
+        gamma_slots = palette_green_size / crtc->gamma_size;
+        for (i = 0; i < crtc->gamma_size; i++) {
+            value = palette[i * gamma_slots].green;
+            if (gamma_green)
+                value = gamma_green[value];
+            else
+                value <<= shift;
 
-        for (j = 0; j < gamma_slots; j++)
-            crtc->gamma_green[i * gamma_slots + j] = value;
+            crtc->gamma_green[i] = value;
+        }
     }
 
-    gamma_slots = crtc->gamma_size / palette_blue_size;
-    for (i = 0; i < palette_blue_size; i++) {
-        value = palette[i].blue;
-        if (gamma_blue)
-            value = gamma_blue[value];
-        else
-            value <<= shift;
+    if (crtc->gamma_size >= palette_blue_size) {
+        gamma_slots = crtc->gamma_size / palette_blue_size;
+        for (i = 0; i < palette_blue_size; i++) {
+            value = palette[i].blue;
+            if (gamma_blue)
+                value = gamma_blue[value];
+            else
+                value <<= shift;
 
-        for (j = 0; j < gamma_slots; j++)
-            crtc->gamma_blue[i * gamma_slots + j] = value;
+            for (j = 0; j < gamma_slots; j++)
+                crtc->gamma_blue[i * gamma_slots + j] = value;
+        }
+    }
+    else {
+        gamma_slots = palette_blue_size / crtc->gamma_size;
+        for (i = 0; i < crtc->gamma_size; i++) {
+            value = palette[i * gamma_slots].blue;
+            if (gamma_blue)
+                value = gamma_blue[value];
+            else
+                value <<= shift;
+
+            crtc->gamma_blue[i] = value;
+        }
     }
 }
 
-- 
1.9.1