# [PATCH] Handle matrix computation overflow in RRTransformCompute

Keith Packard keithp at keithp.com
Tue Feb 3 20:25:37 PST 2009

```If the computation of the composite fixed-point transform for RandR
overflows at any point, take the resulting floating point transform and
scale that back to fit in a fixed point matrix. This ensures that a matrix
will always be available, although perhaps at reduced precision. Someday we
should add floating point matrices to Render.

Signed-off-by: Keith Packard <keithp at keithp.com>
---
randr/rrtransform.c |   39 ++++++++++++++++++++++++++++++++-------
1 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/randr/rrtransform.c b/randr/rrtransform.c
index a901df4..cc18a06 100644
--- a/randr/rrtransform.c
+++ b/randr/rrtransform.c
@@ -120,6 +120,22 @@ RRTransformCopy (RRTransformPtr dst, RRTransformPtr src)

#define F(x)	IntToxFixed(x)

+static void
+RRTransformRescale(struct pixman_f_transform *f_transform, double limit)
+{
+    double max = 0, v, scale;
+    int i, j;
+
+    for (j = 0; j < 3; j++)
+	for (i = 0; i < 3; i++)
+	    if ((v = abs (f_transform->m[j][i])) > max)
+		max = v;
+    scale = limit / max;
+    for (j = 0; j < 3; j++)
+	for (i = 0; i < 3; i++)
+	    f_transform->m[j][i] *= scale;
+}
+
/*
* Compute the complete transformation matrix including
* client-specified transform, rotation/reflection values and the crtc
@@ -141,6 +157,7 @@ RRTransformCompute (int			    x,
{
PictTransform	    t_transform, inverse;
struct pixman_f_transform tf_transform, tf_inverse;
+    Bool		    overflow = FALSE;

if (!transform) transform = &t_transform;
if (!f_transform) f_transform = &tf_transform;
@@ -234,7 +251,8 @@ RRTransformCompute (int			    x,
#ifdef RANDR_12_INTERFACE
if (rr_transform)
{
-        pixman_transform_multiply (transform, transform, &rr_transform->transform);
+        if (!pixman_transform_multiply (transform, transform, &rr_transform->transform))
+	    overflow = TRUE;
pixman_f_transform_multiply (f_transform, f_transform, &rr_transform->f_transform);
pixman_f_transform_multiply (f_inverse, &rr_transform->f_inverse, f_inverse);
}
@@ -242,19 +260,26 @@ RRTransformCompute (int			    x,
/*
* Compute the class of the resulting transform
*/
-    if (pixman_transform_is_identity (transform))
+    if (!overflow && pixman_transform_is_identity (transform))
{
pixman_transform_init_translate (transform, F ( x), F ( y));

-	pixman_f_transform_init_translate (f_transform, F( x), F( y));
-	pixman_f_transform_init_translate (f_inverse,   F(-x), F(-y));
+	pixman_f_transform_init_translate (f_transform,  x,  y);
+	pixman_f_transform_init_translate (f_inverse,   -x, -y);
return FALSE;
}
else
{
-	pixman_transform_translate (&inverse, transform, x, y);
-	pixman_f_transform_translate (f_inverse, f_transform, x, y);
+	pixman_f_transform_translate (f_inverse, f_transform, -x, -y);
+	if (!pixman_transform_translate (&inverse, transform, F(-x), F(-y)))
+	    overflow = TRUE;
+	if (overflow)
+	{
+	    struct pixman_f_transform f_scaled;
+	    f_scaled = *f_transform;
+	    RRTransformRescale(&f_scaled, 16384.0);
+	    pixman_transform_from_pixman_f_transform(transform, &f_scaled);
+	}
return TRUE;
}
}
-
--
1.5.6.5

```

More information about the xorg mailing list