[PATCH 2/5] Xdmx: integer overflow in GetGLXFBConfigs()

[Alan Coopersmith]

numFBConfigs & numAttribs are CARD32s and need to be bounds checked before
multiplying by structure sizes to come up with the total size to allocate,
to avoid integer overflow leading to underallocation and writing data from
the network past the end of the allocated buffer.

Reported-by: Ilja Van Sprundel <ivansprundel at ioactive.com>
Signed-off-by: Alan Coopersmith <alan.coopersmith at oracle.com>
---
 hw/dmx/dmx_glxvisuals.c |   12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/hw/dmx/dmx_glxvisuals.c b/hw/dmx/dmx_glxvisuals.c
index 027557a..56bd67b 100644
--- a/hw/dmx/dmx_glxvisuals.c
+++ b/hw/dmx/dmx_glxvisuals.c
@@ -279,7 +279,10 @@ GetGLXFBConfigs(Display * dpy, int glxMajorOpcode, int *nconfigs)
         return NULL;
     }
 
-    attrs = (INT32 *) Xmalloc(2 * numAttribs * __GLX_SIZE_CARD32);
+    if (numAttribs < (INT_MAX / (2 * __GLX_SIZE_CARD32)))
+        attrs = Xmalloc(2 * numAttribs * __GLX_SIZE_CARD32);
+    else
+        attrs = NULL;
     if (!attrs) {
         UnlockDisplay(dpy);
         SyncHandle();
@@ -287,15 +290,16 @@ GetGLXFBConfigs(Display * dpy, int glxMajorOpcode, int *nconfigs)
     }
 
     /* Allocate memory for our config structure */
-    config = (__GLXFBConfig *)
-        Xmalloc(numFBConfigs * sizeof(__GLXFBConfig));
+    if (numFBConfigs < (INT_MAX / sizeof(__GLXFBConfig)))
+        config = Xcalloc(numFBConfigs, sizeof(__GLXFBConfig));
+    else
+        config = NULL;
     if (!config) {
         free(attrs);
         UnlockDisplay(dpy);
         SyncHandle();
         return NULL;
     }
-    memset(config, 0, numFBConfigs * sizeof(__GLXFBConfig));
     fbconfigs = config;
 
     /* Convert attribute list into our format */
-- 
1.7.9.2