[PATCH 1/5] Xdmx: integer overflow in GetGLXVisualConfigs()

[Alan Coopersmith]

numVisuals & numProps are both CARD32 and need to be bounds checked before
multiplying by structure sizes to come up with the total size to allocate,
to avoid integer overflow leading to underallocation and writing data from
the network past the end of the allocated buffer.

Reported-by: Ilja Van Sprundel <ivansprundel at ioactive.com>
Signed-off-by: Alan Coopersmith <alan.coopersmith at oracle.com>
---
 hw/dmx/dmx_glxvisuals.c |   13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/hw/dmx/dmx_glxvisuals.c b/hw/dmx/dmx_glxvisuals.c
index f903b74..027557a 100644
--- a/hw/dmx/dmx_glxvisuals.c
+++ b/hw/dmx/dmx_glxvisuals.c
@@ -37,6 +37,7 @@
 #include <GL/glxproto.h>
 #include <X11/extensions/Xext.h>
 #include <X11/extensions/extutil.h>
+#include <limits.h>
 
 #include "dmx_glxvisuals.h"
 
@@ -84,7 +85,10 @@ GetGLXVisualConfigs(Display * dpy, int screen, int *nconfigs)
         SyncHandle();
         return NULL;
     }
-    props = (INT32 *) Xmalloc(nprops * __GLX_SIZE_CARD32);
+    if (nprops < (INT_MAX / __GLX_SIZE_CARD32))
+        props = Xmalloc(nprops * __GLX_SIZE_CARD32);
+    else
+        props = NULL;
     if (!props) {
         UnlockDisplay(dpy);
         SyncHandle();
@@ -92,15 +96,16 @@ GetGLXVisualConfigs(Display * dpy, int screen, int *nconfigs)
     }
 
     /* Allocate memory for our config structure */
-    config = (__GLXvisualConfig *)
-        Xmalloc(nvisuals * sizeof(__GLXvisualConfig));
+    if (nvisuals < (INT_MAX / sizeof(__GLXvisualConfig)))
+        config = Xcalloc(nvisuals, sizeof(__GLXvisualConfig));
+    else
+        config = NULL;
     if (!config) {
         free(props);
         UnlockDisplay(dpy);
         SyncHandle();
         return NULL;
     }
-    memset(config, 0, nvisuals * sizeof(__GLXvisualConfig));
     configs = config;
     num_good_visuals = 0;
 
-- 
1.7.9.2