[PATCH v2 03/17] Add XNFcallocarray() to allow xnfcalloc() to check for overflow

[Alan Coopersmith]

The xnfcalloc() macro took two arguments but simply multiplied them
together without checking for overflow and defeating any overflow
checking that calloc() might have done.  Let's not do that.

The original XNFcalloc() function is left for now to preserve driver
ABI, but is marked as deprecated so it can be removed in a future round
of ABI break/cleanup.

Signed-off-by: Alan Coopersmith <alan.coopersmith at oracle.com>
Reviewed-by: Matt Turner <mattst88 at gmail.com>
---
 config/config.c            |    3 ++-
 hw/xfree86/utils/cvt/cvt.c |    4 ++--
 include/os.h               |   11 +++++++++--
 os/utils.c                 |   12 +++++++++++-
 4 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/config/config.c b/config/config.c
index b5d634b..de45cc3 100644
--- a/config/config.c
+++ b/config/config.c
@@ -130,7 +130,8 @@ device_is_duplicate(const char *config_info)
 struct OdevAttributes *
 config_odev_allocate_attributes(void)
 {
-    struct OdevAttributes *attribs = XNFcalloc(sizeof (struct OdevAttributes));
+    struct OdevAttributes *attribs =
+        xnfcalloc(1, sizeof (struct OdevAttributes));
     attribs->fd = -1;
     return attribs;
 }
diff --git a/hw/xfree86/utils/cvt/cvt.c b/hw/xfree86/utils/cvt/cvt.c
index d5df17f..9413c20 100644
--- a/hw/xfree86/utils/cvt/cvt.c
+++ b/hw/xfree86/utils/cvt/cvt.c
@@ -54,11 +54,11 @@ XNFalloc(unsigned long n)
 
 /* xnfcalloc implementation used by the server code we built in */
 void *
-XNFcalloc(unsigned long n)
+XNFcallocarray(size_t nmemb, size_t size)
 {
     void *r;
 
-    r = calloc(1, n);
+    r = calloc(nmemb, size);
     if (!r) {
         perror("calloc failed");
         exit(1);
diff --git a/include/os.h b/include/os.h
index 3e68c49..9d8b859 100644
--- a/include/os.h
+++ b/include/os.h
@@ -69,7 +69,7 @@ typedef struct _NewClientRec *NewClientPtr;
 
 #ifndef xnfalloc
 #define xnfalloc(size) XNFalloc((unsigned long)(size))
-#define xnfcalloc(_num, _size) XNFcalloc((unsigned long)(_num)*(unsigned long)(_size))
+#define xnfcalloc(_num, _size) XNFcallocarray((_num), (_size))
 #define xnfrealloc(ptr, size) XNFrealloc((void *)(ptr), (unsigned long)(size))
 
 #define xstrdup(s) Xstrdup(s)
@@ -222,7 +222,14 @@ XNFalloc(unsigned long /*amount */ );
  * enough memory.
  */
 extern _X_EXPORT void *
-XNFcalloc(unsigned long /*amount */ );
+XNFcalloc(unsigned long /*amount */ ) _X_DEPRECATED;
+
+/*
+ * This function calloc(3)s buffer, terminating the server if there is not
+ * enough memory or the arguments overflow when multiplied
+ */
+extern _X_EXPORT void *
+XNFcallocarray(size_t nmemb, size_t size);
 
 /*
  * This function realloc(3)s passed buffer, terminating the server if there is
diff --git a/os/utils.c b/os/utils.c
index 75769f1..24a8751 100644
--- a/os/utils.c
+++ b/os/utils.c
@@ -1128,10 +1128,20 @@ XNFalloc(unsigned long amount)
     return ptr;
 }
 
+/* The original XNFcalloc was used with the xnfcalloc macro which multiplied
+ * the arguments at the call site without allowing calloc to check for overflow.
+ * XNFcallocarray was added to fix that without breaking ABI.
+ */
 void *
 XNFcalloc(unsigned long amount)
 {
-    void *ret = calloc(1, amount);
+    return XNFcallocarray(1, amount);
+}
+
+void *
+XNFcallocarray(size_t nmemb, size_t size)
+{
+    void *ret = calloc(nmemb, size);
 
     if (!ret)
         FatalError("XNFcalloc: Out of memory");
-- 
1.7.9.2