[PATCH] xdr_array and calloc security fix

[Jakub Jelinek <jakub at redhat dot com>]

Hi!

2002-08-02  Jakub Jelinek  <jakub@redhat.com>

	* malloc/malloc.c (public_cALLOc): Check for overflow on
	multiplication.
	* sunrpc/xdr_array.c (xdr_array): Likewise.
	* sunrpc/rpc/types.h (mem_free): Add comment.
	Patch by Solar Designer <solar@openwall.com>.

--- libc/malloc/malloc.c.jj	2002-06-21 11:37:55.000000000 +0200
+++ libc/malloc/malloc.c	2002-08-02 00:58:16.000000000 +0200
@@ -3452,16 +3452,23 @@ public_cALLOc(size_t n, size_t elem_size
 {
   mstate av;
   mchunkptr oldtop, p;
-  INTERNAL_SIZE_T sz, csz, oldtopsize;
+  INTERNAL_SIZE_T bytes, sz, csz, oldtopsize;
   Void_t* mem;
   unsigned long clearsize;
   unsigned long nclears;
   INTERNAL_SIZE_T* d;
-
   __malloc_ptr_t (*hook) __MALLOC_PMT ((size_t, __const __malloc_ptr_t)) =
     __malloc_hook;
+
+  /* size_t is unsigned so the behavior on overflow is defined.  */
+  bytes = n * elem_size;
+  if (bytes / elem_size != n) {
+    MALLOC_FAILURE_ACTION;
+    return 0;
+  }
+
   if (hook != NULL) {
-    sz = n * elem_size;
+    sz = bytes;
     mem = (*hook)(sz, RETURN_ADDRESS (0));
     if(mem == 0)
       return 0;
@@ -3473,8 +3480,7 @@ public_cALLOc(size_t n, size_t elem_size
 #endif
   }
 
-  /* FIXME: check for overflow on multiplication.  */
-  sz = n * elem_size;
+  sz = bytes;
 
   arena_get(av, sz);
   if(!av)
--- libc/sunrpc/rpc/types.h.jj	2001-06-25 10:34:46.000000000 +0200
+++ libc/sunrpc/rpc/types.h	2002-08-02 00:59:16.000000000 +0200
@@ -58,6 +58,10 @@ typedef unsigned long rpcport_t;
 
 #include <stdlib.h>		/* For malloc decl.  */
 #define mem_alloc(bsize)	malloc(bsize)
+/*
+ * XXX: This must not use the second argument, or code in xdr_array.c needs
+ * to be modified.
+ */
 #define mem_free(ptr, bsize)	free(ptr)
 
 #ifndef makedev /* ie, we haven't already included it */
--- libc/sunrpc/xdr_array.c.jj	2002-02-28 12:32:13.000000000 +0100
+++ libc/sunrpc/xdr_array.c	2002-08-02 01:00:39.000000000 +0200
@@ -45,6 +45,7 @@ static char sccsid[] = "@(#)xdr_array.c 
 #include <rpc/types.h>
 #include <rpc/xdr.h>
 #include <libintl.h>
+#include <limits.h>
 
 #ifdef USE_IN_LIBIO
 # include <wchar.h>
@@ -81,7 +82,11 @@ xdr_array (xdrs, addrp, sizep, maxsize, 
       return FALSE;
     }
   c = *sizep;
-  if ((c > maxsize) && (xdrs->x_op != XDR_FREE))
+  /*
+   * XXX: Let the overflow possibly happen with XDR_FREE because mem_free()
+   * doesn't actually use its second argument anyway.
+   */
+  if ((c > maxsize || c > UINT_MAX / elsize) && (xdrs->x_op != XDR_FREE))
     {
       return FALSE;
     }

	Jakub