build: detect overflow for [re]allocate_array.

Use reallocarray() when we have it (i.e. in OpenBSD system). When we
don't, use equivalent overflow detection for our allocate_array and
reallocate_array functions.

Remove lib/reallocarray.c from our LIBOBJS as we no longer need it.

Provide a fallback SIZE_MAX macro definition in flexint.h (not
preprocessor friendly, but enough for our reallocate_array use case).

1 files changed, 16 insertions, 6 deletions

diff --git a/src/misc.c b/src/misc.c
index 6eee161..e17b1a2 100644
--- a/src/misc.c
+++ b/src/misc.c
@@ -142,9 +142,14 @@ void add_action (const char *new_text)
 void   *allocate_array (int size, size_t element_size)
 {
 	void *mem;
-	size_t  num_bytes = element_size * (size_t) size;
-
-	mem = malloc(num_bytes);
+#if HAVE_REALLOCARRAY
+	/* reallocarray has built-in overflow detection */
+	mem = reallocarray(NULL, (size_t) size, element_size);
+#else
+	size_t num_bytes = (size_t) size * element_size;
+	mem = (size && SIZE_MAX / (size_t) size < element_size) ? NULL :
+		malloc(num_bytes);
+#endif
 	if (!mem)
 		flexfatal (_
 			   ("memory allocation failed in allocate_array()"));
@@ -681,9 +686,14 @@ char   *readable_form (int c)
 void   *reallocate_array (void *array, int size, size_t element_size)
 {
 	void *new_array;
-	size_t  num_bytes = element_size * (size_t) size;
-
-	new_array = realloc(array, num_bytes);
+#if HAVE_REALLOCARRAY
+	/* reallocarray has built-in overflow detection */
+	new_array = reallocarray(array, (size_t) size, element_size);
+#else
+	size_t num_bytes = (size_t) size * element_size;
+	new_array = (size && SIZE_MAX / (size_t) size < element_size) ? NULL :
+		realloc(array, num_bytes);
+#endif
 	if (!new_array)
 		flexfatal (_("attempt to increase array size failed"));