basic/hexdecoct: check for overflow

LGTM was complaining:
> Multiplication result may overflow 'int' before it is converted to 'long'.
Fix this by changing all types to ssize_t and add a check for overflow
while at it.

(cherry picked from commit 3d6c1844744f631995af72867d5f293430d8015b)

1 files changed, 4 insertions, 2 deletions

diff --git a/src/basic/hexdecoct.c b/src/basic/hexdecoct.c
index 7748e8352..e402ba82e 100644
--- a/src/basic/hexdecoct.c
+++ b/src/basic/hexdecoct.c
@@ -592,8 +592,7 @@ static int base64_append_width(
 
         _cleanup_free_ char *x = NULL;
         char *t, *s;
-        ssize_t slen, len, avail;
-        int line, lines;
+        ssize_t len, slen, avail, line, lines;
 
         len = base64mem(p, l, &x);
         if (len <= 0)
@@ -602,6 +601,9 @@ static int base64_append_width(
         lines = DIV_ROUND_UP(len, width);
 
         slen = strlen_ptr(sep);
+        if (lines > (SSIZE_MAX - plen - 1 - slen) / (indent + width + 1))
+                return -ENOMEM;
+
         t = realloc(*prefix, plen + 1 + slen + (indent + width + 1) * lines);
         if (!t)
                 return -ENOMEM;