Disable sntrup761 if compiler doesn't support VLAs.

The sntrup761 code sourced from supercop uses variable length
arrays.  Although widely supported, they are not part of the ANSI
C89 spec so if the compiler does not support VLAs, disable the
sntrup761x25519-sha512@openssh.com KEX method by replacing the kex
functions with no-op ones similar to what we do in kexecdh.c.

This should allow OpenSSH to build with a plain C89 compiler again.
Spotted by tim@, ok djm@.

5 files changed, 54 insertions, 1 deletions

diff --git a/configure.ac b/configure.ac
index 35d1aca9f..0cd1025f6 100644
--- a/configure.ac
+++ b/configure.ac
@@ -297,6 +297,16 @@ typedef void foo(const char *, ...) __attribute__((format(printf, 1, 2)));]],
 	 [compiler does not accept __attribute__ on prototype args]) ]
 )
 
+AC_MSG_CHECKING([if compiler supports variable length arrays])
+AC_COMPILE_IFELSE(
+    [AC_LANG_PROGRAM([[#include <stdlib.h>]],
+    [[ int i; for (i=0; i<3; i++){int a[i]; a[i-1]=0;} exit(0); ]])],
+    [ AC_MSG_RESULT([yes])
+      AC_DEFINE(VARIABLE_LENGTH_ARRAYS, [1],
+	 [compiler supports variable length arrays]) ],
+    [ AC_MSG_RESULT([no]) ]
+)
+
 if test "x$no_attrib_nonnull" != "x1" ; then
 	AC_DEFINE([HAVE_ATTRIBUTE__NONNULL__], [1], [Have attribute nonnull])
 fi
diff --git a/defines.h b/defines.h
index 79dcb507f..d6a1d014c 100644
--- a/defines.h
+++ b/defines.h
@@ -894,4 +894,11 @@ struct winsize {
 # define USE_SYSTEM_GLOB
 #endif
 
+/*
+ * sntrup761 uses variable length arrays, only enable if the compiler
+ * supports them.
+ */
+#ifdef VARIABLE_LENGTH_ARRAYS
+# define USE_SNTRUP761X25519 1
+#endif
 #endif /* _DEFINES_H */
diff --git a/kex.c b/kex.c
index f08143a5d..3269b2c31 100644
--- a/kex.c
+++ b/kex.c
@@ -110,8 +110,10 @@ static const struct kexalg kexalgs[] = {
 #if defined(HAVE_EVP_SHA256) || !defined(WITH_OPENSSL)
 	{ KEX_CURVE25519_SHA256, KEX_C25519_SHA256, 0, SSH_DIGEST_SHA256 },
 	{ KEX_CURVE25519_SHA256_OLD, KEX_C25519_SHA256, 0, SSH_DIGEST_SHA256 },
+#ifdef USE_SNTRUP761X25519
 	{ KEX_SNTRUP761X25519_SHA512, KEX_KEM_SNTRUP761X25519_SHA512, 0,
 	    SSH_DIGEST_SHA512 },
+#endif
 #endif /* HAVE_EVP_SHA256 || !WITH_OPENSSL */
 	{ NULL, 0, -1, -1},
 };
diff --git a/kexsntrup761x25519.c b/kexsntrup761x25519.c
index 3d5c6bdf0..e3007fa29 100644
--- a/kexsntrup761x25519.c
+++ b/kexsntrup761x25519.c
@@ -25,6 +25,8 @@
 
 #include "includes.h"
 
+#ifdef USE_SNTRUP761X25519
+
 #include <sys/types.h>
 
 #include <stdio.h>
@@ -217,3 +219,33 @@ kex_kem_sntrup761x25519_dec(struct kex *kex,
 	sshbuf_free(buf);
 	return r;
 }
+
+#else
+
+#include "ssherr.h"
+
+struct kex;
+struct sshbuf;
+struct sshkey;
+
+int
+kex_kem_sntrup761x25519_keypair(struct kex *kex)
+{
+	return SSH_ERR_SIGN_ALG_UNSUPPORTED;
+}
+
+int
+kex_kem_sntrup761x25519_enc(struct kex *kex,
+   const struct sshbuf *client_blob, struct sshbuf **server_blobp,
+   struct sshbuf **shared_secretp)
+{
+	return SSH_ERR_SIGN_ALG_UNSUPPORTED;
+}
+
+int
+kex_kem_sntrup761x25519_dec(struct kex *kex,
+    const struct sshbuf *server_blob, struct sshbuf **shared_secretp)
+{
+	return SSH_ERR_SIGN_ALG_UNSUPPORTED;
+}
+#endif /* USE_SNTRUP761X25519 */
diff --git a/sntrup761.c b/sntrup761.c
index 01f1bc344..c63e600fb 100644
--- a/sntrup761.c
+++ b/sntrup761.c
@@ -10,6 +10,8 @@
 
 #include "includes.h"
 
+#ifdef USE_SNTRUP761X25519
+
 #include <string.h>
 #include "crypto_api.h"
 
@@ -1268,4 +1270,4 @@ int crypto_kem_sntrup761_dec(unsigned char *k,const unsigned char *c,const unsig
   Decap(k,c,sk);
   return 0;
 }
-
+#endif /* USE_SNTRUP761X25519 */