--- a/m4/ax_check_openssl.m4
+++ b/m4/ax_check_openssl.m4
@@ -42,9 +42,11 @@
             [root of the OpenSSL directory])],
         [
             case "$withval" in
-            "" | y | ye | yes | n | no)
+            y | ye | yes)
             AC_MSG_ERROR([Invalid --with-openssl value])
               ;;
+            "" | n | no) ssldirs=""
+              ;;
             *) ssldirs="$withval"
               ;;
             esac