Re #1975: Add autoconf detection of OpenSSL elliptic curve and sigalg support

git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@5483 74dad513-b988-da41-8d7b-12977e46ad98

4 files changed, 176 insertions, 7 deletions

diff --git a/aconfigure b/aconfigure
index db88162c..bc3c8147 100755
--- a/aconfigure
+++ b/aconfigure
@@ -641,9 +641,13 @@ opencore_amrnb_present
 opencore_amrnb_h_present
 ac_no_opencore_amrwb
 ac_no_opencore_amrnb
+ec_curve_present
+set_curve_present
 libcrypto_present
 libssl_present
 openssl_h_present
+ac_ssl_has_sigalg
+ac_ssl_has_ec
 ac_ssl_has_aes_gcm
 ac_no_ssl
 ac_openh264_ldflags
@@ -5414,12 +5418,14 @@ if test "x$ac_cv_header_net_if_h" = xyes; then :
 fi
 
 
+
 ac_fn_c_check_func "$LINENO" "localtime_r" "ac_cv_func_localtime_r"
 if test "x$ac_cv_func_localtime_r" = xyes; then :
   $as_echo "#define PJ_HAS_LOCALTIME_R 1" >>confdefs.h
 
 fi
 
+
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: Setting PJ_OS_NAME to $target" >&5
 $as_echo "Setting PJ_OS_NAME to $target" >&6; }
 cat >>confdefs.h <<_ACEOF
@@ -7783,6 +7789,10 @@ fi
 
 ac_ssl_has_aes_gcm=0
 
+ac_ssl_has_ec=0
+
+ac_ssl_has_sigalg=0
+
 # Check whether --enable-ssl was given.
 if test "${enable_ssl+set}" = set; then :
   enableval=$enable_ssl;
@@ -7945,6 +7955,120 @@ $as_echo "OpenSSL has AES GCM support, SRTP will use OpenSSL" >&6; }
 $as_echo "OpenSSL AES GCM support not found, SRTP will only support AES CM cryptos" >&6; }
 			fi
 
+			# Check if OpenSSL supports setting curve algorithm
+			# and has elliptic curve
+
+			{ $as_echo "$as_me:${as_lineno-$LINENO}: checking OpenSSL setting curve functions" >&5
+$as_echo_n "checking OpenSSL setting curve functions... " >&6; }
+			set_curve_present=0
+
+			ec_curve_present=0
+
+			cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <openssl/ssl.h>
+
+int
+main ()
+{
+ SSL_set1_curves(NULL, NULL, 0);
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+   set_curve_present=1
+		  		   { $as_echo "$as_me:${as_lineno-$LINENO}: result: ok" >&5
+$as_echo "ok" >&6; }
+
+else
+
+		  		   { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+
+			{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for EC_curve_nid2nist in -lssl" >&5
+$as_echo_n "checking for EC_curve_nid2nist in -lssl... " >&6; }
+if ${ac_cv_lib_ssl_EC_curve_nid2nist+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lssl  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char EC_curve_nid2nist ();
+int
+main ()
+{
+return EC_curve_nid2nist ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_ssl_EC_curve_nid2nist=yes
+else
+  ac_cv_lib_ssl_EC_curve_nid2nist=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_EC_curve_nid2nist" >&5
+$as_echo "$ac_cv_lib_ssl_EC_curve_nid2nist" >&6; }
+if test "x$ac_cv_lib_ssl_EC_curve_nid2nist" = xyes; then :
+  ec_curve_present=1
+fi
+
+			if test "x$set_curve_present" = "x1" -a "x$ec_curve_present" = "x1"; then
+				ac_ssl_has_ec=1
+				{ $as_echo "$as_me:${as_lineno-$LINENO}: result: OpenSSL has elliptic curve support" >&5
+$as_echo "OpenSSL has elliptic curve support" >&6; }
+			else
+				{ $as_echo "$as_me:${as_lineno-$LINENO}: result: OpenSSL elliptic curve algorithm unsupported" >&5
+$as_echo "OpenSSL elliptic curve algorithm unsupported" >&6; }
+			fi
+
+			{ $as_echo "$as_me:${as_lineno-$LINENO}: checking OpenSSL setting sigalg" >&5
+$as_echo_n "checking OpenSSL setting sigalg... " >&6; }
+			cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <openssl/ssl.h>
+
+int
+main ()
+{
+SSL_set1_sigalgs_list(NULL, NULL);
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+   ac_ssl_has_sigalg=1
+		  		   { $as_echo "$as_me:${as_lineno-$LINENO}: result: ok" >&5
+$as_echo "ok" >&6; }
+
+else
+
+		  		   { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+
 			# PJSIP_HAS_TLS_TRANSPORT setting follows PJ_HAS_SSL_SOCK
 			#AC_DEFINE(PJSIP_HAS_TLS_TRANSPORT, 1)
 			$as_echo "#define PJ_HAS_SSL_SOCK 1" >>confdefs.h
diff --git a/aconfigure.ac b/aconfigure.ac
index 387a37ce..50595682 100644
--- a/aconfigure.ac
+++ b/aconfigure.ac
@@ -1533,6 +1533,8 @@ fi
 dnl # Include SSL support
 AC_SUBST(ac_no_ssl)
 AC_SUBST(ac_ssl_has_aes_gcm,0)
+AC_SUBST(ac_ssl_has_ec,0)
+AC_SUBST(ac_ssl_has_sigalg,0)
 AC_ARG_ENABLE(ssl,
 	      AS_HELP_STRING([--disable-ssl],
 			     [Exclude SSL support the build (default: autodetect)])
@@ -1568,6 +1570,43 @@ AC_ARG_ENABLE(ssl,
 				AC_MSG_RESULT([OpenSSL AES GCM support not found, SRTP will only support AES CM cryptos])
 			fi
 
+			# Check if OpenSSL supports setting curve algorithm
+			# and has elliptic curve
+			
+			AC_MSG_CHECKING([OpenSSL setting curve functions])
+			AC_SUBST(set_curve_present,0)
+			AC_SUBST(ec_curve_present,0)
+			AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <openssl/ssl.h>
+					          ]],
+					          [ SSL_set1_curves(NULL, NULL, 0);]
+					         )],
+		  		 [ set_curve_present=1
+		  		   AC_MSG_RESULT(ok)
+		  		  ],
+		  		 [
+		  		   AC_MSG_RESULT(no)
+		  		 ])
+
+			AC_CHECK_LIB(ssl,EC_curve_nid2nist,[ec_curve_present=1])
+			if test "x$set_curve_present" = "x1" -a "x$ec_curve_present" = "x1"; then
+				[ac_ssl_has_ec=1]
+				AC_MSG_RESULT([OpenSSL has elliptic curve support])
+			else
+				AC_MSG_RESULT([OpenSSL elliptic curve algorithm unsupported])
+			fi
+
+			AC_MSG_CHECKING([OpenSSL setting sigalg])
+			AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <openssl/ssl.h>
+					          ]],
+					          [SSL_set1_sigalgs_list(NULL, NULL);]
+					         )],
+		  		 [ ac_ssl_has_sigalg=1
+		  		   AC_MSG_RESULT(ok)
+		  		  ],
+		  		 [
+		  		   AC_MSG_RESULT(no)
+		  		 ])
+
 			# PJSIP_HAS_TLS_TRANSPORT setting follows PJ_HAS_SSL_SOCK
 			#AC_DEFINE(PJSIP_HAS_TLS_TRANSPORT, 1)
 			AC_DEFINE(PJ_HAS_SSL_SOCK, 1)
diff --git a/pjlib/build/os-auto.mak.in b/pjlib/build/os-auto.mak.in
index 478cc80f..e4a7970d 100644
--- a/pjlib/build/os-auto.mak.in
+++ b/pjlib/build/os-auto.mak.in
@@ -33,4 +33,10 @@ export TEST_OBJS +=	@ac_main_obj@
 export TARGETS	    =	$(PJLIB_LIB) $(PJLIB_SONAME)
 export TARGETS_EXE  =   $(TEST_EXE)
 
+ifeq (@ac_ssl_has_ec@,1)
+export PJLIB_CFLAGS += -DPJ_SSL_SOCK_OSSL_HAS_EC=1
+endif
 
+ifeq (@ac_ssl_has_sigalg@,1)
+export PJLIB_CFLAGS += -DPJ_SSL_SOCK_OSSL_HAS_SIGALG=1
+endif
diff --git a/pjlib/src/pj/ssl_sock_ossl.c b/pjlib/src/pj/ssl_sock_ossl.c
index e4ed7842..5d0cf919 100644
--- a/pjlib/src/pj/ssl_sock_ossl.c
+++ b/pjlib/src/pj/ssl_sock_ossl.c
@@ -52,7 +52,7 @@
 #include <openssl/rand.h>
 #include <openssl/engine.h>
 
-#if !defined(OPENSSL_NO_EC)
+#if defined(PJ_SSL_SOCK_OSSL_HAS_EC) && PJ_SSL_SOCK_OSSL_HAS_EC==1
    extern int tls1_ec_nid2curve_id(int nid);
    extern int tls1_ec_curve_id2nid(int curve_id);
 #endif
@@ -385,7 +385,7 @@ static pj_status_t init_openssl(void)
 
 	ssl->session = SSL_SESSION_new();
 
-#if !defined(OPENSSL_NO_EC)
+#if defined(PJ_SSL_SOCK_OSSL_HAS_EC) && PJ_SSL_SOCK_OSSL_HAS_EC==1
 	openssl_curves_num = SSL_get_shared_curve(ssl,-1);
 	if (openssl_curves_num > PJ_ARRAY_SIZE(openssl_curves))
 	    openssl_curves_num = PJ_ARRAY_SIZE(openssl_curves);
@@ -999,7 +999,7 @@ static pj_status_t set_cipher_list(pj_ssl_sock_t *ssock)
 
 static pj_status_t set_curves_list(pj_ssl_sock_t *ssock)
 {
-#if !defined(OPENSSL_NO_EC)
+#if defined(PJ_SSL_SOCK_OSSL_HAS_EC) && PJ_SSL_SOCK_OSSL_HAS_EC==1
     int ret;
     int curves[PJ_SSL_SOCK_MAX_CURVES];
     int cnt;
@@ -1022,15 +1022,14 @@ static pj_status_t set_curves_list(pj_ssl_sock_t *ssock)
 	if (ret < 1)
 	    return GET_SSL_STATUS(ssock);
     }
+#endif
 
     return PJ_SUCCESS;
-#else
-    return PJ_ENOTSUP;
-#endif
 }
 
 static pj_status_t set_sigalgs(pj_ssl_sock_t *ssock)
 {
+#if defined(PJ_SSL_SOCK_OSSL_HAS_SIGALG) && PJ_SSL_SOCK_OSSL_HAS_SIGALG==1
     int ret;
 
     if (ssock->param.sigalgs.ptr && ssock->param.sigalgs.slen) {
@@ -1045,6 +1044,7 @@ static pj_status_t set_sigalgs(pj_ssl_sock_t *ssock)
 	if (ret < 1)
 	    return GET_SSL_STATUS(ssock);
     }
+#endif
 
     return PJ_SUCCESS;
 }
@@ -2393,7 +2393,7 @@ PJ_DEF(pj_status_t) pj_ssl_curve_get_availables(pj_ssl_curve curves[],
     *curve_num = PJ_MIN(*curve_num, openssl_curves_num);
 
     for (i = 0; i < *curve_num; ++i)
-    curves[i] = openssl_curves[i].id;
+	curves[i] = openssl_curves[i].id;
 
     return PJ_SUCCESS;
 }