Close #1932: Support OpenSSL 1.1.0.

git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@5537 74dad513-b988-da41-8d7b-12977e46ad98

3 files changed, 50 insertions, 64 deletions

diff --git a/aconfigure b/aconfigure
index d7938e9a..1480e5ae 100755
--- a/aconfigure
+++ b/aconfigure
@@ -755,7 +755,6 @@ infodir
 docdir
 oldincludedir
 includedir
-runstatedir
 localstatedir
 sharedstatedir
 sysconfdir
@@ -878,7 +877,6 @@ datadir='${datarootdir}'
 sysconfdir='${prefix}/etc'
 sharedstatedir='${prefix}/com'
 localstatedir='${prefix}/var'
-runstatedir='${localstatedir}/run'
 includedir='${prefix}/include'
 oldincludedir='/usr/include'
 docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@@ -1131,15 +1129,6 @@ do
   | -silent | --silent | --silen | --sile | --sil)
     silent=yes ;;
 
-  -runstatedir | --runstatedir | --runstatedi | --runstated \
-  | --runstate | --runstat | --runsta | --runst | --runs \
-  | --run | --ru | --r)
-    ac_prev=runstatedir ;;
-  -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
-  | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
-  | --run=* | --ru=* | --r=*)
-    runstatedir=$ac_optarg ;;
-
   -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
     ac_prev=sbindir ;;
   -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@@ -1277,7 +1266,7 @@ fi
 for ac_var in	exec_prefix prefix bindir sbindir libexecdir datarootdir \
 		datadir sysconfdir sharedstatedir localstatedir includedir \
 		oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
-		libdir localedir mandir runstatedir
+		libdir localedir mandir
 do
   eval ac_val=\$$ac_var
   # Remove trailing slashes.
@@ -1430,7 +1419,6 @@ Fine tuning of the installation directories:
   --sysconfdir=DIR        read-only single-machine data [PREFIX/etc]
   --sharedstatedir=DIR    modifiable architecture-independent data [PREFIX/com]
   --localstatedir=DIR     modifiable single-machine data [PREFIX/var]
-  --runstatedir=DIR       modifiable per-process data [LOCALSTATEDIR/run]
   --libdir=DIR            object code libraries [EPREFIX/lib]
   --includedir=DIR        C header files [PREFIX/include]
   --oldincludedir=DIR     C header files for non-gcc [/usr/include]
@@ -7865,9 +7853,9 @@ if test "x$ac_cv_lib_crypto_ERR_load_BIO_strings" = xyes; then :
   libcrypto_present=1 && LIBS="-lcrypto $LIBS"
 fi
 
-		{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for SSL_library_init in -lssl" >&5
-$as_echo_n "checking for SSL_library_init in -lssl... " >&6; }
-if ${ac_cv_lib_ssl_SSL_library_init+:} false; then :
+		{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for SSL_CTX_new in -lssl" >&5
+$as_echo_n "checking for SSL_CTX_new in -lssl... " >&6; }
+if ${ac_cv_lib_ssl_SSL_CTX_new+:} false; then :
   $as_echo_n "(cached) " >&6
 else
   ac_check_lib_save_LIBS=$LIBS
@@ -7881,27 +7869,27 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 #ifdef __cplusplus
 extern "C"
 #endif
-char SSL_library_init ();
+char SSL_CTX_new ();
 int
 main ()
 {
-return SSL_library_init ();
+return SSL_CTX_new ();
   ;
   return 0;
 }
 _ACEOF
 if ac_fn_c_try_link "$LINENO"; then :
-  ac_cv_lib_ssl_SSL_library_init=yes
+  ac_cv_lib_ssl_SSL_CTX_new=yes
 else
-  ac_cv_lib_ssl_SSL_library_init=no
+  ac_cv_lib_ssl_SSL_CTX_new=no
 fi
 rm -f core conftest.err conftest.$ac_objext \
     conftest$ac_exeext conftest.$ac_ext
 LIBS=$ac_check_lib_save_LIBS
 fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_SSL_library_init" >&5
-$as_echo "$ac_cv_lib_ssl_SSL_library_init" >&6; }
-if test "x$ac_cv_lib_ssl_SSL_library_init" = xyes; then :
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_SSL_CTX_new" >&5
+$as_echo "$ac_cv_lib_ssl_SSL_CTX_new" >&6; }
+if test "x$ac_cv_lib_ssl_SSL_CTX_new" = xyes; then :
   libssl_present=1 && LIBS="-lssl $LIBS"
 fi
 
@@ -7910,47 +7898,25 @@ fi
 $as_echo "OpenSSL library found, SSL support enabled" >&6; }
 
 			# Check if SRTP should be compiled with OpenSSL
-			# support, to enable cryptos such as AES GCM
-			{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for EVP_aes_128_gcm in -lcrypto" >&5
-$as_echo_n "checking for EVP_aes_128_gcm in -lcrypto... " >&6; }
-if ${ac_cv_lib_crypto_EVP_aes_128_gcm+:} false; then :
-  $as_echo_n "(cached) " >&6
-else
-  ac_check_lib_save_LIBS=$LIBS
-LIBS="-lcrypto  $LIBS"
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
+			# support, to enable cryptos such as AES GCM.
 
-/* Override any GCC internal prototype to avoid an error.
-   Use char because int might match the return type of a GCC
-   builtin and then its argument prototype would still apply.  */
-#ifdef __cplusplus
-extern "C"
-#endif
-char EVP_aes_128_gcm ();
+			# EVP_CIPHER_CTX is now opaque in OpenSSL 1.1.0, libsrtp 1.5.4 uses it as a transparent type.
+			# AC_CHECK_LIB(crypto,EVP_aes_128_gcm,[ac_ssl_has_aes_gcm=1])
+			cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <openssl/evp.h>
 int
 main ()
 {
-return EVP_aes_128_gcm ();
+EVP_CIPHER_CTX ctx;EVP_aes_128_gcm();
   ;
   return 0;
 }
 _ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
-  ac_cv_lib_crypto_EVP_aes_128_gcm=yes
-else
-  ac_cv_lib_crypto_EVP_aes_128_gcm=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
-    conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_crypto_EVP_aes_128_gcm" >&5
-$as_echo "$ac_cv_lib_crypto_EVP_aes_128_gcm" >&6; }
-if test "x$ac_cv_lib_crypto_EVP_aes_128_gcm" = xyes; then :
+if ac_fn_c_try_compile "$LINENO"; then :
   ac_ssl_has_aes_gcm=1
 fi
-
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
 			if test "x$ac_ssl_has_aes_gcm" = "x1"; then
 				{ $as_echo "$as_me:${as_lineno-$LINENO}: result: OpenSSL has AES GCM support, SRTP will use OpenSSL" >&5
 $as_echo "OpenSSL has AES GCM support, SRTP will use OpenSSL" >&6; }
diff --git a/aconfigure.ac b/aconfigure.ac
index 4d40f307..c7e8d45a 100644
--- a/aconfigure.ac
+++ b/aconfigure.ac
@@ -1555,13 +1555,18 @@ AC_ARG_ENABLE(ssl,
 		AC_SUBST(libcrypto_present)
 		AC_CHECK_HEADER(openssl/ssl.h,[openssl_h_present=1])
 		AC_CHECK_LIB(crypto,ERR_load_BIO_strings,[libcrypto_present=1 && LIBS="-lcrypto $LIBS"])
-		AC_CHECK_LIB(ssl,SSL_library_init,[libssl_present=1 && LIBS="-lssl $LIBS"])
+		AC_CHECK_LIB(ssl,SSL_CTX_new,[libssl_present=1 && LIBS="-lssl $LIBS"])
 		if test "x$openssl_h_present" = "x1" -a "x$libssl_present" = "x1" -a "x$libcrypto_present" = "x1"; then
 	        	AC_MSG_RESULT([OpenSSL library found, SSL support enabled])
 			
 			# Check if SRTP should be compiled with OpenSSL
-			# support, to enable cryptos such as AES GCM
-			AC_CHECK_LIB(crypto,EVP_aes_128_gcm,[ac_ssl_has_aes_gcm=1])
+			# support, to enable cryptos such as AES GCM.
+			
+			# EVP_CIPHER_CTX is now opaque in OpenSSL 1.1.0, libsrtp 1.5.4 uses it as a transparent type.
+			# AC_CHECK_LIB(crypto,EVP_aes_128_gcm,[ac_ssl_has_aes_gcm=1])
+			AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <openssl/evp.h>]],
+							  [EVP_CIPHER_CTX ctx;EVP_aes_128_gcm();])],
+					  [ac_ssl_has_aes_gcm=1])
 			if test "x$ac_ssl_has_aes_gcm" = "x1"; then
 				AC_MSG_RESULT([OpenSSL has AES GCM support, SRTP will use OpenSSL])
 			else
diff --git a/pjlib/src/pj/ssl_sock_ossl.c b/pjlib/src/pj/ssl_sock_ossl.c
index 15a243f8..86a9351a 100644
--- a/pjlib/src/pj/ssl_sock_ossl.c
+++ b/pjlib/src/pj/ssl_sock_ossl.c
@@ -45,6 +45,7 @@
 /* 
  * Include OpenSSL headers 
  */
+#include <openssl/asn1.h>
 #include <openssl/bio.h>
 #include <openssl/ssl.h>
 #include <openssl/err.h>
@@ -110,9 +111,21 @@ static unsigned get_nid_from_cid(unsigned cid)
 
 #endif
 
+
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+#  define OPENSSL_NO_SSL2	    /* seems to be removed in 1.1.0 */
+#  define M_ASN1_STRING_data(x)	    ASN1_STRING_get0_data(x)
+#  define M_ASN1_STRING_length(x)   ASN1_STRING_length(x)
+#else
+#  define SSL_CIPHER_get_id(c)	    (c)->id
+#  define SSL_set_session(ssl, s)   (ssl)->session = (s)
+#endif
+
+
 #ifdef _MSC_VER
 #  pragma comment( lib, "libeay32")
 #  pragma comment( lib, "ssleay32")
+#  pragma comment( lib, "crypt32")
 #endif
 
 
@@ -431,12 +444,13 @@ static pj_status_t init_openssl(void)
 	    const SSL_CIPHER *c;
 	    c = sk_SSL_CIPHER_value(sk_cipher,i);
 	    openssl_ciphers[i].id = (pj_ssl_cipher)
-				    (pj_uint32_t)c->id & 0x00FFFFFF;
+				    (pj_uint32_t)SSL_CIPHER_get_id(c) &
+				    0x00FFFFFF;
 	    openssl_ciphers[i].name = SSL_CIPHER_get_name(c);
 	}
 	openssl_cipher_num = n;
 
-	ssl->session = SSL_SESSION_new();
+	SSL_set_session(ssl, SSL_SESSION_new());
 
 #if !defined(OPENSSL_NO_EC) && OPENSSL_VERSION_NUMBER >= 0x1000200fL
 	openssl_curves_num = SSL_get_shared_curve(ssl,-1);
@@ -1013,7 +1027,8 @@ static pj_status_t set_cipher_list(pj_ssl_sock_t *ssock)
 	    const SSL_CIPHER *c;
 	    c = sk_SSL_CIPHER_value(sk_cipher, j);
 	    if (ssock->param.ciphers[i] == (pj_ssl_cipher)
-					   ((pj_uint32_t)c->id & 0x00FFFFFF))
+					   ((pj_uint32_t)SSL_CIPHER_get_id(c) &
+					   0x00FFFFFF))
 	    {
 		const char *c_name;
 
@@ -1066,7 +1081,7 @@ static pj_status_t set_curves_list(pj_ssl_sock_t *ssock)
 	curves[cnt] = get_nid_from_cid(ssock->param.curves[cnt]);
     }
 
-    if( ssock->ossl_ssl->server ) {
+    if( SSL_is_server(ssock->ossl_ssl) ) {
 	ret = SSL_set1_curves(ssock->ossl_ssl, curves,
 			      ssock->param.curves_num);
 	if (ret < 1)
@@ -1225,7 +1240,7 @@ static void get_cert_info(pj_pool_t *pool, pj_ssl_cert_info *ci, X509 *x,
     pj_bool_t update_needed;
     char buf[512];
     pj_uint8_t serial_no[64] = {0}; /* should be >= sizeof(ci->serial_no) */
-    pj_uint8_t *q;
+    const pj_uint8_t *q;
     unsigned len;
     GENERAL_NAMES *names = NULL;
 
@@ -1235,7 +1250,7 @@ static void get_cert_info(pj_pool_t *pool, pj_ssl_cert_info *ci, X509 *x,
     X509_NAME_oneline(X509_get_issuer_name(x), buf, sizeof(buf));
 
     /* Get serial no */
-    q = (pj_uint8_t*) M_ASN1_STRING_data(X509_get_serialNumber(x));
+    q = (const pj_uint8_t*) M_ASN1_STRING_data(X509_get_serialNumber(x));
     len = M_ASN1_STRING_length(X509_get_serialNumber(x));
     if (len > sizeof(ci->serial_no)) 
 	len = sizeof(ci->serial_no);
@@ -2642,7 +2657,7 @@ PJ_DEF(pj_status_t) pj_ssl_sock_get_info (pj_ssl_sock_t *ssock,
 
 	/* Current cipher */
 	cipher = SSL_get_current_cipher(ssock->ossl_ssl);
-	info->cipher = (cipher->id & 0x00FFFFFF);
+	info->cipher = (SSL_CIPHER_get_id(cipher) & 0x00FFFFFF);
 
 	/* Remote address */
 	pj_sockaddr_cp(&info->remote_addr, &ssock->rem_addr);