Fixed #1888: Support for WebRtc AEC

git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@5186 74dad513-b988-da41-8d7b-12977e46ad98

7 files changed, 482 insertions, 6 deletions

diff --git a/pjmedia/build/Makefile b/pjmedia/build/Makefile
index 702efdcb..26f09538 100644
--- a/pjmedia/build/Makefile
+++ b/pjmedia/build/Makefile
@@ -62,7 +62,7 @@ export PJMEDIA_OBJS += $(OS_OBJS) $(M_OBJS) $(CC_OBJS) $(HOST_OBJS) \
 			bidirectional.o clock_thread.o codec.o conference.o \
 			conf_switch.o converter.o  converter_libswscale.o converter_libyuv.o \
 			delaybuf.o echo_common.o \
-			echo_port.o echo_suppress.o endpoint.o errno.o \
+			echo_port.o echo_suppress.o echo_webrtc.o endpoint.o errno.o \
 			event.o format.o ffmpeg_util.o \
 			g711.o jbuf.o master_port.o mem_capture.o mem_player.o \
 			null_port.o plc_common.o port.o splitcomb.o \
diff --git a/pjmedia/build/os-auto.mak.in b/pjmedia/build/os-auto.mak.in
index 7a072745..a6280a28 100644
--- a/pjmedia/build/os-auto.mak.in
+++ b/pjmedia/build/os-auto.mak.in
@@ -37,14 +37,18 @@ LIBYUV_LDFLAGS = @ac_libyuv_ldflags@
 OPENH264_CFLAGS = @ac_openh264_cflags@
 OPENH264_LDFLAGS = @ac_openh264_ldflags@
 
+# WebRtc
+WEBRTC_CFLAGS = @ac_webrtc_cflags@
+WEBRTC_LDFLAGS = @ac_webrtc_ldflags@
+
 
 # PJMEDIA features exclusion
 export CFLAGS += @ac_no_small_filter@ @ac_no_large_filter@ @ac_no_speex_aec@ \
 		 $(SDL_CFLAGS) $(FFMPEG_CFLAGS) $(V4L2_CFLAGS) $(QT_CFLAGS) \
 		 $(IOS_CFLAGS) $(ANDROID_CFLAGS) $(LIBYUV_CFLAGS) \
-		 $(OPENH264_CFLAGS)
+		 $(OPENH264_CFLAGS) $(WEBRTC_CFLAGS)
 export LDFLAGS += $(SDL_LDFLAGS) $(FFMPEG_LDFLAGS) $(V4L2_LDFLAGS) \
-		  $(LIBYUV_LDFLAGS) $(OPENH264_LDFLAGS)
+		  $(LIBYUV_LDFLAGS) $(OPENH264_LDFLAGS) $(WEBRTC_LDFLAGS)
 
 # Define the desired sound device backend
 # Valid values are:
diff --git a/pjmedia/include/pjmedia/config.h b/pjmedia/include/pjmedia/config.h
index 8bd622ba..f2f8e829 100644
--- a/pjmedia/include/pjmedia/config.h
+++ b/pjmedia/include/pjmedia/config.h
@@ -657,6 +657,24 @@
 
 
 /**
+ * WebRtc Accoustic Echo Cancellation (AEC).
+ * By default is disabled.
+ */
+#ifndef PJMEDIA_HAS_WEBRTC_AEC
+#   define PJMEDIA_HAS_WEBRTC_AEC		0
+#endif
+
+/**
+ * Specify whether WebRtc EC should use its mobile version AEC.
+ *
+ * Default: 0 (no)
+ */
+#ifndef PJMEDIA_WEBRTC_AEC_USE_MOBILE
+#   define PJMEDIA_WEBRTC_AEC_USE_MOBILE 	0
+#endif
+
+
+/**
  * Maximum number of parameters in SDP fmtp attribute.
  *
  * Default: 16
diff --git a/pjmedia/include/pjmedia/echo.h b/pjmedia/include/pjmedia/echo.h
index 4ed62ee9..82c6809c 100644
--- a/pjmedia/include/pjmedia/echo.h
+++ b/pjmedia/include/pjmedia/echo.h
@@ -66,7 +66,8 @@ typedef enum pjmedia_echo_flag
 
     /**
      * Force to use Speex AEC as the backend echo canceller algorithm.
-     * This setting is mutually exclusive with PJMEDIA_ECHO_SIMPLE.
+     * This setting is mutually exclusive with PJMEDIA_ECHO_SIMPLE and
+     * PJMEDIA_ECHO_WEBRTC.
      */
     PJMEDIA_ECHO_SPEEX	= 1,
 
@@ -74,11 +75,18 @@ typedef enum pjmedia_echo_flag
      * If PJMEDIA_ECHO_SIMPLE flag is specified during echo canceller
      * creation, then a simple echo suppressor will be used instead of
      * an accoustic echo cancellation. This setting is mutually exclusive
-     * with PJMEDIA_ECHO_SPEEX.
+     * with PJMEDIA_ECHO_SPEEX and PJMEDIA_ECHO_WEBRTC.
      */
     PJMEDIA_ECHO_SIMPLE	= 2,
 
     /**
+     * Force to use WebRTC AEC as the backend echo canceller algorithm.
+     * This setting is mutually exclusive with PJMEDIA_ECHO_SIMPLE and
+     * PJMEDIA_ECHO_SPEEX.
+     */
+    PJMEDIA_ECHO_WEBRTC = 3,
+
+    /**
      * For internal use.
      */
     PJMEDIA_ECHO_ALGO_MASK = 15,
@@ -101,7 +109,46 @@ typedef enum pjmedia_echo_flag
      * If PJMEDIA_ECHO_USE_SW_ECHO flag is specified, software echo canceller
      * will be used instead of device EC.
      */
-    PJMEDIA_ECHO_USE_SW_ECHO = 64
+    PJMEDIA_ECHO_USE_SW_ECHO = 64,
+    
+    /**
+     * If PJMEDIA_ECHO_USE_NOISE_SUPPRESSOR flag is specified, the echo
+     * canceller will also apply noise suppressor method to reduce noise.
+     */
+    PJMEDIA_ECHO_USE_NOISE_SUPPRESSOR = 128,
+    
+    /**
+     * Use default aggressiveness setting for the echo canceller algorithm. 
+     * This setting is mutually exclusive with the other aggressiveness
+     * settings.
+     */
+    PJMEDIA_ECHO_AGGRESSIVENESS_DEFAULT = 0,
+    
+    /**
+     * Use conservative aggressiveness setting for the echo canceller
+     * algorithm. This setting is mutually exclusive with the other
+     * aggressiveness settings.
+     */
+    PJMEDIA_ECHO_AGGRESSIVENESS_CONSERVATIVE = 0x100,
+    
+    /**
+     * Use moderate aggressiveness setting for the echo canceller algorithm. 
+     * This setting is mutually exclusive with the other aggressiveness
+     * settings.
+     */
+    PJMEDIA_ECHO_AGGRESSIVENESS_MODERATE = 0x200,
+    
+    /**
+     * Use aggressive aggressiveness setting for the echo canceller
+     * algorithm. This setting is mutually exclusive with the other
+     * aggressiveness settings.
+     */
+    PJMEDIA_ECHO_AGGRESSIVENESS_AGGRESSIVE = 0x300,
+    
+    /**
+     * For internal use.
+     */
+    PJMEDIA_ECHO_AGGRESSIVENESS_MASK = 0xF00
 
 } pjmedia_echo_flag;
 
diff --git a/pjmedia/src/pjmedia/echo_common.c b/pjmedia/src/pjmedia/echo_common.c
index b6bacbec..66cfdf72 100644
--- a/pjmedia/src/pjmedia/echo_common.c
+++ b/pjmedia/src/pjmedia/echo_common.c
@@ -125,6 +125,20 @@ static struct ec_operations ipp_aec_op =
 #endif
 
 /*
+ * WebRTC AEC prototypes
+ */
+#if defined(PJMEDIA_HAS_WEBRTC_AEC) && PJMEDIA_HAS_WEBRTC_AEC!=0
+static struct ec_operations webrtc_aec_op =
+{
+    "WebRTC AEC",
+    &webrtc_aec_create,
+    &webrtc_aec_destroy,
+    &webrtc_aec_reset,
+    &webrtc_aec_cancel_echo
+};
+#endif
+
+/*
  * Create the echo canceller. 
  */
 PJ_DEF(pj_status_t) pjmedia_echo_create( pj_pool_t *pool,
@@ -185,6 +199,13 @@ PJ_DEF(pj_status_t) pjmedia_echo_create2(pj_pool_t *pool,
 
 #endif
 
+#if defined(PJMEDIA_HAS_WEBRTC_AEC) && PJMEDIA_HAS_WEBRTC_AEC!=0
+    } else if ((options & PJMEDIA_ECHO_ALGO_MASK) == PJMEDIA_ECHO_WEBRTC ||
+               (options & PJMEDIA_ECHO_ALGO_MASK) == PJMEDIA_ECHO_DEFAULT)
+    {
+        ec->op = &webrtc_aec_op;
+#endif
+        
     } else {
 	ec->op = &echo_supp_op;
     }
diff --git a/pjmedia/src/pjmedia/echo_internal.h b/pjmedia/src/pjmedia/echo_internal.h
index 1b0b9d57..bc75e1aa 100644
--- a/pjmedia/src/pjmedia/echo_internal.h
+++ b/pjmedia/src/pjmedia/echo_internal.h
@@ -77,6 +77,21 @@ PJ_DECL(pj_status_t) ipp_aec_cancel_echo(void *state,
 					 unsigned options,
 					 void *reserved );
 
+PJ_DECL(pj_status_t) webrtc_aec_create(pj_pool_t *pool,
+                                       unsigned clock_rate,
+                                       unsigned channel_count,
+                                       unsigned samples_per_frame,
+                                       unsigned tail_ms,
+                                       unsigned options,
+                                       void **p_echo );
+PJ_DECL(pj_status_t) webrtc_aec_destroy(void *state );
+PJ_DECL(void) webrtc_aec_reset(void *state );
+PJ_DECL(pj_status_t) webrtc_aec_cancel_echo(void *state,
+                                            pj_int16_t *rec_frm,
+                                            const pj_int16_t *play_frm,
+                                            unsigned options,
+                                            void *reserved );
+
 
 PJ_END_DECL
 
diff --git a/pjmedia/src/pjmedia/echo_webrtc.c b/pjmedia/src/pjmedia/echo_webrtc.c
new file mode 100644
index 00000000..b1355d04
--- /dev/null
+++ b/pjmedia/src/pjmedia/echo_webrtc.c
@@ -0,0 +1,371 @@
+/* $Id$ */
+/* 
+ * Copyright (C) 2011-2015 Teluu Inc. (http://www.teluu.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
+ */
+
+#include <pjmedia/echo.h>
+#include <pjmedia/errno.h>
+#include <pj/assert.h>
+#include <pj/log.h>
+#include <pj/pool.h>
+
+#if defined(PJMEDIA_HAS_WEBRTC_AEC) && PJMEDIA_HAS_WEBRTC_AEC != 0
+
+#include <webrtc/modules/audio_processing/aec/include/echo_cancellation.h>
+#include <webrtc/modules/audio_processing/aecm/include/echo_control_mobile.h>
+#include <webrtc/modules/audio_processing/aec/aec_core.h>
+
+#include "echo_internal.h"
+
+#define THIS_FILE		"echo_webrtc.c"
+
+#if PJMEDIA_WEBRTC_AEC_USE_MOBILE == 1
+    #include <webrtc/modules/audio_processing/ns/include/noise_suppression_x.h>
+
+    #define NsHandle NsxHandle
+    #define WebRtcNs_Create WebRtcNsx_Create
+    #define WebRtcNs_Init WebRtcNsx_Init
+    #define WebRtcNs_Free WebRtcNsx_Free
+
+    #define WebRtcAec_Create WebRtcAecm_Create
+    #define WebRtcAec_Init(handle, clock, sclock) WebRtcAecm_Init(handle, clock)
+    #define WebRtcAec_Free WebRtcAecm_Free
+    #define WebRtcAec_get_error_code WebRtcAecm_get_error_code
+    #define WebRtcAec_enable_delay_agnostic(core, enable)
+    #define WebRtcAec_set_config WebRtcAecm_set_config
+    #define WebRtcAec_BufferFarend WebRtcAecm_BufferFarend
+    #define AecConfig AecmConfig
+    #define SHOW_DELAY_METRICS	0
+    typedef short sample;
+#else
+    #include <webrtc/modules/audio_processing/ns/include/noise_suppression.h>
+
+    typedef float sample;
+
+    /* If SHOW_DELAY_METRICS is set to non-zero, delay metrics stats will
+     * be printed every SHOW_DELAY_METRICS-th call to webrtc_aec_cancel_echo().
+     * For example, if ptime is 20ms, set this to 250 to print the metrics
+     * every 250*20/1000=5 seconds.
+     */
+    #define SHOW_DELAY_METRICS	0
+    
+#endif
+
+#define BUF_LEN			160
+
+typedef struct webrtc_ec
+{
+    void       *AEC_inst;
+    NsHandle   *NS_inst;
+    unsigned    options;
+    unsigned	samples_per_frame;
+    unsigned	tail;
+    unsigned    clock_rate;
+    unsigned	channel_count;
+    unsigned    subframe_len;
+    sample      tmp_buf[BUF_LEN];
+    sample      tmp_buf2[BUF_LEN];
+#if SHOW_DELAY_METRICS
+    unsigned	counter;
+#endif
+} webrtc_ec;
+
+
+static void print_webrtc_aec_error(const char *tag, void *AEC_inst)
+{
+    unsigned status = WebRtcAec_get_error_code(AEC_inst);
+    PJ_LOG(3, (THIS_FILE, "WebRTC AEC error (%s) %d ", tag, status));
+}
+
+static void set_config(void *AEC_inst, unsigned options)
+{
+    unsigned aggr_opt = options & PJMEDIA_ECHO_AGGRESSIVENESS_MASK;
+    int status;
+    AecConfig aec_config;
+
+#if PJMEDIA_WEBRTC_AEC_USE_MOBILE
+    aec_config.echoMode = 3;
+    if (aggr_opt == PJMEDIA_ECHO_AGGRESSIVENESS_CONSERVATIVE)
+        aec_config.echoMode = 0;
+    else if (aggr_opt == PJMEDIA_ECHO_AGGRESSIVENESS_AGGRESSIVE)
+   	aec_config.echoMode = 4;
+    aec_config.cngMode = AecmTrue;
+#else
+    
+    aec_config.nlpMode = kAecNlpModerate;
+    if (aggr_opt == PJMEDIA_ECHO_AGGRESSIVENESS_CONSERVATIVE)
+	aec_config.nlpMode = kAecNlpConservative;
+    else if (aggr_opt == PJMEDIA_ECHO_AGGRESSIVENESS_AGGRESSIVE)
+        aec_config.nlpMode = kAecNlpAggressive;
+    else
+        aec_config.nlpMode = kAecNlpModerate;
+    
+    aec_config.skewMode = kAecFalse;
+#if SHOW_DELAY_METRICS
+    aec_config.metricsMode = kAecTrue;
+    aec_config.delay_logging = kAecTrue;
+#else
+    aec_config.metricsMode = kAecFalse;
+    aec_config.delay_logging = kAecFalse;
+#endif
+
+#endif
+
+    status = WebRtcAec_set_config(AEC_inst, aec_config);
+    if (status != 0) {
+        print_webrtc_aec_error("Init config", AEC_inst);
+    }
+}
+
+/*
+ * Create the AEC.
+ */
+PJ_DEF(pj_status_t) webrtc_aec_create(pj_pool_t *pool,
+                                      unsigned clock_rate,
+                                      unsigned channel_count,
+                                      unsigned samples_per_frame,
+                                      unsigned tail_ms,
+                                      unsigned options,
+                                      void **p_echo )
+{
+    webrtc_ec *echo;
+    int status;
+    
+    *p_echo = NULL;
+    
+    echo = PJ_POOL_ZALLOC_T(pool, webrtc_ec);
+    PJ_ASSERT_RETURN(echo != NULL, PJ_ENOMEM);
+    
+    /* Currently we only support mono. */
+    if (channel_count != 1)
+    	return PJ_ENOTSUP;
+
+    echo->channel_count = channel_count;
+    echo->samples_per_frame = samples_per_frame;
+    echo->tail = tail_ms;
+    echo->clock_rate = clock_rate;
+    /* SWB is processed as 160 frame size */
+    if (clock_rate > 8000)
+        echo->subframe_len = 160;
+    else
+    	echo->subframe_len = 80;
+    echo->options = options;
+    
+    /* Create WebRTC AEC */
+    echo->AEC_inst = WebRtcAec_Create();
+    if (!echo->AEC_inst) {
+    	return PJ_ENOMEM;
+    }
+    
+    /* Init WebRTC AEC */
+    status = WebRtcAec_Init(echo->AEC_inst, clock_rate, clock_rate);
+    if (status != 0) {
+        print_webrtc_aec_error("Init", echo->AEC_inst);
+        WebRtcAec_Free(echo->AEC_inst);
+    	return PJ_ENOTSUP;
+    }
+
+    /* WebRtc is very dependent on delay calculation, which will be passed
+     * to WebRtcAec_Process() below. A poor estimate, even by as little as
+     * 40ms, may affect the echo cancellation results greatly.
+     * Hence, we need to enable delay-agnostic echo cancellation. This
+     * low-level feature relies on internally estimated delays between
+     * the process and reverse streams, thus not relying on reported
+     * system delays.
+     * Still, with the delay agnostic feature, it may take some time (5-10s
+     * or more) for the Aec module to learn the optimal delay, thus
+     * a good initial estimate is necessary for good EC quality in
+     * the beginning of a call.
+     */
+    WebRtcAec_enable_delay_agnostic(WebRtcAec_aec_core(echo->AEC_inst), 1);
+    
+    set_config(echo->AEC_inst, options);
+
+    if (options & PJMEDIA_ECHO_USE_NOISE_SUPPRESSOR) {
+        echo->NS_inst = WebRtcNs_Create();
+        if (echo->NS_inst) {
+            status = WebRtcNs_Init(echo->NS_inst, clock_rate);
+            if (status != 0) {
+                WebRtcNs_Free(echo->NS_inst);
+                echo->NS_inst = NULL;
+            }
+        }
+        if (!echo->NS_inst) {
+            PJ_LOG(3, (THIS_FILE, "Unable to create WebRTC noise suppressor"));
+        }
+    }
+
+    PJ_LOG(3, (THIS_FILE, "WebRTC AEC%s successfully created with options %d",
+#if PJMEDIA_WEBRTC_AEC_USE_MOBILE
+			  " mobile", options));
+#else
+			  "", options));
+#endif
+
+    /* Done */
+    *p_echo = echo;
+    return PJ_SUCCESS;
+    
+}
+
+
+/*
+ * Destroy AEC
+ */
+PJ_DEF(pj_status_t) webrtc_aec_destroy(void *state )
+{
+    webrtc_ec *echo = (webrtc_ec*) state;
+    PJ_ASSERT_RETURN(echo, PJ_EINVAL);
+    
+    if (echo->AEC_inst) {
+    	WebRtcAec_Free(echo->AEC_inst);
+    	echo->AEC_inst = NULL;
+    }
+    if (echo->NS_inst) {
+        WebRtcNs_Free(echo->NS_inst);
+        echo->NS_inst = NULL;
+    }
+    
+    return PJ_SUCCESS;
+}
+
+
+/*
+ * Reset AEC
+ */
+PJ_DEF(void) webrtc_aec_reset(void *state )
+{
+    webrtc_ec *echo = (webrtc_ec*) state;
+    int status;
+    
+    pj_assert(echo != NULL);
+    
+    /* Re-initialize the EC */
+    status = WebRtcAec_Init(echo->AEC_inst, echo->clock_rate, echo->clock_rate);
+    if (status != 0) {
+        print_webrtc_aec_error("reset", echo->AEC_inst);
+        return;
+    }
+    
+    set_config(echo->AEC_inst, echo->options);
+    
+    PJ_LOG(4, (THIS_FILE, "WebRTC AEC reset succeeded"));
+}
+
+
+/*
+ * Perform echo cancellation.
+ */
+PJ_DEF(pj_status_t) webrtc_aec_cancel_echo( void *state,
+					    pj_int16_t *rec_frm,
+					    const pj_int16_t *play_frm,
+					    unsigned options,
+					    void *reserved )
+{
+    webrtc_ec *echo = (webrtc_ec*) state;
+    int status;
+    unsigned i, j, frm_idx = 0;
+    const sample * buf_ptr;
+    sample * out_buf_ptr;
+
+    /* Sanity checks */
+    PJ_ASSERT_RETURN(echo && rec_frm && play_frm, PJ_EINVAL);
+    
+    for(i = echo->samples_per_frame / echo->subframe_len; i > 0; i--) {
+#if PJMEDIA_WEBRTC_AEC_USE_MOBILE
+	buf_ptr = &play_frm[frm_idx];
+#else
+        for (j = 0; j < echo->subframe_len; j++) {
+            echo->tmp_buf[j] = rec_frm[frm_idx+j];
+            echo->tmp_buf2[j] = play_frm[frm_idx+j];
+        }
+        buf_ptr = echo->tmp_buf2;
+#endif
+        
+        /* Feed farend buffer */
+        status = WebRtcAec_BufferFarend(echo->AEC_inst, buf_ptr,
+                                        echo->subframe_len);
+        if (status != 0) {
+            print_webrtc_aec_error("Buffer farend", echo->AEC_inst);
+            return PJ_EUNKNOWN;
+        }        
+        
+	buf_ptr = echo->tmp_buf;
+	out_buf_ptr = echo->tmp_buf2;
+        if (echo->NS_inst) {
+#if PJMEDIA_WEBRTC_AEC_USE_MOBILE
+	    buf_ptr = &rec_frm[frm_idx];
+            WebRtcNsx_Process(echo->NS_inst, &buf_ptr, echo->channel_count,
+            		      &out_buf_ptr);
+            buf_ptr = out_buf_ptr;
+            out_buf_ptr = echo->tmp_buf;
+#else
+            WebRtcNs_Analyze(echo->NS_inst, buf_ptr);
+#endif
+        }
+        
+        /* Process echo cancellation */
+#if PJMEDIA_WEBRTC_AEC_USE_MOBILE
+        status = WebRtcAecm_Process(echo->AEC_inst, &rec_frm[frm_idx],
+        			    (echo->NS_inst? buf_ptr: NULL),
+        			    out_buf_ptr, echo->subframe_len,
+        			    echo->tail);
+#else
+        status = WebRtcAec_Process(echo->AEC_inst, &buf_ptr,
+                                   echo->channel_count, &out_buf_ptr,
+                                   echo->subframe_len, echo->tail, 0);
+#endif
+        if (status != 0) {
+            print_webrtc_aec_error("Process echo", echo->AEC_inst);
+            return PJ_EUNKNOWN;
+        }
+
+#if !PJMEDIA_WEBRTC_AEC_USE_MOBILE
+    	if (echo->NS_inst) {
+            /* Noise suppression */
+	    buf_ptr = echo->tmp_buf2;
+	    out_buf_ptr = echo->tmp_buf;
+            WebRtcNs_Process(echo->NS_inst, &buf_ptr,
+                             echo->channel_count, &out_buf_ptr);
+    	}
+#endif
+    
+       	for (j = 0; j < echo->subframe_len; j++) {
+ 	    rec_frm[frm_idx++] = (pj_int16_t)out_buf_ptr[j];
+    	}
+    }
+
+#if SHOW_DELAY_METRICS
+    if (++echo->counter >= SHOW_DELAY_METRICS) {
+    	int median, std;
+    	float frac_delay;
+
+        if (WebRtcAec_GetDelayMetrics(echo->AEC_inst, &median, &std,
+        			      &frac_delay) == 0)
+        {
+            PJ_LOG(3, (THIS_FILE, "WebRTC delay metrics: median=%d, std=%d, "
+            			  "fraction of poor delays=%f",
+            			  median, std, frac_delay));
+        }
+        echo->counter = 0;
+    }
+#endif
+
+    return PJ_SUCCESS;
+}
+
+#endif