Worked on the AEC. Apply constant delay bufferring for the AEC,

and also consider sound device latency when applying EC. It
sounds like working although it still doesn't perfectly cancel
the echo.

EC is now by default enabled in PJSUA.



git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@822 74dad513-b988-da41-8d7b-12977e46ad98

6 files changed, 310 insertions, 61 deletions

diff --git a/pjmedia/include/pjmedia/echo.h b/pjmedia/include/pjmedia/echo.h
index c872bce9..96e2bffe 100644
--- a/pjmedia/include/pjmedia/echo.h
+++ b/pjmedia/include/pjmedia/echo.h
@@ -82,6 +82,9 @@ typedef enum pjmedia_echo_flag
  * @param clock_rate	    Media clock rate/sampling rate.
  * @param samples_per_frame Number of samples per frame.
  * @param tail_ms	    Tail length, miliseconds.
+ * @param latency_ms	    Total lacency introduced by playback and 
+ *			    recording device. Set to zero if the latency
+ *			    is not known.
  * @param options	    Options. If PJMEDIA_ECHO_SIMPLE is specified,
  *			    then a simple echo suppressor implementation 
  *			    will be used instead of an accoustic echo 
@@ -95,6 +98,7 @@ PJ_DECL(pj_status_t) pjmedia_echo_create(pj_pool_t *pool,
 					 unsigned clock_rate,
 					 unsigned samples_per_frame,
 					 unsigned tail_ms,
+					 unsigned latency_ms,
 					 unsigned options,
 					 pjmedia_echo_state **p_echo );
 
diff --git a/pjmedia/include/pjmedia/echo_port.h b/pjmedia/include/pjmedia/echo_port.h
index d05c1a43..9b2d89e4 100644
--- a/pjmedia/include/pjmedia/echo_port.h
+++ b/pjmedia/include/pjmedia/echo_port.h
@@ -46,6 +46,9 @@ PJ_BEGIN_DECL
  * @param pool		Pool to allocate memory.
  * @param dn_port	Downstream port.
  * @param tail_ms	Tail length in miliseconds.
+ * @param latency_ms	Total lacency introduced by playback and 
+ *			recording device. Set to zero if the latency
+ *			is not known.
  * @param options	Options, as in #pjmedia_echo_create().
  * @param p_port	Pointer to receive the port instance.
  *
@@ -54,6 +57,7 @@ PJ_BEGIN_DECL
 PJ_DECL(pj_status_t) pjmedia_echo_port_create(pj_pool_t *pool,
 					      pjmedia_port *dn_port,
 					      unsigned tail_ms,
+					      unsigned latency_ms,
 					      unsigned options,
 					      pjmedia_port **p_port );
 
diff --git a/pjmedia/src/pjmedia/echo_common.c b/pjmedia/src/pjmedia/echo_common.c
index 7dad3037..af5b038f 100644
--- a/pjmedia/src/pjmedia/echo_common.c
+++ b/pjmedia/src/pjmedia/echo_common.c
@@ -37,6 +37,7 @@ struct ec_operations
 			    unsigned clock_rate,
 			    unsigned samples_per_frame,
 			    unsigned tail_ms,
+			    unsigned latency_ms,
 			    unsigned options,
 			    void **p_state );
     pj_status_t (*ec_destroy)(void *state );
@@ -61,6 +62,7 @@ PJ_DECL(pj_status_t) echo_supp_create(pj_pool_t *pool,
 				      unsigned clock_rate,
 				      unsigned samples_per_frame,
 				      unsigned tail_ms,
+				      unsigned latency_ms,
 				      unsigned options,
 				      void **p_state );
 PJ_DECL(pj_status_t) echo_supp_destroy(void *state);
@@ -94,6 +96,7 @@ PJ_DECL(pj_status_t) speex_aec_create(pj_pool_t *pool,
 				      unsigned clock_rate,
 				      unsigned samples_per_frame,
 				      unsigned tail_ms,
+				      unsigned latency_ms,
 				      unsigned options,
 				      void **p_state );
 PJ_DECL(pj_status_t) speex_aec_destroy(void *state );
@@ -130,6 +133,7 @@ PJ_DEF(pj_status_t) pjmedia_echo_create( pj_pool_t *pool,
 					 unsigned clock_rate,
 					 unsigned samples_per_frame,
 					 unsigned tail_ms,
+					 unsigned latency_ms,
 					 unsigned options,
 					 pjmedia_echo_state **p_echo )
 {
@@ -145,15 +149,14 @@ PJ_DEF(pj_status_t) pjmedia_echo_create( pj_pool_t *pool,
 
     if (options & PJMEDIA_ECHO_SIMPLE) {
 	ec->op = &echo_supp_op;
-	status = (*echo_supp_op.ec_create)(pool, clock_rate, 
-					   samples_per_frame,
-					   tail_ms, options,
+	status = (*echo_supp_op.ec_create)(pool, clock_rate, samples_per_frame,
+					   tail_ms, latency_ms, options,
 					   &ec->state);
     } else {
 	ec->op = &aec_op;
 	status = (*aec_op.ec_create)(pool, clock_rate, 
 				     samples_per_frame,
-				     tail_ms, options,
+				     tail_ms, latency_ms, options,
 				     &ec->state);
     }
 
diff --git a/pjmedia/src/pjmedia/echo_port.c b/pjmedia/src/pjmedia/echo_port.c
index 82fbca11..c9128c06 100644
--- a/pjmedia/src/pjmedia/echo_port.c
+++ b/pjmedia/src/pjmedia/echo_port.c
@@ -46,6 +46,7 @@ static pj_status_t ec_on_destroy(pjmedia_port *this_port);
 PJ_DEF(pj_status_t) pjmedia_echo_port_create(pj_pool_t *pool,
 					     pjmedia_port *dn_port,
 					     unsigned tail_ms,
+					     unsigned latency_ms,
 					     unsigned options,
 					     pjmedia_port **p_port )
 {
@@ -68,7 +69,7 @@ PJ_DEF(pj_status_t) pjmedia_echo_port_create(pj_pool_t *pool,
 
     status = pjmedia_echo_create(pool, dn_port->info.clock_rate, 
 				 dn_port->info.samples_per_frame,
-				 tail_ms, options, &ec->ec);
+				 tail_ms, latency_ms, options, &ec->ec);
     if (status != PJ_SUCCESS)
 	return status;
 
diff --git a/pjmedia/src/pjmedia/echo_speex.c b/pjmedia/src/pjmedia/echo_speex.c
index bf8f3f72..af5f2a95 100644
--- a/pjmedia/src/pjmedia/echo_speex.c
+++ b/pjmedia/src/pjmedia/echo_speex.c
@@ -29,8 +29,201 @@
 #include <speex/speex_preprocess.h>
 
 
-#define THIS_FILE   "echo_speex.c"
-#define BUF_COUNT   8
+#define THIS_FILE	"echo_speex.c"
+#define BUF_COUNT	20
+#define MIN_PREFETCH	4
+#define MAX_PREFETCH	12
+
+
+
+#if 0
+# define TRACE_(expr)  PJ_LOG(5,expr)
+#else
+# define TRACE_(expr)
+#endif
+
+
+typedef struct pjmedia_frame_queue pjmedia_frame_queue;
+
+struct fq_frame
+{
+    PJ_DECL_LIST_MEMBER(struct fq_frame);
+    void	*buf;
+    unsigned	 size;
+    pj_uint32_t	 seq;
+};
+
+struct pjmedia_frame_queue
+{
+    char	     obj_name[PJ_MAX_OBJ_NAME];
+    unsigned	     frame_size;
+    int		     samples_per_frame;
+    unsigned	     count;
+    unsigned	     max_count;
+    struct fq_frame  frame_list;
+    struct fq_frame  free_list;
+
+    int		     seq_delay;
+    int		     prefetch_count;
+};
+
+PJ_DEF(pj_status_t) pjmedia_frame_queue_create( pj_pool_t *pool,
+					        const char *name,
+					        unsigned frame_size,
+					        unsigned samples_per_frame,
+					        unsigned max_count,
+					        pjmedia_frame_queue **p_fq)
+{
+    pjmedia_frame_queue *fq;
+    unsigned i;
+
+    fq = pj_pool_zalloc(pool, sizeof(pjmedia_frame_queue));
+
+    pj_ansi_snprintf(fq->obj_name, sizeof(fq->obj_name), name, fq);
+    fq->obj_name[sizeof(fq->obj_name)-1] = '\0';
+
+    fq->max_count = max_count;
+    fq->frame_size = frame_size;
+    fq->samples_per_frame = samples_per_frame;
+    fq->count = 0;
+
+    pj_list_init(&fq->frame_list);
+    pj_list_init(&fq->free_list);
+
+    for (i=0; i<max_count; ++i) {
+	struct fq_frame *f;
+
+	f = pj_pool_zalloc(pool, sizeof(struct fq_frame));
+	f->buf = pj_pool_alloc(pool, frame_size);
+
+	pj_list_push_back(&fq->free_list, f);
+	
+    }
+
+    *p_fq = fq;
+    return PJ_SUCCESS;
+}
+
+PJ_DEF(pj_status_t) pjmedia_frame_queue_init( pjmedia_frame_queue *fq,
+					      int seq_delay,
+					      int prefetch_count)
+{
+    if (prefetch_count > MAX_PREFETCH)
+	prefetch_count = MAX_PREFETCH;
+
+    fq->seq_delay = seq_delay;
+    fq->prefetch_count = prefetch_count;
+    fq->count = 0;
+    pj_list_merge_first(&fq->free_list, &fq->frame_list);
+
+    PJ_LOG(5,(fq->obj_name, "AEC reset, delay=%d, prefetch=%d", 
+	      fq->seq_delay, fq->prefetch_count));
+
+    return PJ_SUCCESS;
+}
+
+PJ_DEF(pj_bool_t) pjmedia_frame_queue_empty( pjmedia_frame_queue *fq )
+{
+    return pj_list_empty(&fq->frame_list);
+}
+
+PJ_DEF(int) pjmedia_frame_queue_get_prefetch( pjmedia_frame_queue *fq )
+{
+    return fq->prefetch_count;
+}
+
+PJ_DEF(pj_status_t) pjmedia_frame_queue_put( pjmedia_frame_queue *fq,
+					     const void *framebuf,
+					     unsigned size,
+					     pj_uint32_t timestamp )
+{
+    struct fq_frame *f;
+
+    TRACE_((fq->obj_name, "PUT seq=%d, count=%d", 
+	    timestamp / fq->samples_per_frame, fq->count));
+
+    if (pj_list_empty(&fq->free_list)) {
+	PJ_LOG(5,(fq->obj_name, 
+		  " AEC info: queue is full, frame discarded "
+		  "[count=%d, seq=%d]",
+		  fq->max_count, timestamp / fq->samples_per_frame));
+	return PJ_ETOOMANY;
+    }
+
+    PJ_ASSERT_RETURN(size <= fq->frame_size, PJ_ETOOBIG);
+
+    f = fq->free_list.next;
+    pj_list_erase(f);
+
+    pj_memcpy(f->buf, framebuf, size);
+    f->size = size;
+    f->seq = timestamp / fq->samples_per_frame;
+
+    pj_list_push_back(&fq->frame_list, f);
+    ++fq->count;
+
+    return PJ_SUCCESS;
+}
+
+PJ_DEF(pj_status_t) pjmedia_frame_queue_get( pjmedia_frame_queue *fq,
+					     pj_uint32_t get_timestamp,
+					     void **framebuf,
+					     unsigned *size )
+{
+    pj_uint32_t frame_seq;
+    struct fq_frame *f;
+
+    frame_seq = get_timestamp/fq->samples_per_frame + fq->seq_delay -
+		fq->prefetch_count;
+
+    TRACE_((fq->obj_name, "GET seq=%d for seq=%d delay=%d, prefetch=%d", 
+	    get_timestamp/fq->samples_per_frame, frame_seq, fq->seq_delay, 
+	    fq->prefetch_count));
+
+    *size = 0;
+
+    /* Remove old frames */
+    for (;!pj_list_empty(&fq->frame_list);) {
+	f = fq->frame_list.next;
+	if (f->seq >= frame_seq)
+	    break;
+
+	PJ_LOG(5,(fq->obj_name, 
+		  " AEC Info: old frame removed (seq=%d, want=%d, count=%d)",
+		  f->seq, frame_seq, fq->count));
+	pj_list_erase(f);
+	--fq->count;
+	pj_list_push_back(&fq->free_list, f);
+    }
+
+    if (pj_list_empty(&fq->frame_list)) {
+	PJ_LOG(5,(fq->obj_name, 
+		  " AEC Info: empty queue for seq=%d!",
+		  frame_seq));
+	return PJ_ENOTFOUND;
+    }
+
+    f = fq->frame_list.next;
+
+    if (f->seq > frame_seq) {
+	PJ_LOG(5,(fq->obj_name, 
+		  " AEC Info: prefetching (first seq=%d)",
+		  f->seq));
+	return -1;
+    }
+
+    pj_list_erase(f);
+    --fq->count;
+
+    *framebuf = (void*)f->buf;
+    *size = f->size;
+
+    TRACE_((fq->obj_name, " returning frame with seq=%d, count=%d", 
+	    f->seq, fq->count));
+
+    pj_list_push_front(&fq->free_list, f);
+    return PJ_SUCCESS;
+}
 
 /*
  * Prototypes
@@ -39,6 +232,7 @@ PJ_DECL(pj_status_t) speex_aec_create(pj_pool_t *pool,
 				      unsigned clock_rate,
 				      unsigned samples_per_frame,
 				      unsigned tail_ms,
+				      unsigned latency_ms,
 				      unsigned options,
 				      void **p_state );
 PJ_DECL(pj_status_t) speex_aec_destroy(void *state );
@@ -54,9 +248,11 @@ PJ_DECL(pj_status_t) speex_aec_cancel_echo(void *state,
 					   void *reserved );
 
 
-struct frame
+enum
 {
-    pj_int16_t	   *buf;
+    TS_FLAG_PLAY = 1,
+    TS_FLAG_REC	 = 2,
+    TS_FLAG_OK	 = 3,
 };
 
 typedef struct speex_ec
@@ -64,16 +260,18 @@ typedef struct speex_ec
     SpeexEchoState	 *state;
     SpeexPreprocessState *preprocess;
 
-    unsigned	     samples_per_frame;
-    unsigned	     options;
-    pj_int16_t	    *tmp_frame;
-    spx_int32_t	    *residue;
+    unsigned		  samples_per_frame;
+    unsigned		  prefetch;
+    unsigned		  options;
+    pj_int16_t		 *tmp_frame;
+    spx_int32_t		 *residue;
 
-    pj_lock_t	    *lock;		/* To protect buffers, if required  */
+    pj_uint32_t		  play_ts,
+			  rec_ts,
+			  ts_flag;
 
-    unsigned	     rpos;		/* Index to get oldest frame.	    */
-    unsigned	     wpos;		/* Index to put newest frame.	    */
-    struct frame     frames[BUF_COUNT];	/* Playback frame buffers.	    */
+    pjmedia_frame_queue	 *frame_queue;
+    pj_lock_t		 *lock;		/* To protect buffers, if required  */
 } speex_ec;
 
 
@@ -85,13 +283,13 @@ PJ_DEF(pj_status_t) speex_aec_create(pj_pool_t *pool,
 				     unsigned clock_rate,
 				     unsigned samples_per_frame,
 				     unsigned tail_ms,
+				     unsigned latency_ms,
 				     unsigned options,
 				     void **p_echo )
 {
     speex_ec *echo;
     int sampling_rate;
-    unsigned i;
-    int disabled;
+    int disabled, enabled;
     pj_status_t status;
 
     *p_echo = NULL;
@@ -110,6 +308,11 @@ PJ_DEF(pj_status_t) speex_aec_create(pj_pool_t *pool,
     }
 
     echo->samples_per_frame = samples_per_frame;
+    echo->prefetch = (latency_ms * clock_rate / 1000) / samples_per_frame;
+    if (echo->prefetch < MIN_PREFETCH)
+	echo->prefetch = MIN_PREFETCH;
+    if (echo->prefetch > MAX_PREFETCH)
+	echo->prefetch = MAX_PREFETCH;
     echo->options = options;
 
     echo->state = speex_echo_state_init(samples_per_frame,
@@ -129,8 +332,9 @@ PJ_DEF(pj_status_t) speex_aec_create(pj_pool_t *pool,
 
     /* Disable all preprocessing, we only want echo cancellation */
     disabled = 0;
+    enabled = 1;
     speex_preprocess_ctl(echo->preprocess, SPEEX_PREPROCESS_SET_DENOISE, 
-			 &disabled);
+			 &enabled);
     speex_preprocess_ctl(echo->preprocess, SPEEX_PREPROCESS_SET_AGC, 
 			 &disabled);
     speex_preprocess_ctl(echo->preprocess, SPEEX_PREPROCESS_SET_VAD, 
@@ -149,24 +353,27 @@ PJ_DEF(pj_status_t) speex_aec_create(pj_pool_t *pool,
 
     /* Create temporary frame to receive residue */
     echo->residue = pj_pool_zalloc(pool, sizeof(spx_int32_t) * 
-					    samples_per_frame);
+					    (samples_per_frame+1));
     PJ_ASSERT_RETURN(echo->residue != NULL, PJ_ENOMEM);
 
-    /* Create internal playback buffers */
-    for (i=0; i<BUF_COUNT; ++i) {
-	echo->frames[i].buf = pj_pool_zalloc(pool, samples_per_frame * 2);
-	PJ_ASSERT_RETURN(echo->frames[i].buf != NULL, PJ_ENOMEM);
+    /* Create frame queue */
+    status = pjmedia_frame_queue_create(pool, "aec%p", samples_per_frame*2,
+					samples_per_frame, BUF_COUNT, 
+					&echo->frame_queue);
+    if (status != PJ_SUCCESS) {
+	speex_preprocess_state_destroy(echo->preprocess);
+	speex_echo_state_destroy(echo->state);
+	pj_lock_destroy(echo->lock);
+	return status;
     }
 
-
     /* Done */
     *p_echo = echo;
 
     PJ_LOG(4,(THIS_FILE, "Speex Echo canceller/AEC created, clock_rate=%d, "
-			 "samples per frame=%d, tail length=%d ms", 
-			 clock_rate,
-			 samples_per_frame,
-			 tail_ms));
+			 "samples per frame=%d, tail length=%d ms, "
+			 "latency=%d ms", 
+			 clock_rate, samples_per_frame, tail_ms, latency_ms));
     return PJ_SUCCESS;
 
 }
@@ -215,23 +422,29 @@ PJ_DEF(pj_status_t) speex_aec_playback(void *state,
     PJ_ASSERT_RETURN(echo && play_frm, PJ_EINVAL);
 
     /* The AEC must be configured to support internal playback buffer */
-    PJ_ASSERT_RETURN(echo->frames[0].buf != NULL, PJ_EINVALIDOP);
+    PJ_ASSERT_RETURN(echo->frame_queue!= NULL, PJ_EINVALIDOP);
 
     pj_lock_acquire(echo->lock);
 
-    /* Check for overflows */
-    if (echo->wpos == echo->rpos) {
-	PJ_LOG(5,(THIS_FILE, "Speex AEC overflow (playback runs faster, "
-			     "wpos=%d, rpos=%d)",
-			     echo->wpos, echo->rpos));
-	echo->rpos = (echo->wpos - BUF_COUNT/2) % BUF_COUNT;
-	speex_echo_state_reset(echo->state);
+    /* Inc timestamp */
+    echo->play_ts += echo->samples_per_frame;
+
+    /* Initialize frame delay. */
+    if ((echo->ts_flag & TS_FLAG_PLAY) == 0) {
+	echo->ts_flag |= TS_FLAG_PLAY;
+
+	if (echo->ts_flag == TS_FLAG_OK) {
+	    int seq_delay;
+
+	    seq_delay = ((int)echo->play_ts - (int)echo->rec_ts) / 
+			    (int)echo->samples_per_frame;
+	    pjmedia_frame_queue_init(echo->frame_queue, seq_delay, 
+				     echo->prefetch);
+	}
     }
 
-    /* Save fhe frame */
-    pjmedia_copy_samples(echo->frames[echo->wpos].buf,
-			 play_frm, echo->samples_per_frame);
-    echo->wpos = (echo->wpos+1) % BUF_COUNT;
+    pjmedia_frame_queue_put(echo->frame_queue, play_frm, 
+			    echo->samples_per_frame*2, echo->play_ts);
 
     pj_lock_release(echo->lock);
 
@@ -247,42 +460,64 @@ PJ_DEF(pj_status_t) speex_aec_capture( void *state,
 				       unsigned options )
 {
     speex_ec *echo = state;
-    pj_status_t status;
+    pj_status_t status = PJ_SUCCESS;
 
     /* Sanity checks */
     PJ_ASSERT_RETURN(echo && rec_frm, PJ_EINVAL);
 
     /* The AEC must be configured to support internal playback buffer */
-    PJ_ASSERT_RETURN(echo->frames[0].buf != NULL, PJ_EINVALIDOP);
+    PJ_ASSERT_RETURN(echo->frame_queue!= NULL, PJ_EINVALIDOP);
 
     /* Lock mutex */
     pj_lock_acquire(echo->lock);
 
+    /* Inc timestamp */
+    echo->rec_ts += echo->samples_per_frame;
 
-    /* Check for underflow */
-    if (echo->rpos == echo->wpos) {
-	/* Return frame as it is */
-	pj_lock_release(echo->lock);
+    /* Init frame delay. */
+    if ((echo->ts_flag & TS_FLAG_REC) == 0) {
+	echo->ts_flag |= TS_FLAG_REC;
 
-	PJ_LOG(5,(THIS_FILE, "Speex AEC underflow (capture runs faster than "
-			     "playback, wpos=%d, rpos=%d)", 
-			     echo->wpos, echo->rpos));
-	echo->rpos = (echo->wpos - BUF_COUNT/2) % BUF_COUNT;
-	speex_echo_state_reset(echo->state);
+	if (echo->ts_flag == TS_FLAG_OK) {
+	    int seq_delay;
 
-	return PJ_SUCCESS;
+	    seq_delay = ((int)echo->play_ts - (int)echo->rec_ts) / 
+			    (int)echo->samples_per_frame;
+	    pjmedia_frame_queue_init(echo->frame_queue, seq_delay, 
+				     echo->prefetch);
+	}
     }
-    
 
     /* Cancel echo */
-    status = speex_aec_cancel_echo(echo, rec_frm, 
-				   echo->frames[echo->rpos].buf, options,
-				   NULL);
+    if (echo->ts_flag == TS_FLAG_OK) {
+	void *play_buf;
+	unsigned size = 0;
+	
+	if (pjmedia_frame_queue_empty(echo->frame_queue)) {
+	    int seq_delay, prefetch;
+
+	    seq_delay = ((int)echo->play_ts - (int)echo->rec_ts) / 
+			    (int)echo->samples_per_frame;
+	    prefetch = pjmedia_frame_queue_get_prefetch(echo->frame_queue);
+	    //++prefetch;
+	    pjmedia_frame_queue_init(echo->frame_queue, seq_delay, prefetch);
+	    status = -1;
+
+	} else {
+	    status = pjmedia_frame_queue_get(echo->frame_queue, echo->rec_ts,
+					     &play_buf, &size);
+	    if (size != 0) {
+		speex_aec_cancel_echo(echo, rec_frm, (pj_int16_t*)play_buf,
+				      options, NULL);
+	    }	
+	}
 
-    echo->rpos = (echo->rpos + 1) % BUF_COUNT;
+	if (status != PJ_SUCCESS)
+	    speex_echo_state_reset(echo->state);
+    }
 
     pj_lock_release(echo->lock);
-    return status;
+    return PJ_SUCCESS;
 }
 
 
@@ -319,3 +554,4 @@ PJ_DEF(pj_status_t) speex_aec_cancel_echo( void *state,
 
 }
 
+ 
\ No newline at end of file
diff --git a/pjmedia/src/pjmedia/sound_port.c b/pjmedia/src/pjmedia/sound_port.c
index 02478e06..afb4effa 100644
--- a/pjmedia/src/pjmedia/sound_port.c
+++ b/pjmedia/src/pjmedia/sound_port.c
@@ -492,7 +492,8 @@ PJ_DEF(pj_status_t) pjmedia_snd_port_set_ec( pjmedia_snd_port *snd_port,
 		   snd_port->clock_rate;
 	status = pjmedia_echo_create(pool, snd_port->clock_rate, 
 				    snd_port->samples_per_frame, 
-				    tail_ms, options, &snd_port->ec_state);
+				    tail_ms, delay_ms,
+				    options, &snd_port->ec_state);
 	if (status != PJ_SUCCESS)
 	    snd_port->ec_state = NULL;
 	else