Initial commit for ticket #929: Improve packet lost concealment (PLC) when handling burst of lost packets

WSOLA improvements:
 - Introduce fade-out and fade-in effect
 - Limit the number of continuous synthetic samples (only take effect when fading is used)
 - Export many settings as macros:
   - PJMEDIA_WSOLA_DELAY_MSEC (was HANNING_PTIME)
   - PJMEDIA_WSOLA_TEMPLATE_LENGTH_MSEC (was TEMPLATE_PTIME)
   - PJMEDIA_WSOLA_MAX_EXPAND_MSEC

PLC:
 - added compile time macro PJMEDIA_WSOLA_PLC_NO_FADING to disable fading (default enabled)

Stream:
 - fixed bug when stream is not PLC-ing subsequent packet loss (only the first)
 - also add maximum PLC limit just as precaution if PLC doesn't limit number of synthetic frames
 - unrelated: fixed warning about unused send_keep_alive() function

Delaybuf:
 - modified to NOT use fading in WSOLA since we don't expect it to generate many continuous synthetic frames



git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@2850 74dad513-b988-da41-8d7b-12977e46ad98

4 files changed, 218 insertions, 22 deletions

diff --git a/pjmedia/src/pjmedia/delaybuf.c b/pjmedia/src/pjmedia/delaybuf.c
index e39cda79..c0a316fd 100644
--- a/pjmedia/src/pjmedia/delaybuf.c
+++ b/pjmedia/src/pjmedia/delaybuf.c
@@ -128,7 +128,7 @@ PJ_DEF(pj_status_t) pjmedia_delay_buf_create( pj_pool_t *pool,
 
     /* Create WSOLA */
     status = pjmedia_wsola_create(pool, clock_rate, samples_per_frame, 1,
-				  0, &b->wsola);
+				  PJMEDIA_WSOLA_NO_FADING, &b->wsola);
     if (status != PJ_SUCCESS)
 	return status;
 
diff --git a/pjmedia/src/pjmedia/plc_common.c b/pjmedia/src/pjmedia/plc_common.c
index aa9b685e..ea54accd 100644
--- a/pjmedia/src/pjmedia/plc_common.c
+++ b/pjmedia/src/pjmedia/plc_common.c
@@ -125,6 +125,7 @@ static void* plc_wsola_create(pj_pool_t *pool, unsigned clock_rate,
 			      unsigned samples_per_frame)
 {
     struct wsola_plc *o;
+    unsigned flag;
     pj_status_t status;
 
     PJ_UNUSED_ARG(clock_rate);
@@ -132,8 +133,12 @@ static void* plc_wsola_create(pj_pool_t *pool, unsigned clock_rate,
     o = PJ_POOL_ZALLOC_T(pool, struct wsola_plc);
     o->prev_lost = PJ_FALSE;
 
+    flag = PJMEDIA_WSOLA_NO_DISCARD;
+    if (PJMEDIA_WSOLA_PLC_NO_FADING)
+	flag |= PJMEDIA_WSOLA_NO_FADING;
+
     status = pjmedia_wsola_create(pool, clock_rate, samples_per_frame, 1,
-				  PJMEDIA_WSOLA_NO_DISCARD, &o->wsola);
+				  flag, &o->wsola);
     if (status != PJ_SUCCESS)
 	return NULL;
 
diff --git a/pjmedia/src/pjmedia/stream.c b/pjmedia/src/pjmedia/stream.c
index 587370e3..36490839 100644
--- a/pjmedia/src/pjmedia/stream.c
+++ b/pjmedia/src/pjmedia/stream.c
@@ -43,6 +43,13 @@
 
 #define BYTES_PER_SAMPLE		2
 
+/* Limit the number of synthetic audio samples that are generated by PLC.
+ * Normally PLC should have it's own means to limit the number of
+ * synthetic frames, so we need to set this to a reasonably large value
+ * just as precaution
+ */
+#define MAX_PLC_MSEC			240
+
 /**
  * Media channel.
  */
@@ -99,6 +106,9 @@ struct pjmedia_stream
     unsigned		     enc_buf_count; /**< Number of samples in the
 						 encoding buffer.	    */
 
+    unsigned		     plc_cnt;	    /**< # of consecutive PLC frames*/
+    unsigned		     max_plc_cnt;   /**< Max # of PLC frames	    */
+
     unsigned		     vad_enabled;   /**< VAD enabled in param.	    */
     unsigned		     frame_size;    /**< Size of encoded base frame.*/
     pj_bool_t		     is_streaming;  /**< Currently streaming?. This
@@ -197,13 +207,13 @@ static void stream_perror(const char *sender, const char *title,
     PJ_LOG(4,(sender, "%s: %s [err:%d]", title, errmsg, status));
 }
 
+#if defined(PJMEDIA_STREAM_ENABLE_KA) && PJMEDIA_STREAM_ENABLE_KA != 0
 /*
- * Send keep-alive packet.
+ * Send keep-alive packet using non-codec frame.
  */
 static void send_keep_alive_packet(pjmedia_stream *stream)
 {
-#if defined(PJMEDIA_STREAM_ENABLE_KA) && \
-    PJMEDIA_STREAM_ENABLE_KA == PJMEDIA_STREAM_KA_EMPTY_RTP
+#if PJMEDIA_STREAM_ENABLE_KA == PJMEDIA_STREAM_KA_EMPTY_RTP
 
     /* Keep-alive packet is empty RTP */
     pj_status_t status;
@@ -224,8 +234,7 @@ static void send_keep_alive_packet(pjmedia_stream *stream)
 			       pkt_len);
     TRC_((stream->port.info.name.ptr, "Keep-alive sent (empty RTP)"));
 
-#elif defined(PJMEDIA_STREAM_ENABLE_KA) && \
-      PJMEDIA_STREAM_ENABLE_KA == PJMEDIA_STREAM_KA_USER
+#elif PJMEDIA_STREAM_ENABLE_KA == PJMEDIA_STREAM_KA_USER
 
     /* Keep-alive packet is defined in PJMEDIA_STREAM_KA_USER_PKT */
     int pkt_len;
@@ -243,6 +252,7 @@ static void send_keep_alive_packet(pjmedia_stream *stream)
 
 #endif
 }
+#endif	/* defined(PJMEDIA_STREAM_ENABLE_KA) */
 
 /*
  * play_callback()
@@ -294,7 +304,8 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame)
 	    
 	    /* Activate PLC */
 	    if (stream->codec->op->recover && 
-		stream->codec_param.setting.plc) 
+		stream->codec_param.setting.plc &&
+		stream->plc_cnt < stream->max_plc_cnt) 
 	    {
 		pjmedia_frame frame_out;
 
@@ -304,6 +315,8 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame)
 						       frame_out.size,
 						       &frame_out);
 
+		++stream->plc_cnt;
+
 	    } else {
 		status = -1;
 	    }
@@ -318,20 +331,24 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame)
 		PJ_LOG(5,(stream->port.info.name.ptr, 
 			  "Lost frame recovered"));
 	    }
-	    
+
 	} else if (frame_type == PJMEDIA_JB_ZERO_EMPTY_FRAME) {
 
 	    /* Jitter buffer is empty. If this is the first "empty" state,
 	     * activate PLC to smoothen the fade-out, otherwise zero
 	     * the frame. 
 	     */
-	    if (frame_type != stream->jb_last_frm) {
+	    //Using this "if" will only invoke PLC for the first packet
+	    //lost and not the subsequent ones.
+	    //if (frame_type != stream->jb_last_frm) {
+	    if (1) {
 		pjmedia_jb_state jb_state;
 		const char *with_plc = "";
 
 		/* Activate PLC to smoothen the missing frame */
 		if (stream->codec->op->recover && 
-		    stream->codec_param.setting.plc) 
+		    stream->codec_param.setting.plc &&
+		    stream->plc_cnt < stream->max_plc_cnt) 
 		{
 		    pjmedia_frame frame_out;
 
@@ -343,9 +360,12 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame)
 							       &frame_out);
 			if (status != PJ_SUCCESS)
 			    break;
+
 			samples_count += samples_per_frame;
+			++stream->plc_cnt;
 
-		    } while (samples_count < samples_required);
+		    } while (samples_count < samples_required &&
+			     stream->plc_cnt < stream->max_plc_cnt);
 
 		    with_plc = ", plc invoked";
 		} 
@@ -379,7 +399,8 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame)
 
 	    /* Always activate PLC when it's available.. */
 	    if (stream->codec->op->recover && 
-		stream->codec_param.setting.plc) 
+		stream->codec_param.setting.plc &&
+		stream->plc_cnt < stream->max_plc_cnt) 
 	    {
 		pjmedia_frame frame_out;
 
@@ -393,9 +414,13 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame)
 			break;
 		    samples_count += samples_per_frame;
 
-		} while (samples_count < samples_required);
+		    ++stream->plc_cnt;
 
-		if (stream->jb_last_frm != frame_type) {
+		} while (samples_count < samples_required &&
+			 stream->plc_cnt < stream->max_plc_cnt);
+
+		//if (stream->jb_last_frm != frame_type) {
+		if (1) {
 		    PJ_LOG(5,(stream->port.info.name.ptr, 
 			      "Jitter buffer is bufferring with plc (prefetch=%d)",
 			      jb_state.prefetch));
@@ -419,6 +444,8 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame)
 	    /* Got "NORMAL" frame from jitter buffer */
 	    pjmedia_frame frame_in, frame_out;
 
+	    stream->plc_cnt = 0;
+
 	    /* Decode */
 	    frame_in.buf = channel->out_pkt;
 	    frame_in.size = frame_size;
@@ -1822,6 +1849,10 @@ PJ_DEF(pj_status_t) pjmedia_stream_create( pjmedia_endpt *endpt,
 	++stream->frame_size;
     }
 
+    /* How many consecutive PLC frames can be generated */
+    stream->max_plc_cnt = (MAX_PLC_MSEC+stream->codec_param.info.frm_ptime-1)/
+			    stream->codec_param.info.frm_ptime;
+
 #if defined(PJMEDIA_HANDLE_G722_MPEG_BUG) && (PJMEDIA_HANDLE_G722_MPEG_BUG!=0)
     stream->rtp_rx_check_cnt = 5;
     stream->has_g722_mpeg_bug = PJ_FALSE;
diff --git a/pjmedia/src/pjmedia/wsola.c b/pjmedia/src/pjmedia/wsola.c
index b0a87807..e27233e4 100644
--- a/pjmedia/src/pjmedia/wsola.c
+++ b/pjmedia/src/pjmedia/wsola.c
@@ -70,10 +70,10 @@
 #define HIST_CNT	1.5
 
 /* Template size, in msec */
-#define TEMPLATE_PTIME	5
+#define TEMPLATE_PTIME	PJMEDIA_WSOLA_TEMPLATE_LENGTH_MSEC
 
 /* Hanning window size, in msec */
-#define HANNING_PTIME	5
+#define HANNING_PTIME	PJMEDIA_WSOLA_DELAY_MSEC
 
 /* Number of frames in erase buffer */
 #define ERASE_CNT	((unsigned)3)
@@ -84,6 +84,11 @@
 /* Maximum distance from template for find_pitch() of expansion, in frames */
 #define EXP_MAX_DIST	HIST_CNT
 
+/* Duration of a continuous synthetic frames after which the volume 
+ * of the synthetic frame will be set to zero with fading-out effect.
+ */
+#define MAX_EXPAND_MSEC	PJMEDIA_WSOLA_MAX_EXPAND_MSEC
+
 
 /* Buffer content:
  *
@@ -131,7 +136,8 @@ struct pjmedia_wsola
     pj_uint16_t		 hist_size;	    /* History size (const)	    */
 
     pj_uint16_t		 min_extra;	    /* Minimum extra (const)	    */
-    pj_uint16_t		 expand_cnt;	    /* Consecutive expansion count  */
+    unsigned		 max_expand_cnt;    /* Max # of synthetic samples   */
+    unsigned		 fade_out_pos;	    /* Last fade-out position	    */
     pj_uint16_t		 expand_sr_min_dist;/* Minimum distance from template 
 					       for find_pitch() on expansion
 					       (const)			    */
@@ -146,6 +152,7 @@ struct pjmedia_wsola
 #endif
 
     pj_timestamp	 ts;		    /* Running timestamp.	    */
+
 };
 
 #if (PJMEDIA_WSOLA_IMP==PJMEDIA_WSOLA_IMP_WSOLA_LITE)
@@ -432,6 +439,64 @@ static void create_win(pj_pool_t *pool, pj_uint16_t **pw, unsigned count)
 
 #endif	/* PJ_HAS_FLOATING_POINT */
 
+/* Apply fade-in to the buffer.
+ *  - fade_cnt is the number of samples on which the volume
+ *       will go from zero to 100%
+ *  - fade_pos is current sample position within fade_cnt range.
+ *       It is zero for the first sample, so the first sample will
+ *	 have zero volume. This value is increasing.
+ */
+static void fade_in(pj_int16_t buf[], int count,
+		    int fade_in_pos, int fade_cnt)
+{
+#if defined(PJ_HAS_FLOATING_POINT) && PJ_HAS_FLOATING_POINT!=0
+    float fade_pos = (float)fade_in_pos;
+#else
+    int fade_pos = fade_in_pos;
+#endif
+
+    if (fade_cnt - fade_pos < count) {
+	for (; fade_pos < fade_cnt; ++fade_pos, ++buf) {
+	    *buf = (pj_int16_t)(*buf * fade_pos / fade_cnt);
+	}
+	/* Leave the remaining samples as is */
+    } else {
+	pj_int16_t *end = buf + count;
+	for (; buf != end; ++fade_pos, ++buf) {
+	    *buf = (pj_int16_t)(*buf * fade_pos / fade_cnt);
+	}
+    }
+}
+
+/* Apply fade-out to the buffer. */
+static void wsola_fade_out(pjmedia_wsola *wsola, 
+			   pj_int16_t buf[], int count)
+{
+    pj_int16_t *end = buf + count;
+    int fade_cnt = wsola->max_expand_cnt;
+#if defined(PJ_HAS_FLOATING_POINT) && PJ_HAS_FLOATING_POINT!=0
+    float fade_pos = (float)wsola->fade_out_pos;
+#else
+    int fade_pos = wsola->fade_out_pos;
+#endif
+
+    if (wsola->fade_out_pos == 0) {
+	pjmedia_zero_samples(buf, count);
+    } else if (fade_pos < count) {
+	for (; fade_pos; --fade_pos, ++buf) {
+	    *buf = (pj_int16_t)(*buf * fade_pos / fade_cnt);
+	}
+	if (buf != end)
+	    pjmedia_zero_samples(buf, end - buf);
+	wsola->fade_out_pos = 0;
+    } else {
+	for (; buf != end; --fade_pos, ++buf) {
+	    *buf = (pj_int16_t)(*buf * fade_pos / fade_cnt);
+	}
+	wsola->fade_out_pos -= count;
+    }
+}
+
 
 PJ_DEF(pj_status_t) pjmedia_wsola_create( pj_pool_t *pool, 
 					  unsigned clock_rate,
@@ -455,6 +520,8 @@ PJ_DEF(pj_status_t) pjmedia_wsola_create( pj_pool_t *pool,
     wsola->samples_per_frame = (pj_uint16_t) samples_per_frame;
     wsola->channel_count = (pj_uint16_t) channel_count;
     wsola->options = (pj_uint16_t) options;
+    wsola->max_expand_cnt = clock_rate * MAX_EXPAND_MSEC / 1000;
+    wsola->fade_out_pos = wsola->max_expand_cnt;
 
     /* Create circular buffer */
     wsola->buf_size = (pj_uint16_t) (samples_per_frame * FRAME_CNT);
@@ -523,6 +590,13 @@ PJ_DEF(pj_status_t) pjmedia_wsola_destroy(pjmedia_wsola *wsola)
     return PJ_SUCCESS;
 }
 
+PJ_DEF(pj_status_t) pjmedia_wsola_set_max_expand(pjmedia_wsola *wsola,
+						  unsigned msec)
+{
+    PJ_ASSERT_RETURN(wsola, PJ_EINVAL);
+    wsola->max_expand_cnt = msec * wsola->clock_rate / 1000;
+    return PJ_SUCCESS;
+}
 
 PJ_DEF(pj_status_t) pjmedia_wsola_reset( pjmedia_wsola *wsola,
 					 unsigned options)
@@ -533,6 +607,7 @@ PJ_DEF(pj_status_t) pjmedia_wsola_reset( pjmedia_wsola *wsola,
     pjmedia_circ_buf_reset(wsola->buf);
     pjmedia_circ_buf_set_len(wsola->buf, wsola->hist_size + wsola->min_extra);
     pjmedia_zero_samples(wsola->buf->start, wsola->buf->len); 
+    wsola->fade_out_pos = wsola->max_expand_cnt;
 
     return PJ_SUCCESS;
 }
@@ -682,7 +757,6 @@ PJ_DEF(pj_status_t) pjmedia_wsola_save( pjmedia_wsola *wsola,
     buf_len = pjmedia_circ_buf_get_len(wsola->buf);
 
     /* Update vars */
-    wsola->expand_cnt = 0;
     wsola->ts.u64 += wsola->samples_per_frame;
 
     /* If previous frame was lost, smoothen this frame with the generated one */
@@ -691,12 +765,34 @@ PJ_DEF(pj_status_t) pjmedia_wsola_save( pjmedia_wsola *wsola,
 	unsigned reg1_len, reg2_len;
 	pj_int16_t *ola_left;
 
+	/* Trim excessive len */
+	if ((int)buf_len > wsola->hist_size + (wsola->min_extra<<1)) {
+	    buf_len = wsola->hist_size + (wsola->min_extra<<1);
+	    pjmedia_circ_buf_set_len(wsola->buf, buf_len);
+	}
+
 	pjmedia_circ_buf_get_read_regions(wsola->buf, &reg1, &reg1_len, 
 					  &reg2, &reg2_len);
 
 	CHECK_(pjmedia_circ_buf_get_len(wsola->buf) >= 
 	       (unsigned)(wsola->hist_size + (wsola->min_extra<<1)));
 
+	/* Continue applying fade out to the extra samples */
+	if ((wsola->options & PJMEDIA_WSOLA_NO_FADING)==0) {
+	    if (reg2_len == 0) {
+		wsola_fade_out(wsola, reg1 + reg1_len - (wsola->min_extra<<1),
+			       (wsola->min_extra<<1));
+	    } else if ((int)reg2_len >= (wsola->min_extra<<1)) {
+		wsola_fade_out(wsola, reg2 + reg2_len - (wsola->min_extra<<1),
+			       (wsola->min_extra<<1));
+	    } else {
+		unsigned tmp = (wsola->min_extra<<1) - reg2_len;
+		wsola_fade_out(wsola, reg1 + reg1_len - tmp, tmp);
+		wsola_fade_out(wsola, reg2, reg2_len);
+	    }
+	}
+
+	/* Get the region in buffer to be merged with the frame */
 	if (reg2_len == 0) {
 	    ola_left = reg1 + reg1_len - wsola->min_extra;
 	} else if (reg2_len >= wsola->min_extra) {
@@ -710,12 +806,74 @@ PJ_DEF(pj_status_t) pjmedia_wsola_save( pjmedia_wsola *wsola,
 	    ola_left = wsola->merge_buf;
 	}
 
+	/* Apply fade-in to the frame before merging */
+	if ((wsola->options & PJMEDIA_WSOLA_NO_FADING)==0) {
+	    unsigned count = wsola->min_extra;
+	    int fade_in_pos;
+
+	    /* Scale fade_in position based on last fade-out */
+	    fade_in_pos = wsola->fade_out_pos * count /
+			  wsola->max_expand_cnt;
+
+	    /* Fade-in it */
+	    fade_in(frm, wsola->samples_per_frame,
+		    fade_in_pos, count);
+	}
+
+	/* Merge it */
 	overlapp_add_simple(frm, wsola->min_extra, ola_left, frm);
 
+	/* Trim len */
 	buf_len -= wsola->min_extra;
 	pjmedia_circ_buf_set_len(wsola->buf, buf_len);
+
+    } else if ((wsola->options & PJMEDIA_WSOLA_NO_FADING)==0 &&
+	       wsola->fade_out_pos != wsola->max_expand_cnt) 
+    {
+	unsigned count = wsola->min_extra;
+	int fade_in_pos;
+
+	/* Fade out the remaining synthetic samples */
+	if (buf_len > wsola->hist_size) {
+	    pj_int16_t *reg1, *reg2;
+	    unsigned reg1_len, reg2_len;
+
+	    /* Number of samples to fade out */
+	    count = buf_len - wsola->hist_size;
+
+	    pjmedia_circ_buf_get_read_regions(wsola->buf, &reg1, &reg1_len, 
+					      &reg2, &reg2_len);
+
+	    CHECK_(pjmedia_circ_buf_get_len(wsola->buf) >= 
+		   (unsigned)(wsola->hist_size + (wsola->min_extra<<1)));
+
+	    /* Continue applying fade out to the extra samples */
+	    if (reg2_len == 0) {
+		wsola_fade_out(wsola, reg1 + reg1_len - count, count);
+	    } else if ((int)reg2_len >= count) {
+		wsola_fade_out(wsola, reg2 + reg2_len - count, count);
+	    } else {
+		unsigned tmp = count - reg2_len;
+		wsola_fade_out(wsola, reg1 + reg1_len - tmp, tmp);
+		wsola_fade_out(wsola, reg2, reg2_len);
+	    }
+	}
+
+	/* Apply fade-in to the frame */
+	count = wsola->min_extra;
+
+	/* Scale fade_in position based on last fade-out */
+	fade_in_pos = wsola->fade_out_pos * count /
+		      wsola->max_expand_cnt;
+
+	/* Fade it in */
+	fade_in(frm, wsola->samples_per_frame,
+		fade_in_pos, count);
+
     }
 
+    wsola->fade_out_pos = wsola->max_expand_cnt;
+
     status = pjmedia_circ_buf_write(wsola->buf, frm, wsola->samples_per_frame);
     if (status != PJ_SUCCESS) {
 	TRACE_((THIS_FILE, "Failed writing to circbuf [err=%d]", status));
@@ -737,8 +895,6 @@ PJ_DEF(pj_status_t) pjmedia_wsola_generate( pjmedia_wsola *wsola,
 					    pj_int16_t frm[])
 {
     unsigned samples_len, samples_req;
-
-
     pj_status_t status = PJ_SUCCESS;
 
     CHECK_(pjmedia_circ_buf_get_len(wsola->buf) >= wsola->hist_size + 
@@ -757,7 +913,6 @@ PJ_DEF(pj_status_t) pjmedia_wsola_generate( pjmedia_wsola *wsola,
 	expand(wsola, samples_req - samples_len);
 	TRACE_((THIS_FILE, "Buf size after expanded = %d", 
 		pjmedia_circ_buf_get_len(wsola->buf)));
-	wsola->expand_cnt++;
     }
 
     status = pjmedia_circ_buf_copy(wsola->buf, wsola->hist_size, frm, 
@@ -769,6 +924,11 @@ PJ_DEF(pj_status_t) pjmedia_wsola_generate( pjmedia_wsola *wsola,
 
     pjmedia_circ_buf_adv_read_ptr(wsola->buf, wsola->samples_per_frame);
 
+    /* Apply fade-out to the frame */
+    if ((wsola->options & PJMEDIA_WSOLA_NO_FADING)==0) {
+	wsola_fade_out(wsola, frm, wsola->samples_per_frame);
+    }
+
     return PJ_SUCCESS;
 }