From b6027c6bf59e9c6d32840b50f4dcaea9ed3d13de Mon Sep 17 00:00:00 2001 From: Benny Prijono Date: Sat, 1 Aug 2009 09:20:59 +0000 Subject: Initial commit for ticket #929: Improve packet lost concealment (PLC) when handling burst of lost packets WSOLA improvements: - Introduce fade-out and fade-in effect - Limit the number of continuous synthetic samples (only take effect when fading is used) - Export many settings as macros: - PJMEDIA_WSOLA_DELAY_MSEC (was HANNING_PTIME) - PJMEDIA_WSOLA_TEMPLATE_LENGTH_MSEC (was TEMPLATE_PTIME) - PJMEDIA_WSOLA_MAX_EXPAND_MSEC PLC: - added compile time macro PJMEDIA_WSOLA_PLC_NO_FADING to disable fading (default enabled) Stream: - fixed bug when stream is not PLC-ing subsequent packet loss (only the first) - also add maximum PLC limit just as precaution if PLC doesn't limit number of synthetic frames - unrelated: fixed warning about unused send_keep_alive() function Delaybuf: - modified to NOT use fading in WSOLA since we don't expect it to generate many continuous synthetic frames git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@2850 74dad513-b988-da41-8d7b-12977e46ad98 --- pjmedia/src/pjmedia/delaybuf.c | 2 +- pjmedia/src/pjmedia/plc_common.c | 7 +- pjmedia/src/pjmedia/stream.c | 57 ++++++++++--- pjmedia/src/pjmedia/wsola.c | 174 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 218 insertions(+), 22 deletions(-) (limited to 'pjmedia/src') diff --git a/pjmedia/src/pjmedia/delaybuf.c b/pjmedia/src/pjmedia/delaybuf.c index e39cda79..c0a316fd 100644 --- a/pjmedia/src/pjmedia/delaybuf.c +++ b/pjmedia/src/pjmedia/delaybuf.c @@ -128,7 +128,7 @@ PJ_DEF(pj_status_t) pjmedia_delay_buf_create( pj_pool_t *pool, /* Create WSOLA */ status = pjmedia_wsola_create(pool, clock_rate, samples_per_frame, 1, - 0, &b->wsola); + PJMEDIA_WSOLA_NO_FADING, &b->wsola); if (status != PJ_SUCCESS) return status; diff --git a/pjmedia/src/pjmedia/plc_common.c b/pjmedia/src/pjmedia/plc_common.c index aa9b685e..ea54accd 100644 --- a/pjmedia/src/pjmedia/plc_common.c +++ b/pjmedia/src/pjmedia/plc_common.c @@ -125,6 +125,7 @@ static void* plc_wsola_create(pj_pool_t *pool, unsigned clock_rate, unsigned samples_per_frame) { struct wsola_plc *o; + unsigned flag; pj_status_t status; PJ_UNUSED_ARG(clock_rate); @@ -132,8 +133,12 @@ static void* plc_wsola_create(pj_pool_t *pool, unsigned clock_rate, o = PJ_POOL_ZALLOC_T(pool, struct wsola_plc); o->prev_lost = PJ_FALSE; + flag = PJMEDIA_WSOLA_NO_DISCARD; + if (PJMEDIA_WSOLA_PLC_NO_FADING) + flag |= PJMEDIA_WSOLA_NO_FADING; + status = pjmedia_wsola_create(pool, clock_rate, samples_per_frame, 1, - PJMEDIA_WSOLA_NO_DISCARD, &o->wsola); + flag, &o->wsola); if (status != PJ_SUCCESS) return NULL; diff --git a/pjmedia/src/pjmedia/stream.c b/pjmedia/src/pjmedia/stream.c index 587370e3..36490839 100644 --- a/pjmedia/src/pjmedia/stream.c +++ b/pjmedia/src/pjmedia/stream.c @@ -43,6 +43,13 @@ #define BYTES_PER_SAMPLE 2 +/* Limit the number of synthetic audio samples that are generated by PLC. + * Normally PLC should have it's own means to limit the number of + * synthetic frames, so we need to set this to a reasonably large value + * just as precaution + */ +#define MAX_PLC_MSEC 240 + /** * Media channel. */ @@ -99,6 +106,9 @@ struct pjmedia_stream unsigned enc_buf_count; /**< Number of samples in the encoding buffer. */ + unsigned plc_cnt; /**< # of consecutive PLC frames*/ + unsigned max_plc_cnt; /**< Max # of PLC frames */ + unsigned vad_enabled; /**< VAD enabled in param. */ unsigned frame_size; /**< Size of encoded base frame.*/ pj_bool_t is_streaming; /**< Currently streaming?. This @@ -197,13 +207,13 @@ static void stream_perror(const char *sender, const char *title, PJ_LOG(4,(sender, "%s: %s [err:%d]", title, errmsg, status)); } +#if defined(PJMEDIA_STREAM_ENABLE_KA) && PJMEDIA_STREAM_ENABLE_KA != 0 /* - * Send keep-alive packet. + * Send keep-alive packet using non-codec frame. */ static void send_keep_alive_packet(pjmedia_stream *stream) { -#if defined(PJMEDIA_STREAM_ENABLE_KA) && \ - PJMEDIA_STREAM_ENABLE_KA == PJMEDIA_STREAM_KA_EMPTY_RTP +#if PJMEDIA_STREAM_ENABLE_KA == PJMEDIA_STREAM_KA_EMPTY_RTP /* Keep-alive packet is empty RTP */ pj_status_t status; @@ -224,8 +234,7 @@ static void send_keep_alive_packet(pjmedia_stream *stream) pkt_len); TRC_((stream->port.info.name.ptr, "Keep-alive sent (empty RTP)")); -#elif defined(PJMEDIA_STREAM_ENABLE_KA) && \ - PJMEDIA_STREAM_ENABLE_KA == PJMEDIA_STREAM_KA_USER +#elif PJMEDIA_STREAM_ENABLE_KA == PJMEDIA_STREAM_KA_USER /* Keep-alive packet is defined in PJMEDIA_STREAM_KA_USER_PKT */ int pkt_len; @@ -243,6 +252,7 @@ static void send_keep_alive_packet(pjmedia_stream *stream) #endif } +#endif /* defined(PJMEDIA_STREAM_ENABLE_KA) */ /* * play_callback() @@ -294,7 +304,8 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame) /* Activate PLC */ if (stream->codec->op->recover && - stream->codec_param.setting.plc) + stream->codec_param.setting.plc && + stream->plc_cnt < stream->max_plc_cnt) { pjmedia_frame frame_out; @@ -304,6 +315,8 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame) frame_out.size, &frame_out); + ++stream->plc_cnt; + } else { status = -1; } @@ -318,20 +331,24 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame) PJ_LOG(5,(stream->port.info.name.ptr, "Lost frame recovered")); } - + } else if (frame_type == PJMEDIA_JB_ZERO_EMPTY_FRAME) { /* Jitter buffer is empty. If this is the first "empty" state, * activate PLC to smoothen the fade-out, otherwise zero * the frame. */ - if (frame_type != stream->jb_last_frm) { + //Using this "if" will only invoke PLC for the first packet + //lost and not the subsequent ones. + //if (frame_type != stream->jb_last_frm) { + if (1) { pjmedia_jb_state jb_state; const char *with_plc = ""; /* Activate PLC to smoothen the missing frame */ if (stream->codec->op->recover && - stream->codec_param.setting.plc) + stream->codec_param.setting.plc && + stream->plc_cnt < stream->max_plc_cnt) { pjmedia_frame frame_out; @@ -343,9 +360,12 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame) &frame_out); if (status != PJ_SUCCESS) break; + samples_count += samples_per_frame; + ++stream->plc_cnt; - } while (samples_count < samples_required); + } while (samples_count < samples_required && + stream->plc_cnt < stream->max_plc_cnt); with_plc = ", plc invoked"; } @@ -379,7 +399,8 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame) /* Always activate PLC when it's available.. */ if (stream->codec->op->recover && - stream->codec_param.setting.plc) + stream->codec_param.setting.plc && + stream->plc_cnt < stream->max_plc_cnt) { pjmedia_frame frame_out; @@ -393,9 +414,13 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame) break; samples_count += samples_per_frame; - } while (samples_count < samples_required); + ++stream->plc_cnt; - if (stream->jb_last_frm != frame_type) { + } while (samples_count < samples_required && + stream->plc_cnt < stream->max_plc_cnt); + + //if (stream->jb_last_frm != frame_type) { + if (1) { PJ_LOG(5,(stream->port.info.name.ptr, "Jitter buffer is bufferring with plc (prefetch=%d)", jb_state.prefetch)); @@ -419,6 +444,8 @@ static pj_status_t get_frame( pjmedia_port *port, pjmedia_frame *frame) /* Got "NORMAL" frame from jitter buffer */ pjmedia_frame frame_in, frame_out; + stream->plc_cnt = 0; + /* Decode */ frame_in.buf = channel->out_pkt; frame_in.size = frame_size; @@ -1822,6 +1849,10 @@ PJ_DEF(pj_status_t) pjmedia_stream_create( pjmedia_endpt *endpt, ++stream->frame_size; } + /* How many consecutive PLC frames can be generated */ + stream->max_plc_cnt = (MAX_PLC_MSEC+stream->codec_param.info.frm_ptime-1)/ + stream->codec_param.info.frm_ptime; + #if defined(PJMEDIA_HANDLE_G722_MPEG_BUG) && (PJMEDIA_HANDLE_G722_MPEG_BUG!=0) stream->rtp_rx_check_cnt = 5; stream->has_g722_mpeg_bug = PJ_FALSE; diff --git a/pjmedia/src/pjmedia/wsola.c b/pjmedia/src/pjmedia/wsola.c index b0a87807..e27233e4 100644 --- a/pjmedia/src/pjmedia/wsola.c +++ b/pjmedia/src/pjmedia/wsola.c @@ -70,10 +70,10 @@ #define HIST_CNT 1.5 /* Template size, in msec */ -#define TEMPLATE_PTIME 5 +#define TEMPLATE_PTIME PJMEDIA_WSOLA_TEMPLATE_LENGTH_MSEC /* Hanning window size, in msec */ -#define HANNING_PTIME 5 +#define HANNING_PTIME PJMEDIA_WSOLA_DELAY_MSEC /* Number of frames in erase buffer */ #define ERASE_CNT ((unsigned)3) @@ -84,6 +84,11 @@ /* Maximum distance from template for find_pitch() of expansion, in frames */ #define EXP_MAX_DIST HIST_CNT +/* Duration of a continuous synthetic frames after which the volume + * of the synthetic frame will be set to zero with fading-out effect. + */ +#define MAX_EXPAND_MSEC PJMEDIA_WSOLA_MAX_EXPAND_MSEC + /* Buffer content: * @@ -131,7 +136,8 @@ struct pjmedia_wsola pj_uint16_t hist_size; /* History size (const) */ pj_uint16_t min_extra; /* Minimum extra (const) */ - pj_uint16_t expand_cnt; /* Consecutive expansion count */ + unsigned max_expand_cnt; /* Max # of synthetic samples */ + unsigned fade_out_pos; /* Last fade-out position */ pj_uint16_t expand_sr_min_dist;/* Minimum distance from template for find_pitch() on expansion (const) */ @@ -146,6 +152,7 @@ struct pjmedia_wsola #endif pj_timestamp ts; /* Running timestamp. */ + }; #if (PJMEDIA_WSOLA_IMP==PJMEDIA_WSOLA_IMP_WSOLA_LITE) @@ -432,6 +439,64 @@ static void create_win(pj_pool_t *pool, pj_uint16_t **pw, unsigned count) #endif /* PJ_HAS_FLOATING_POINT */ +/* Apply fade-in to the buffer. + * - fade_cnt is the number of samples on which the volume + * will go from zero to 100% + * - fade_pos is current sample position within fade_cnt range. + * It is zero for the first sample, so the first sample will + * have zero volume. This value is increasing. + */ +static void fade_in(pj_int16_t buf[], int count, + int fade_in_pos, int fade_cnt) +{ +#if defined(PJ_HAS_FLOATING_POINT) && PJ_HAS_FLOATING_POINT!=0 + float fade_pos = (float)fade_in_pos; +#else + int fade_pos = fade_in_pos; +#endif + + if (fade_cnt - fade_pos < count) { + for (; fade_pos < fade_cnt; ++fade_pos, ++buf) { + *buf = (pj_int16_t)(*buf * fade_pos / fade_cnt); + } + /* Leave the remaining samples as is */ + } else { + pj_int16_t *end = buf + count; + for (; buf != end; ++fade_pos, ++buf) { + *buf = (pj_int16_t)(*buf * fade_pos / fade_cnt); + } + } +} + +/* Apply fade-out to the buffer. */ +static void wsola_fade_out(pjmedia_wsola *wsola, + pj_int16_t buf[], int count) +{ + pj_int16_t *end = buf + count; + int fade_cnt = wsola->max_expand_cnt; +#if defined(PJ_HAS_FLOATING_POINT) && PJ_HAS_FLOATING_POINT!=0 + float fade_pos = (float)wsola->fade_out_pos; +#else + int fade_pos = wsola->fade_out_pos; +#endif + + if (wsola->fade_out_pos == 0) { + pjmedia_zero_samples(buf, count); + } else if (fade_pos < count) { + for (; fade_pos; --fade_pos, ++buf) { + *buf = (pj_int16_t)(*buf * fade_pos / fade_cnt); + } + if (buf != end) + pjmedia_zero_samples(buf, end - buf); + wsola->fade_out_pos = 0; + } else { + for (; buf != end; --fade_pos, ++buf) { + *buf = (pj_int16_t)(*buf * fade_pos / fade_cnt); + } + wsola->fade_out_pos -= count; + } +} + PJ_DEF(pj_status_t) pjmedia_wsola_create( pj_pool_t *pool, unsigned clock_rate, @@ -455,6 +520,8 @@ PJ_DEF(pj_status_t) pjmedia_wsola_create( pj_pool_t *pool, wsola->samples_per_frame = (pj_uint16_t) samples_per_frame; wsola->channel_count = (pj_uint16_t) channel_count; wsola->options = (pj_uint16_t) options; + wsola->max_expand_cnt = clock_rate * MAX_EXPAND_MSEC / 1000; + wsola->fade_out_pos = wsola->max_expand_cnt; /* Create circular buffer */ wsola->buf_size = (pj_uint16_t) (samples_per_frame * FRAME_CNT); @@ -523,6 +590,13 @@ PJ_DEF(pj_status_t) pjmedia_wsola_destroy(pjmedia_wsola *wsola) return PJ_SUCCESS; } +PJ_DEF(pj_status_t) pjmedia_wsola_set_max_expand(pjmedia_wsola *wsola, + unsigned msec) +{ + PJ_ASSERT_RETURN(wsola, PJ_EINVAL); + wsola->max_expand_cnt = msec * wsola->clock_rate / 1000; + return PJ_SUCCESS; +} PJ_DEF(pj_status_t) pjmedia_wsola_reset( pjmedia_wsola *wsola, unsigned options) @@ -533,6 +607,7 @@ PJ_DEF(pj_status_t) pjmedia_wsola_reset( pjmedia_wsola *wsola, pjmedia_circ_buf_reset(wsola->buf); pjmedia_circ_buf_set_len(wsola->buf, wsola->hist_size + wsola->min_extra); pjmedia_zero_samples(wsola->buf->start, wsola->buf->len); + wsola->fade_out_pos = wsola->max_expand_cnt; return PJ_SUCCESS; } @@ -682,7 +757,6 @@ PJ_DEF(pj_status_t) pjmedia_wsola_save( pjmedia_wsola *wsola, buf_len = pjmedia_circ_buf_get_len(wsola->buf); /* Update vars */ - wsola->expand_cnt = 0; wsola->ts.u64 += wsola->samples_per_frame; /* If previous frame was lost, smoothen this frame with the generated one */ @@ -691,12 +765,34 @@ PJ_DEF(pj_status_t) pjmedia_wsola_save( pjmedia_wsola *wsola, unsigned reg1_len, reg2_len; pj_int16_t *ola_left; + /* Trim excessive len */ + if ((int)buf_len > wsola->hist_size + (wsola->min_extra<<1)) { + buf_len = wsola->hist_size + (wsola->min_extra<<1); + pjmedia_circ_buf_set_len(wsola->buf, buf_len); + } + pjmedia_circ_buf_get_read_regions(wsola->buf, ®1, ®1_len, ®2, ®2_len); CHECK_(pjmedia_circ_buf_get_len(wsola->buf) >= (unsigned)(wsola->hist_size + (wsola->min_extra<<1))); + /* Continue applying fade out to the extra samples */ + if ((wsola->options & PJMEDIA_WSOLA_NO_FADING)==0) { + if (reg2_len == 0) { + wsola_fade_out(wsola, reg1 + reg1_len - (wsola->min_extra<<1), + (wsola->min_extra<<1)); + } else if ((int)reg2_len >= (wsola->min_extra<<1)) { + wsola_fade_out(wsola, reg2 + reg2_len - (wsola->min_extra<<1), + (wsola->min_extra<<1)); + } else { + unsigned tmp = (wsola->min_extra<<1) - reg2_len; + wsola_fade_out(wsola, reg1 + reg1_len - tmp, tmp); + wsola_fade_out(wsola, reg2, reg2_len); + } + } + + /* Get the region in buffer to be merged with the frame */ if (reg2_len == 0) { ola_left = reg1 + reg1_len - wsola->min_extra; } else if (reg2_len >= wsola->min_extra) { @@ -710,12 +806,74 @@ PJ_DEF(pj_status_t) pjmedia_wsola_save( pjmedia_wsola *wsola, ola_left = wsola->merge_buf; } + /* Apply fade-in to the frame before merging */ + if ((wsola->options & PJMEDIA_WSOLA_NO_FADING)==0) { + unsigned count = wsola->min_extra; + int fade_in_pos; + + /* Scale fade_in position based on last fade-out */ + fade_in_pos = wsola->fade_out_pos * count / + wsola->max_expand_cnt; + + /* Fade-in it */ + fade_in(frm, wsola->samples_per_frame, + fade_in_pos, count); + } + + /* Merge it */ overlapp_add_simple(frm, wsola->min_extra, ola_left, frm); + /* Trim len */ buf_len -= wsola->min_extra; pjmedia_circ_buf_set_len(wsola->buf, buf_len); + + } else if ((wsola->options & PJMEDIA_WSOLA_NO_FADING)==0 && + wsola->fade_out_pos != wsola->max_expand_cnt) + { + unsigned count = wsola->min_extra; + int fade_in_pos; + + /* Fade out the remaining synthetic samples */ + if (buf_len > wsola->hist_size) { + pj_int16_t *reg1, *reg2; + unsigned reg1_len, reg2_len; + + /* Number of samples to fade out */ + count = buf_len - wsola->hist_size; + + pjmedia_circ_buf_get_read_regions(wsola->buf, ®1, ®1_len, + ®2, ®2_len); + + CHECK_(pjmedia_circ_buf_get_len(wsola->buf) >= + (unsigned)(wsola->hist_size + (wsola->min_extra<<1))); + + /* Continue applying fade out to the extra samples */ + if (reg2_len == 0) { + wsola_fade_out(wsola, reg1 + reg1_len - count, count); + } else if ((int)reg2_len >= count) { + wsola_fade_out(wsola, reg2 + reg2_len - count, count); + } else { + unsigned tmp = count - reg2_len; + wsola_fade_out(wsola, reg1 + reg1_len - tmp, tmp); + wsola_fade_out(wsola, reg2, reg2_len); + } + } + + /* Apply fade-in to the frame */ + count = wsola->min_extra; + + /* Scale fade_in position based on last fade-out */ + fade_in_pos = wsola->fade_out_pos * count / + wsola->max_expand_cnt; + + /* Fade it in */ + fade_in(frm, wsola->samples_per_frame, + fade_in_pos, count); + } + wsola->fade_out_pos = wsola->max_expand_cnt; + status = pjmedia_circ_buf_write(wsola->buf, frm, wsola->samples_per_frame); if (status != PJ_SUCCESS) { TRACE_((THIS_FILE, "Failed writing to circbuf [err=%d]", status)); @@ -737,8 +895,6 @@ PJ_DEF(pj_status_t) pjmedia_wsola_generate( pjmedia_wsola *wsola, pj_int16_t frm[]) { unsigned samples_len, samples_req; - - pj_status_t status = PJ_SUCCESS; CHECK_(pjmedia_circ_buf_get_len(wsola->buf) >= wsola->hist_size + @@ -757,7 +913,6 @@ PJ_DEF(pj_status_t) pjmedia_wsola_generate( pjmedia_wsola *wsola, expand(wsola, samples_req - samples_len); TRACE_((THIS_FILE, "Buf size after expanded = %d", pjmedia_circ_buf_get_len(wsola->buf))); - wsola->expand_cnt++; } status = pjmedia_circ_buf_copy(wsola->buf, wsola->hist_size, frm, @@ -769,6 +924,11 @@ PJ_DEF(pj_status_t) pjmedia_wsola_generate( pjmedia_wsola *wsola, pjmedia_circ_buf_adv_read_ptr(wsola->buf, wsola->samples_per_frame); + /* Apply fade-out to the frame */ + if ((wsola->options & PJMEDIA_WSOLA_NO_FADING)==0) { + wsola_fade_out(wsola, frm, wsola->samples_per_frame); + } + return PJ_SUCCESS; } -- cgit v1.2.3