diff options
author | Benny Prijono <bennylp@teluu.com> | 2006-02-25 21:15:49 +0000 |
---|---|---|
committer | Benny Prijono <bennylp@teluu.com> | 2006-02-25 21:15:49 +0000 |
commit | 478d2bc6e45a2dc66472bb577a77537ea97fac5d (patch) | |
tree | 118a6e71af8bdd3b1b26914ed8814945e20a6d14 | |
parent | e5290641d50566a9aff8eae9bec8d585f79077de (diff) |
Fixed echo in conference, and add ability to shrink buffer in jitter buffer
git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@229 74dad513-b988-da41-8d7b-12977e46ad98
-rw-r--r-- | pjmedia/build/pjmedia.dsp | 2 | ||||
-rw-r--r-- | pjmedia/include/pjmedia/silencedet.h | 38 | ||||
-rw-r--r-- | pjmedia/src/pjmedia/conference.c | 163 | ||||
-rw-r--r-- | pjmedia/src/pjmedia/jbuf.c | 33 | ||||
-rw-r--r-- | pjmedia/src/pjmedia/silencedet.c | 42 | ||||
-rw-r--r-- | pjmedia/src/pjmedia/stream.c | 2 |
6 files changed, 200 insertions, 80 deletions
diff --git a/pjmedia/build/pjmedia.dsp b/pjmedia/build/pjmedia.dsp index 7edaa90e..1475636f 100644 --- a/pjmedia/build/pjmedia.dsp +++ b/pjmedia/build/pjmedia.dsp @@ -65,7 +65,7 @@ LIB32=link.exe -lib # PROP Intermediate_Dir ".\output\pjmedia-i386-win32-vc6-debug"
# PROP Target_Dir ""
# ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_MBCS" /D "_LIB" /YX /FD /GZ /c
-# ADD CPP /nologo /MTd /W4 /Gm /GX /ZI /Od /I "../include" /I "../../pjlib/include" /I "../../pjlib-util/include" /I "../src/pjmedia/portaudio" /D "_DEBUG" /D "PA_NO_ASIO" /D "WIN32" /D "_MBCS" /D "_LIB" /D PJ_WIN32=1 /D PJ_M_I386=1 /FR /FD /GZ /c
+# ADD CPP /nologo /MTd /W4 /Gm /GX /ZI /Od /I "../include" /I "../../pjlib/include" /I "../../pjlib-util/include" /I "../src/pjmedia/portaudio" /D "_DEBUG" /D "PA_NO_ASIO" /D "PA_NO_WMME" /D "WIN32" /D "_MBCS" /D "_LIB" /D PJ_WIN32=1 /D PJ_M_I386=1 /FR /FD /GZ /c
# SUBTRACT CPP /YX
# ADD BASE RSC /l 0x409 /d "_DEBUG"
# ADD RSC /l 0x409 /d "_DEBUG"
diff --git a/pjmedia/include/pjmedia/silencedet.h b/pjmedia/include/pjmedia/silencedet.h index 91da8597..f251c082 100644 --- a/pjmedia/include/pjmedia/silencedet.h +++ b/pjmedia/include/pjmedia/silencedet.h @@ -85,6 +85,25 @@ PJ_DECL(pj_status_t) pjmedia_silence_det_disable( pjmedia_silence_det *sd ); /** + * Perform voice activity detection on the given input samples. This + * function uses #pjmedia_calc_avg_signal() and #pjmedia_silence_det_apply() + * for its calculation. + * + * @param sd The silence detector instance. + * @param samples Pointer to 16-bit PCM input samples. + * @param count Number of samples in the input. + * @param p_level Optional pointer to receive average signal level + * of the input samples. + * + * @return PJ_SUCCESS on success. + */ +PJ_DECL(pj_bool_t) pjmedia_silence_det_detect( pjmedia_silence_det *sd, + const pj_int16_t samples[], + pj_size_t count, + pj_int32_t *p_level); + + +/** * Calculate average signal level for the given samples. * * @param samples Pointer to 16-bit PCM samples. @@ -93,25 +112,22 @@ PJ_DECL(pj_status_t) pjmedia_silence_det_disable( pjmedia_silence_det *sd ); * @return The average signal level, which simply is total level * divided by number of samples. */ -PJ_DECL(pj_int32_t) pjmedia_silence_det_calc_avg_signal( const pj_int16_t samples[], - pj_size_t count ); +PJ_DECL(pj_int32_t) pjmedia_calc_avg_signal( const pj_int16_t samples[], + pj_size_t count ); + /** - * Perform voice activity detection on the given input samples. + * Perform voice activity detection, given the specified average signal + * level. * * @param sd The silence detector instance. - * @param samples Pointer to 16-bit PCM input samples. - * @param count Number of samples in the input. - * @param p_level Optional pointer to receive average signal level - * of the input samples. + * @param level Signal level. * * @return PJ_SUCCESS on success. */ -PJ_DECL(pj_bool_t) pjmedia_silence_det_detect_silence( pjmedia_silence_det *sd, - const pj_int16_t samples[], - pj_size_t count, - pj_int32_t *p_level); +PJ_DECL(pj_bool_t) pjmedia_silence_det_apply( pjmedia_silence_det *sd, + pj_uint32_t level); PJ_END_DECL diff --git a/pjmedia/src/pjmedia/conference.c b/pjmedia/src/pjmedia/conference.c index 0db3ccad..dea2be7e 100644 --- a/pjmedia/src/pjmedia/conference.c +++ b/pjmedia/src/pjmedia/conference.c @@ -91,6 +91,7 @@ struct pjmedia_conf unsigned connect_cnt; /**< Total number of connections */ pj_snd_stream *snd_rec; /**< Sound recorder stream. */ pj_snd_stream *snd_player; /**< Sound player stream. */ + pj_mutex_t *mutex; /**< Conference mutex. */ struct conf_port **ports; /**< Array of ports. */ pj_uint16_t *uns_buf; /**< Buf for unsigned conversion */ unsigned sampling_rate; /**< Sampling rate. */ @@ -100,6 +101,9 @@ struct pjmedia_conf }; +/* Extern */ +unsigned char linear2ulaw(int pcm_val); + /* Prototypes */ static pj_status_t play_cb( /* in */ void *user_data, /* in */ pj_uint32_t timestamp, @@ -260,6 +264,13 @@ PJ_DEF(pj_status_t) pjmedia_conf_create( pj_pool_t *pool, /* Create temporary buffer. */ conf->uns_buf = pj_pool_zalloc(pool, samples_per_frame * sizeof(conf->uns_buf[0])); + + /* Create mutex. */ + status = pj_mutex_create_simple(pool, "conf", &conf->mutex); + if (status != PJ_SUCCESS) + return status; + + /* Done */ *p_conf = conf; @@ -364,6 +375,7 @@ PJ_DEF(pj_status_t) pjmedia_conf_destroy( pjmedia_conf *conf ) suspend_sound(conf); destroy_sound(conf); + pj_mutex_destroy(conf->mutex); return PJ_SUCCESS; } @@ -385,8 +397,11 @@ PJ_DEF(pj_status_t) pjmedia_conf_add_port( pjmedia_conf *conf, PJ_ASSERT_RETURN(conf && pool && strm_port && port_name && p_port, PJ_EINVAL); + pj_mutex_lock(conf->mutex); + if (conf->port_cnt >= conf->max_ports) { pj_assert(!"Too many ports"); + pj_mutex_unlock(conf->mutex); return PJ_ETOOMANY; } @@ -400,8 +415,10 @@ PJ_DEF(pj_status_t) pjmedia_conf_add_port( pjmedia_conf *conf, /* Create port structure. */ status = create_conf_port(pool, conf, port_name, &conf_port); - if (status != PJ_SUCCESS) + if (status != PJ_SUCCESS) { + pj_mutex_unlock(conf->mutex); return status; + } /* Set the port */ conf_port->port = strm_port; @@ -413,6 +430,8 @@ PJ_DEF(pj_status_t) pjmedia_conf_add_port( pjmedia_conf *conf, /* Done. */ *p_port = index; + pj_mutex_unlock(conf->mutex); + return PJ_SUCCESS; } @@ -462,6 +481,8 @@ PJ_DEF(pj_status_t) pjmedia_conf_connect_port( pjmedia_conf *conf, PJ_ASSERT_RETURN(conf->ports[src_slot] != NULL, PJ_EINVAL); PJ_ASSERT_RETURN(conf->ports[sink_slot] != NULL, PJ_EINVAL); + pj_mutex_lock(conf->mutex); + src_port = conf->ports[src_slot]; dst_port = conf->ports[sink_slot]; @@ -480,6 +501,8 @@ PJ_DEF(pj_status_t) pjmedia_conf_connect_port( pjmedia_conf *conf, dst_port->name.ptr)); } + pj_mutex_unlock(conf->mutex); + return PJ_SUCCESS; } @@ -501,6 +524,8 @@ PJ_DEF(pj_status_t) pjmedia_conf_disconnect_port( pjmedia_conf *conf, PJ_ASSERT_RETURN(conf->ports[src_slot] != NULL, PJ_EINVAL); PJ_ASSERT_RETURN(conf->ports[sink_slot] != NULL, PJ_EINVAL); + pj_mutex_lock(conf->mutex); + src_port = conf->ports[src_slot]; dst_port = conf->ports[sink_slot]; @@ -521,6 +546,8 @@ PJ_DEF(pj_status_t) pjmedia_conf_disconnect_port( pjmedia_conf *conf, } } + pj_mutex_unlock(conf->mutex); + return PJ_SUCCESS; } @@ -544,7 +571,8 @@ PJ_DEF(pj_status_t) pjmedia_conf_remove_port( pjmedia_conf *conf, * Don't want to remove port while port is being accessed by sound * device's threads! */ - //suspend_sound(conf); + + pj_mutex_lock(conf->mutex); conf_port = conf->ports[port]; conf_port->tx_setting = PJMEDIA_PORT_DISABLE; @@ -577,14 +605,13 @@ PJ_DEF(pj_status_t) pjmedia_conf_remove_port( pjmedia_conf *conf, conf->ports[port] = NULL; --conf->port_cnt; - /* Reactivate sound device if there are connections */ - if (conf->connect_cnt != 0) { - //resume_sound(conf); - } else { + /* Stop sound if there's no connection. */ + if (conf->connect_cnt == 0) { destroy_sound(conf); } - pj_thread_sleep(60); + pj_mutex_unlock(conf->mutex); + return PJ_SUCCESS; } @@ -658,21 +685,26 @@ static pj_status_t play_cb( /* in */ void *user_data, { pjmedia_conf *conf = user_data; pj_int16_t *output_buf = output; - unsigned i, j; + unsigned ci, cj, i, j; PJ_UNUSED_ARG(timestamp); PJ_UNUSED_ARG(size); TRACE_(("p")); - /* Clear all port's tmp buffers. */ - for (i=0; i<conf->max_ports; ++i) { + pj_mutex_lock(conf->mutex); + + /* Zero all port's temporary buffers. */ + for (i=0, ci=0; i<conf->max_ports && ci < conf->port_cnt; ++i) { struct conf_port *conf_port = conf->ports[i]; pj_uint32_t *sum_buf; + /* Skip empty slot. */ if (!conf_port) continue; + ++ci; + conf_port->sources = 0; sum_buf = conf_port->sum_buf; @@ -680,27 +712,27 @@ static pj_status_t play_cb( /* in */ void *user_data, sum_buf[j] = 0; } - /* Get frames from all ports, and "add" the signal + /* Get frames from all ports, and "mix" the signal * to sum_buf of all listeners of the port. */ - for (i=0; i<conf->max_ports; ++i) { + for (i=0, ci=0; i<conf->max_ports && ci<conf->port_cnt; ++i) { struct conf_port *conf_port = conf->ports[i]; pj_int32_t level; - pj_bool_t silence; /* Skip empty port. */ if (!conf_port) continue; + ++ci; + /* Skip if we're not allowed to receive from this port. */ if (conf_port->rx_setting == PJMEDIA_PORT_DISABLE) { - TRACE_(("rxdis:%d ", i)); continue; } /* Get frame from this port. - * If port has rx_buffer (e.g. sound port), then get the frame - * from the rx_buffer instead. + * For port zero (sound port), get the frame from the rx_buffer + * instead. */ if (i==0) { pj_int16_t *rx_buf; @@ -711,6 +743,17 @@ static pj_status_t play_cb( /* in */ void *user_data, RX_BUF_COUNT; } + /* Skip if this port is muted/disabled. */ + if (conf_port->rx_setting != PJMEDIA_PORT_ENABLE) { + continue; + } + + + /* Skip if no port is listening to the microphone */ + if (conf_port->listener_cnt == 0) { + continue; + } + rx_buf = conf_port->rx_buf[conf_port->rx_read]; for (j=0; j<conf->samples_per_frame; ++j) { ((pj_int16_t*)output)[j] = rx_buf[j]; @@ -725,7 +768,7 @@ static pj_status_t play_cb( /* in */ void *user_data, frame.size = size; pjmedia_port_get_frame(conf_port->port, &frame); - if (frame.type == PJMEDIA_FRAME_TYPE_NONE) + if (frame.type == PJMEDIA_FRAME_TYPE_NONE) continue; } @@ -737,24 +780,21 @@ static pj_status_t play_cb( /* in */ void *user_data, if (conf_port->listener_cnt == 0) continue; - /* Do we have signal? */ - silence = pjmedia_silence_det_detect_silence(conf_port->vad, - output, - conf->samples_per_frame, - &level); + /* Get the signal level. */ + level = pjmedia_calc_avg_signal(output, conf->samples_per_frame); - /* Skip if we don't have signal. */ - if (silence) { - TRACE_(("sil:%d ", i)); - continue; - } + /* Convert level to 8bit complement ulaw */ + level = linear2ulaw(level) ^ 0xff; - /* Convert the buffer to unsigned value */ + /* Convert the buffer to unsigned 16bit value */ for (j=0; j<conf->samples_per_frame; ++j) conf->uns_buf[j] = pcm2unsigned(((pj_int16_t*)output)[j]); /* Add the signal to all listeners. */ - for (j=0; j<conf->max_ports; ++j) { + for (j=0, cj=0; + j<conf->max_ports && cj<(unsigned)conf_port->listener_cnt; + ++j) + { struct conf_port *listener = conf->ports[j]; pj_uint32_t *sum_buf; unsigned k; @@ -762,22 +802,27 @@ static pj_status_t play_cb( /* in */ void *user_data, if (listener == 0) continue; + /* Skip if this is not the listener. */ + if (!conf_port->listeners[j]) + continue; + + ++cj; + /* Skip if this listener doesn't want to receive audio */ if (listener->tx_setting != PJMEDIA_PORT_ENABLE) continue; - //TRACE_(("mix:%d->%d ", i, j)); - + /* Mix the buffer */ sum_buf = listener->sum_buf; for (k=0; k<conf->samples_per_frame; ++k) - sum_buf[k] += conf->uns_buf[k]; + sum_buf[k] += (conf->uns_buf[k] * level); - listener->sources++; + listener->sources += level; } } /* For all ports, calculate avg signal. */ - for (i=0; i<conf->max_ports; ++i) { + for (i=0, ci=0; i<conf->max_ports && ci<conf->port_cnt; ++i) { struct conf_port *conf_port = conf->ports[i]; pjmedia_frame frame; pj_int16_t *target_buf; @@ -785,6 +830,8 @@ static pj_status_t play_cb( /* in */ void *user_data, if (!conf_port) continue; + ++ci; + if (conf_port->tx_setting == PJMEDIA_PORT_MUTE) { frame.type = PJMEDIA_FRAME_TYPE_NONE; frame.buf = NULL; @@ -799,20 +846,13 @@ static pj_status_t play_cb( /* in */ void *user_data, continue; } - // - // TODO: - // When there's no source, not transmit the frame, but instead - // transmit a 'silence' frame. This is to allow the 'port' to - // do some processing, such as updating timestamp for RTP session - // or transmit signal when it's in the middle of transmitting DTMF. - // - target_buf = (conf_port->cur_tx_buf==conf_port->tx_buf1? conf_port->tx_buf2 : conf_port->tx_buf1); if (conf_port->sources) { for (j=0; j<conf->samples_per_frame; ++j) { - target_buf[j] = unsigned2pcm(conf_port->sum_buf[j] / conf_port->sources); + target_buf[j] = unsigned2pcm(conf_port->sum_buf[j] / + conf_port->sources); } } @@ -820,9 +860,21 @@ static pj_status_t play_cb( /* in */ void *user_data, conf_port->cur_tx_buf = target_buf; pj_memset(&frame, 0, sizeof(frame)); - if (conf_port->sources) - frame.type = PJMEDIA_FRAME_TYPE_AUDIO; - else + if (conf_port->sources) { + + pj_bool_t is_silence = PJ_FALSE; + + /* Apply silence detection. */ +#if 0 + is_silence = pjmedia_silence_det_detect(conf_port->vad, + target_buf, + conf->samples_per_frame, + NULL); +#endif + frame.type = is_silence ? PJMEDIA_FRAME_TYPE_NONE : + PJMEDIA_FRAME_TYPE_AUDIO; + + } else frame.type = PJMEDIA_FRAME_TYPE_NONE; frame.buf = conf_port->cur_tx_buf; @@ -835,8 +887,15 @@ static pj_status_t play_cb( /* in */ void *user_data, } /* Return sound playback frame. */ - for (j=0; j<conf->samples_per_frame; ++j) - output_buf[j] = conf->ports[0]->cur_tx_buf[j]; + if (conf->ports[0]->sources) { + for (j=0; j<conf->samples_per_frame; ++j) + output_buf[j] = conf->ports[0]->cur_tx_buf[j]; + } else { + for (j=0; j<conf->samples_per_frame; ++j) + output_buf[j] = 0; + } + + pj_mutex_unlock(conf->mutex); return PJ_SUCCESS; } @@ -863,6 +922,14 @@ static pj_status_t rec_cb( /* in */ void *user_data, TRACE_(("rxerr ")); } + /* Skip if this port is muted/disabled. */ + if (snd_port->rx_setting != PJMEDIA_PORT_ENABLE) + return PJ_SUCCESS; + + /* Skip if no port is listening to the microphone */ + if (snd_port->listener_cnt == 0) + return PJ_SUCCESS; + /* Determine which rx_buffer to fill in */ target_rx_buf = snd_port->rx_buf[snd_port->rx_write]; diff --git a/pjmedia/src/pjmedia/jbuf.c b/pjmedia/src/pjmedia/jbuf.c index e40b1ef5..6f8865ba 100644 --- a/pjmedia/src/pjmedia/jbuf.c +++ b/pjmedia/src/pjmedia/jbuf.c @@ -23,9 +23,12 @@ #include <pjmedia/errno.h> #include <pj/pool.h> #include <pj/assert.h> +#include <pj/log.h> #include <pj/string.h> +#define THIS_FILE "jbuf.c" + struct jb_framelist { @@ -54,6 +57,7 @@ struct pjmedia_jbuf int jb_last_jitter; // last jitter calculated int jb_max_hist_jitter; // max jitter during the last jitter calculations int jb_stable_hist; // num of times the delay has been lower then the prefetch num + unsigned jb_op_count; // of of operations. int jb_last_op; // last operation executed on the framelist->flist_buffer (put/get) int jb_last_seq_no; // seq no. of the last frame inserted to the framelist->flist_buffer int jb_prefetch; // no. of frame to insert before removing some @@ -292,8 +296,9 @@ static void jbuf_calculate_jitter(pjmedia_jbuf *jb) jb->jb_last_jitter = PJ_ABS(jb->jb_level-jb->jb_last_level); jb->jb_last_level = jb->jb_level; jb->jb_max_hist_jitter = PJ_MAX(jb->jb_max_hist_jitter,jb->jb_last_jitter); - - if (jb->jb_last_jitter< jb->jb_prefetch) { + jb->jb_op_count++; + + if (jb->jb_last_jitter < jb->jb_prefetch) { jb->jb_stable_hist += jb->jb_last_jitter; if (jb->jb_stable_hist > STABLE_HISTORY_LIMIT) { int seq_diff = (jb->jb_prefetch - jb->jb_max_hist_jitter)/3; @@ -304,11 +309,35 @@ static void jbuf_calculate_jitter(pjmedia_jbuf *jb) jb->jb_stable_hist = 0; jb->jb_max_hist_jitter = 0; + + if (jb->jb_op_count >= 100 && + (int)jb_framelist_size(&jb->jb_framelist) > jb->jb_prefetch+2) + { + jb_framelist_remove_head(&jb->jb_framelist,1); + + PJ_LOG(5,(THIS_FILE, "jbuf prefetch: %d, size=%d", + jb->jb_prefetch, + jb_framelist_size(&jb->jb_framelist))); + jb->jb_op_count = 0; + } + } } else { jb->jb_prefetch = PJ_MIN(jb->jb_last_jitter,(int)(jb->jb_max_count*4/5)); jb->jb_stable_hist = 0; jb->jb_max_hist_jitter = 0; + + if (jb->jb_op_count >= 100) { + if ((int)jb_framelist_size(&jb->jb_framelist) > jb->jb_prefetch+2) { + jb_framelist_remove_head(&jb->jb_framelist,1); + + PJ_LOG(5,(THIS_FILE, "jbuf prefetch: %d, size=%d", + jb->jb_prefetch, + jb_framelist_size(&jb->jb_framelist))); + } + + jb->jb_op_count = 0; + } } } diff --git a/pjmedia/src/pjmedia/silencedet.c b/pjmedia/src/pjmedia/silencedet.c index 2c7583b0..74c3fd44 100644 --- a/pjmedia/src/pjmedia/silencedet.c +++ b/pjmedia/src/pjmedia/silencedet.c @@ -119,8 +119,8 @@ PJ_DEF(pj_status_t) pjmedia_silence_det_disable( pjmedia_silence_det *sd ) } -PJ_DEF(pj_int32_t) pjmedia_silence_det_calc_avg_signal(const pj_int16_t samples[], - pj_size_t count) +PJ_DEF(pj_int32_t) pjmedia_calc_avg_signal( const pj_int16_t samples[], + pj_size_t count) { pj_uint32_t sum = 0; @@ -140,27 +140,14 @@ PJ_DEF(pj_int32_t) pjmedia_silence_det_calc_avg_signal(const pj_int16_t samples[ return (pj_int32_t)(sum / count); } -PJ_DEF(pj_bool_t) pjmedia_silence_det_detect_silence( pjmedia_silence_det *sd, - const pj_int16_t samples[], - pj_size_t count, - pj_int32_t *p_level) +PJ_DEF(pj_bool_t) pjmedia_silence_det_apply( pjmedia_silence_det *sd, + pj_uint32_t level) { - pj_uint32_t level; pj_bool_t have_signal; /* Always return false if VAD is disabled */ - if (sd->mode == VAD_MODE_NONE) { - if (p_level) - *p_level = -1; + if (sd->mode == VAD_MODE_NONE) return PJ_FALSE; - } - - /* Calculate average signal level. */ - level = pjmedia_silence_det_calc_avg_signal(samples, count); - - /* Report to caller, if required. */ - if (p_level) - *p_level = level; /* Convert PCM level to ulaw */ level = linear2ulaw(level) ^ 0xff; @@ -265,5 +252,24 @@ PJ_DEF(pj_bool_t) pjmedia_silence_det_detect_silence( pjmedia_silence_det *sd, } return !sd->in_talk; + +} + + +PJ_DEF(pj_bool_t) pjmedia_silence_det_detect( pjmedia_silence_det *sd, + const pj_int16_t samples[], + pj_size_t count, + pj_int32_t *p_level) +{ + pj_uint32_t level; + + /* Calculate average signal level. */ + level = pjmedia_calc_avg_signal(samples, count); + + /* Report to caller, if required. */ + if (p_level) + *p_level = level; + + return pjmedia_silence_det_apply(sd, level); } diff --git a/pjmedia/src/pjmedia/stream.c b/pjmedia/src/pjmedia/stream.c index cbeaa2b7..6526d5cd 100644 --- a/pjmedia/src/pjmedia/stream.c +++ b/pjmedia/src/pjmedia/stream.c @@ -305,6 +305,8 @@ static pj_status_t put_frame( pjmedia_port *port, return status; } + //printf("p"); fflush(stdout); + /* Encapsulate. */ status = pjmedia_rtp_encode_rtp( &channel->rtp, channel->pt, 0, |