From 4a01731c11f29c2042459f37516ec313c4e8a495 Mon Sep 17 00:00:00 2001 From: Benny Prijono Date: Fri, 4 Aug 2006 11:08:00 +0000 Subject: More experimentation with AEC: (1) added media port to create bidirectional port from two unidirectional ports, (2) split AEC functionality into AEC algorithm (aec.h) and AEC media port (aec_port.h), (3) Added the AEC functionality in the sound_port.c. git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@646 74dad513-b988-da41-8d7b-12977e46ad98 --- pjmedia/src/pjmedia/aec_port.c | 95 +++++-------- pjmedia/src/pjmedia/aec_speex.c | 257 ++++++++++++++++++++++++++++++++++++ pjmedia/src/pjmedia/bidirectional.c | 75 +++++++++++ pjmedia/src/pjmedia/sound_port.c | 43 +++++- 4 files changed, 406 insertions(+), 64 deletions(-) create mode 100644 pjmedia/src/pjmedia/aec_speex.c create mode 100644 pjmedia/src/pjmedia/bidirectional.c (limited to 'pjmedia/src') diff --git a/pjmedia/src/pjmedia/aec_port.c b/pjmedia/src/pjmedia/aec_port.c index f772d859..aa56b175 100644 --- a/pjmedia/src/pjmedia/aec_port.c +++ b/pjmedia/src/pjmedia/aec_port.c @@ -17,7 +17,7 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include -#include "../pjmedia-codec/speex/speex_echo.h" +#include #include #include #include @@ -26,16 +26,13 @@ #define THIS_FILE "aec_port.c" #define SIGNATURE PJMEDIA_PORT_SIGNATURE('A', 'E', 'C', ' ') +#define BUF_COUNT 32 - -struct aec_port +struct aec { pjmedia_port base; pjmedia_port *dn_port; - SpeexEchoState *state; - pj_int16_t *tmp_frame; - pj_bool_t has_frame; - pj_int16_t *last_frame; + pjmedia_aec *aec; }; @@ -52,54 +49,37 @@ PJ_DEF(pj_status_t) pjmedia_aec_port_create( pj_pool_t *pool, pjmedia_port **p_port ) { const pj_str_t AEC = { "AEC", 3 }; - struct aec_port *aec_port; - int sampling_rate; + struct aec *aec; + pj_status_t status; PJ_ASSERT_RETURN(pool && dn_port && p_port, PJ_EINVAL); PJ_ASSERT_RETURN(dn_port->info.bits_per_sample==16 && tail_length, PJ_EINVAL); - /* Create and initialize the port */ - aec_port = pj_pool_zalloc(pool, sizeof(struct aec_port)); + /* Create the port and the AEC itself */ + aec = pj_pool_zalloc(pool, sizeof(struct aec)); - pjmedia_port_info_init(&aec_port->base.info, &AEC, SIGNATURE, + pjmedia_port_info_init(&aec->base.info, &AEC, SIGNATURE, dn_port->info.clock_rate, dn_port->info.channel_count, dn_port->info.bits_per_sample, dn_port->info.samples_per_frame); - aec_port->state = speex_echo_state_init(dn_port->info.samples_per_frame, - tail_length); - - /* Set sampling rate */ - sampling_rate = 0; - speex_echo_ctl(aec_port->state, SPEEX_ECHO_GET_SAMPLING_RATE, - &sampling_rate); - sampling_rate = dn_port->info.clock_rate; - speex_echo_ctl(aec_port->state, SPEEX_ECHO_SET_SAMPLING_RATE, - &sampling_rate); + status = pjmedia_aec_create(pool, dn_port->info.clock_rate, + dn_port->info.samples_per_frame, + tail_length, 0, &aec->aec); + if (status != PJ_SUCCESS) + return status; /* More init */ - aec_port->dn_port = dn_port; - aec_port->base.get_frame = &aec_get_frame; - aec_port->base.put_frame = &aec_put_frame; - aec_port->base.on_destroy = &aec_on_destroy; - - aec_port->last_frame = pj_pool_zalloc(pool, sizeof(pj_int16_t) * - dn_port->info.samples_per_frame); - aec_port->tmp_frame = pj_pool_zalloc(pool, sizeof(pj_int16_t) * - dn_port->info.samples_per_frame); + aec->dn_port = dn_port; + aec->base.get_frame = &aec_get_frame; + aec->base.put_frame = &aec_put_frame; + aec->base.on_destroy = &aec_on_destroy; /* Done */ - *p_port = &aec_port->base; - - PJ_LOG(4,(THIS_FILE, "AEC created for port %.*s, clock_rate=%d, " - "samples per frame=%d, tail length=%d ms", - (int)dn_port->info.name.slen, - dn_port->info.name.ptr, - dn_port->info.clock_rate, - dn_port->info.samples_per_frame, - tail_length * 1000 / dn_port->info.clock_rate)); + *p_port = &aec->base; + return PJ_SUCCESS; } @@ -107,58 +87,49 @@ PJ_DEF(pj_status_t) pjmedia_aec_port_create( pj_pool_t *pool, static pj_status_t aec_put_frame(pjmedia_port *this_port, const pjmedia_frame *frame) { - struct aec_port *aec_port = (struct aec_port*)this_port; + struct aec *aec = (struct aec*)this_port; PJ_ASSERT_RETURN(this_port->info.signature == SIGNATURE, PJ_EINVAL); - if (frame->type == PJMEDIA_FRAME_TYPE_NONE || !aec_port->has_frame) { - return pjmedia_port_put_frame(aec_port->dn_port, frame); + if (frame->type == PJMEDIA_FRAME_TYPE_NONE ) { + return pjmedia_port_put_frame(aec->dn_port, frame); } PJ_ASSERT_RETURN(frame->size == this_port->info.samples_per_frame * 2, PJ_EINVAL); - speex_echo_cancel(aec_port->state, - (const spx_int16_t*)frame->buf, - (const spx_int16_t*)aec_port->last_frame, - (spx_int16_t*)aec_port->tmp_frame, - NULL); - - pjmedia_copy_samples(frame->buf, aec_port->tmp_frame, - this_port->info.samples_per_frame); + pjmedia_aec_capture(aec->aec, frame->buf, 0); - return pjmedia_port_put_frame(aec_port->dn_port, frame); + return pjmedia_port_put_frame(aec->dn_port, frame); } static pj_status_t aec_get_frame( pjmedia_port *this_port, pjmedia_frame *frame) { - struct aec_port *aec_port = (struct aec_port*)this_port; + struct aec *aec = (struct aec*)this_port; pj_status_t status; PJ_ASSERT_RETURN(this_port->info.signature == SIGNATURE, PJ_EINVAL); - status = pjmedia_port_get_frame(aec_port->dn_port, frame); - if (status==PJ_SUCCESS && frame->type==PJMEDIA_FRAME_TYPE_AUDIO) { - aec_port->has_frame = PJ_TRUE; - pjmedia_copy_samples(aec_port->tmp_frame, frame->buf, - this_port->info.samples_per_frame); - } else { - aec_port->has_frame = PJ_FALSE; + status = pjmedia_port_get_frame(aec->dn_port, frame); + if (status!=PJ_SUCCESS || frame->type!=PJMEDIA_FRAME_TYPE_AUDIO) { + pjmedia_zero_samples(frame->buf, this_port->info.samples_per_frame); } + pjmedia_aec_playback(aec->aec, frame->buf); + return status; } static pj_status_t aec_on_destroy(pjmedia_port *this_port) { - struct aec_port *aec_port = (struct aec_port*)this_port; + struct aec *aec = (struct aec*)this_port; PJ_ASSERT_RETURN(this_port->info.signature == SIGNATURE, PJ_EINVAL); - speex_echo_state_destroy(aec_port->state); + pjmedia_aec_destroy(aec->aec); return PJ_SUCCESS; } diff --git a/pjmedia/src/pjmedia/aec_speex.c b/pjmedia/src/pjmedia/aec_speex.c new file mode 100644 index 00000000..76a8dd44 --- /dev/null +++ b/pjmedia/src/pjmedia/aec_speex.c @@ -0,0 +1,257 @@ +/* $Id$ */ +/* + * Copyright (C) 2003-2006 Benny Prijono + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define THIS_FILE "aec_speex.c" +#define BUF_COUNT 16 + + +struct frame +{ + pj_int16_t *buf; +}; + +struct pjmedia_aec +{ + SpeexEchoState *state; + unsigned samples_per_frame; + unsigned options; + pj_int16_t *tmp_frame; + + pj_lock_t *lock; /* To protect buffers, if required */ + + unsigned rpos; /* Index to get oldest frame. */ + unsigned wpos; /* Index to put newest frame. */ + struct frame frames[BUF_COUNT]; /* Playback frame buffers. */ +}; + + + +/* + * Create the AEC. + */ +PJ_DEF(pj_status_t) pjmedia_aec_create( pj_pool_t *pool, + unsigned clock_rate, + unsigned samples_per_frame, + unsigned tail_size, + unsigned options, + pjmedia_aec **p_aec ) +{ + pjmedia_aec *aec; + int sampling_rate; + unsigned i; + pj_status_t status; + + aec = pj_pool_zalloc(pool, sizeof(pjmedia_aec)); + PJ_ASSERT_RETURN(aec != NULL, PJ_ENOMEM); + + status = pj_lock_create_simple_mutex(pool, "aec%p", &aec->lock); + if (status != PJ_SUCCESS) + return status; + + aec->samples_per_frame = samples_per_frame; + aec->options = options; + + aec->state = speex_echo_state_init(samples_per_frame,tail_size); + if (aec->state == NULL) { + pj_lock_destroy(aec->lock); + return PJ_ENOMEM; + } + + /* Set sampling rate */ + sampling_rate = clock_rate; + speex_echo_ctl(aec->state, SPEEX_ECHO_SET_SAMPLING_RATE, + &sampling_rate); + + /* Create temporary frame for echo cancellation */ + aec->tmp_frame = pj_pool_zalloc(pool, sizeof(pj_int16_t) * + samples_per_frame); + PJ_ASSERT_RETURN(aec->tmp_frame != NULL, PJ_ENOMEM); + + /* Create internal playback buffers */ + for (i=0; iframes[i].buf = pj_pool_zalloc(pool, samples_per_frame * 2); + PJ_ASSERT_RETURN(aec->frames[i].buf != NULL, PJ_ENOMEM); + } + + + /* Done */ + *p_aec = aec; + + PJ_LOG(4,(THIS_FILE, "Echo canceller/AEC created, clock_rate=%d, " + "samples per frame=%d, tail length=%d ms", + clock_rate, + samples_per_frame, + tail_size * 1000 / clock_rate)); + return PJ_SUCCESS; + +} + + +/* + * Destroy AEC + */ +PJ_DEF(pj_status_t) pjmedia_aec_destroy(pjmedia_aec *aec ) +{ + PJ_ASSERT_RETURN(aec && aec->state, PJ_EINVAL); + + if (aec->state) { + speex_echo_state_destroy(aec->state); + aec->state = NULL; + } + + if (aec->lock) { + pj_lock_destroy(aec->lock); + aec->lock = NULL; + } + + return PJ_SUCCESS; +} + + +/* + * Let the AEC knows that a frame has been played to the speaker. + */ +PJ_DEF(pj_status_t) pjmedia_aec_playback(pjmedia_aec *aec, + pj_int16_t *play_frm ) +{ + /* Sanity checks */ + PJ_ASSERT_RETURN(aec && play_frm, PJ_EINVAL); + + /* The AEC must be configured to support internal playback buffer */ + PJ_ASSERT_RETURN(aec->frames[0].buf != NULL, PJ_EINVALIDOP); + + pj_lock_acquire(aec->lock); + + /* Check for overflows */ + if (aec->wpos == aec->rpos) { + PJ_LOG(5,(THIS_FILE, "AEC overflow (playback runs faster, " + "wpos=%d, rpos=%d)", + aec->wpos, aec->rpos)); + aec->rpos = (aec->wpos - BUF_COUNT/2) % BUF_COUNT; + speex_echo_state_reset(aec->state); + } + + /* Save fhe frame */ + pjmedia_copy_samples(aec->frames[aec->wpos].buf, + play_frm, aec->samples_per_frame); + aec->wpos = (aec->wpos+1) % BUF_COUNT; + + pj_lock_release(aec->lock); + + return PJ_SUCCESS; +} + + +/* + * Let the AEC knows that a frame has been captured from the microphone. + */ +PJ_DEF(pj_status_t) pjmedia_aec_capture( pjmedia_aec *aec, + pj_int16_t *rec_frm, + unsigned options ) +{ + pj_status_t status; + + /* Sanity checks */ + PJ_ASSERT_RETURN(aec && rec_frm, PJ_EINVAL); + + /* The AEC must be configured to support internal playback buffer */ + PJ_ASSERT_RETURN(aec->frames[0].buf != NULL, PJ_EINVALIDOP); + + /* Lock mutex */ + pj_lock_acquire(aec->lock); + + + /* Check for underflow */ + if (aec->rpos == aec->wpos) { + /* Return frame as it is */ + pj_lock_release(aec->lock); + + PJ_LOG(5,(THIS_FILE, "AEC underflow (capture runs faster than " + "playback, wpos=%d, rpos=%d)", + aec->wpos, aec->rpos)); + aec->rpos = (aec->wpos - BUF_COUNT/2) % BUF_COUNT; + speex_echo_state_reset(aec->state); + + return PJ_SUCCESS; + } + + + /* Cancel echo */ + status = pjmedia_aec_cancel_echo(aec, rec_frm, + aec->frames[aec->rpos].buf, options, + NULL); + + aec->rpos = (aec->rpos + 1) % BUF_COUNT; + + pj_lock_release(aec->lock); + return status; +} + + +/* + * Perform echo cancellation. + */ +PJ_DEF(pj_status_t) pjmedia_aec_cancel_echo( pjmedia_aec *aec, + pj_int16_t *rec_frm, + const pj_int16_t *play_frm, + unsigned options, + void *reserved ) +{ + unsigned level0, level1; + + /* Sanity checks */ + PJ_ASSERT_RETURN(aec && rec_frm && play_frm && options==0 && + reserved==NULL, PJ_EINVAL); + + /* Cancel echo, put output in temporary buffer */ + speex_echo_cancel(aec->state, (const spx_int16_t*)rec_frm, + (const spx_int16_t*)play_frm, + (spx_int16_t*)aec->tmp_frame, NULL); + +#if 0 + level0 = pjmedia_calc_avg_signal(rec_frm, aec->samples_per_frame); + level1 = pjmedia_calc_avg_signal(aec->tmp_frame, aec->samples_per_frame); + + if (level1 < level0) { + PJ_LOG(5,(THIS_FILE, "Input signal reduced from %d to %d", + level0, level1)); + } +#else + PJ_UNUSED_ARG(level0); + PJ_UNUSED_ARG(level1); +#endif + + /* Copy temporary buffer back to original rec_frm */ + pjmedia_copy_samples(rec_frm, aec->tmp_frame, aec->samples_per_frame); + + return PJ_SUCCESS; + +} + diff --git a/pjmedia/src/pjmedia/bidirectional.c b/pjmedia/src/pjmedia/bidirectional.c new file mode 100644 index 00000000..7f2cb0ba --- /dev/null +++ b/pjmedia/src/pjmedia/bidirectional.c @@ -0,0 +1,75 @@ +/* $Id$ */ +/* + * Copyright (C) 2003-2006 Benny Prijono + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +#include +#include + + +#define THIS_FILE "bidirectional.c" +#define SIGNATURE PJMEDIA_PORT_SIGNATURE('B', 'D', 'I', 'R') + +struct bidir_port +{ + pjmedia_port base; + pjmedia_port *get_port; + pjmedia_port *put_port; +}; + + +static pj_status_t put_frame(pjmedia_port *this_port, + const pjmedia_frame *frame) +{ + struct bidir_port *p = (struct bidir_port*)this_port; + return pjmedia_port_put_frame(p->put_port, frame); +} + + +static pj_status_t get_frame(pjmedia_port *this_port, + pjmedia_frame *frame) +{ + struct bidir_port *p = (struct bidir_port*)this_port; + return pjmedia_port_get_frame(p->get_port, frame); +} + + +PJ_DEF(pj_status_t) pjmedia_bidirectional_port_create( pj_pool_t *pool, + pjmedia_port *get_port, + pjmedia_port *put_port, + pjmedia_port **p_port ) +{ + struct bidir_port *port; + + port = pj_pool_zalloc(pool, sizeof(struct bidir_port)); + + pjmedia_port_info_init(&port->base.info, &get_port->info.name, SIGNATURE, + get_port->info.clock_rate, + get_port->info.channel_count, + get_port->info.bits_per_sample, + get_port->info.samples_per_frame); + + port->get_port = get_port; + port->put_port = put_port; + + port->base.get_frame = &get_frame; + port->base.put_frame = &put_frame; + + *p_port = &port->base; + + return PJ_SUCCESS; +} + diff --git a/pjmedia/src/pjmedia/sound_port.c b/pjmedia/src/pjmedia/sound_port.c index 08f2232a..ee3e61c9 100644 --- a/pjmedia/src/pjmedia/sound_port.c +++ b/pjmedia/src/pjmedia/sound_port.c @@ -17,6 +17,7 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include +#include #include #include #include @@ -24,9 +25,16 @@ #include #include /* pj_memset() */ +#ifndef PJMEDIA_SOUND_HAS_AEC +# define PJMEDIA_SOUND_HAS_AEC 1 +#endif -//#define SIMULATE_LOST_PCT 20 +#if defined(PJMEDIA_SOUND_HAS_AEC) && PJMEDIA_SOUND_HAS_AEC!=0 +# include +#endif +//#define SIMULATE_LOST_PCT 20 +#define AEC_TAIL 500 /* in ms */ #define THIS_FILE "sound_port.c" @@ -48,6 +56,7 @@ struct pjmedia_snd_port pjmedia_port *port; unsigned options; + pjmedia_aec *aec; pjmedia_plc *plc; unsigned clock_rate; @@ -105,6 +114,11 @@ static pj_status_t play_cb(/* in */ void *user_data, if (snd_port->plc) pjmedia_plc_save(snd_port->plc, output); + if (snd_port->aec) { + pjmedia_aec_playback(snd_port->aec, output); + } + + return PJ_SUCCESS; no_frame: @@ -121,6 +135,10 @@ no_frame: } + if (snd_port->aec) { + pjmedia_aec_playback(snd_port->aec, output); + } + return PJ_SUCCESS; } @@ -131,13 +149,18 @@ no_frame: */ static pj_status_t rec_cb(/* in */ void *user_data, /* in */ pj_uint32_t timestamp, - /* in */ const void *input, + /* in */ void *input, /* in*/ unsigned size) { pjmedia_snd_port *snd_port = user_data; pjmedia_port *port; pjmedia_frame frame; + /* Cancel echo */ + if (snd_port->aec) { + pjmedia_aec_capture(snd_port->aec, input, 0); + } + /* We're risking accessing the port without holding any mutex. * It's possible that port is disconnected then destroyed while * we're trying to access it. @@ -227,6 +250,16 @@ static pj_status_t start_sound_device( pj_pool_t *pool, snd_port->plc = NULL; } + /* Create AEC only when direction is full duplex */ + if (snd_port->dir == PJMEDIA_DIR_CAPTURE_PLAYBACK) { + status = pjmedia_aec_create(pool, snd_port->clock_rate, + snd_port->samples_per_frame, + snd_port->clock_rate * AEC_TAIL / 1000, + 0, &snd_port->aec); + if (status != PJ_SUCCESS) + snd_port->aec = NULL; + } + /* Start sound stream. */ status = pjmedia_snd_stream_start(snd_port->snd_stream); if (status != PJ_SUCCESS) { @@ -252,6 +285,12 @@ static pj_status_t stop_sound_device( pjmedia_snd_port *snd_port ) snd_port->snd_stream = NULL; } + /* Destroy AEC */ + if (snd_port->aec) { + pjmedia_aec_destroy(snd_port->aec); + snd_port->aec = NULL; + } + return PJ_SUCCESS; } -- cgit v1.2.3