Implement ticket #40: support for asymmetric encoding/decoding ptime (e.g. with iLBC when local and remote have different mode)

git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@874 74dad513-b988-da41-8d7b-12977e46ad98

7 files changed, 221 insertions, 54 deletions

diff --git a/pjmedia/include/pjmedia/codec.h b/pjmedia/include/pjmedia/codec.h
index 3838e8a7..f93e9333 100644
--- a/pjmedia/include/pjmedia/codec.h
+++ b/pjmedia/include/pjmedia/codec.h
@@ -253,7 +253,9 @@ typedef struct pjmedia_codec_param
        unsigned	   clock_rate;		/**< Sampling rate in Hz	    */
        unsigned	   channel_cnt;		/**< Channel count.		    */
        pj_uint32_t avg_bps;		/**< Average bandwidth in bits/sec  */
-       pj_uint16_t frm_ptime;		/**< Base frame ptime in msec.	    */
+       pj_uint16_t frm_ptime;		/**< Decoder frame ptime in msec.   */
+       pj_uint16_t enc_ptime;		/**< Encoder ptime, or zero if it's
+					     equal to decoder ptime.	    */
        pj_uint8_t  pcm_bits_per_sample;	/**< Bits/sample in the PCM side    */
        pj_uint8_t  pt;			/**< Payload type.		    */
     } info;
@@ -306,7 +308,10 @@ typedef struct pjmedia_codec_op
 			pj_pool_t *pool );
 
     /** 
-     * Open the codec and initialize with the specified parameter..
+     * Open the codec and initialize with the specified parameter.
+     * Upon successful initialization, the codec may modify the parameter
+     * and fills in the unspecified values (such as enc_ptime, when
+     * encoder ptime is different than decoder ptime).
      *
      * @param codec	The codec instance.
      * @param param	Codec initialization parameter.
@@ -314,7 +319,7 @@ typedef struct pjmedia_codec_op
      * @return		PJ_SUCCESS on success.
      */
     pj_status_t	(*open)(pjmedia_codec *codec, 
-			const pjmedia_codec_param *param );
+			pjmedia_codec_param *param );
 
     /** 
      * Close and shutdown codec, releasing all resources allocated by
diff --git a/pjmedia/src/pjmedia-codec/gsm.c b/pjmedia/src/pjmedia-codec/gsm.c
index 00b7d3fd..f774a5e5 100644
--- a/pjmedia/src/pjmedia-codec/gsm.c
+++ b/pjmedia/src/pjmedia-codec/gsm.c
@@ -54,7 +54,7 @@ static pj_status_t gsm_dealloc_codec( pjmedia_codec_factory *factory,
 static pj_status_t  gsm_codec_init( pjmedia_codec *codec, 
 				    pj_pool_t *pool );
 static pj_status_t  gsm_codec_open( pjmedia_codec *codec, 
-				    const pjmedia_codec_param *attr );
+				    pjmedia_codec_param *attr );
 static pj_status_t  gsm_codec_close( pjmedia_codec *codec );
 static pj_status_t  gsm_codec_modify(pjmedia_codec *codec, 
 				     const pjmedia_codec_param *attr );
@@ -388,7 +388,7 @@ static pj_status_t gsm_codec_init( pjmedia_codec *codec,
  * Open codec.
  */
 static pj_status_t gsm_codec_open( pjmedia_codec *codec, 
-				   const pjmedia_codec_param *attr )
+				   pjmedia_codec_param *attr )
 {
     struct gsm_data *gsm_data = codec->codec_data;
 
@@ -440,7 +440,7 @@ static pj_status_t  gsm_codec_modify(pjmedia_codec *codec,
     struct gsm_data *gsm_data = codec->codec_data;
 
     pj_assert(gsm_data != NULL);
-    pj_assert(gsm_data->encoder == NULL && gsm_data->decoder == NULL);
+    pj_assert(gsm_data->encoder != NULL && gsm_data->decoder != NULL);
 
     gsm_data->vad_enabled = (attr->setting.vad != 0);
     gsm_data->plc_enabled = (attr->setting.plc != 0);
diff --git a/pjmedia/src/pjmedia-codec/ilbc.c b/pjmedia/src/pjmedia-codec/ilbc.c
index a16ed1cf..a21b6889 100644
--- a/pjmedia/src/pjmedia-codec/ilbc.c
+++ b/pjmedia/src/pjmedia-codec/ilbc.c
@@ -63,7 +63,7 @@ static pj_status_t ilbc_dealloc_codec(pjmedia_codec_factory *factory,
 static pj_status_t  ilbc_codec_init(pjmedia_codec *codec, 
 				    pj_pool_t *pool );
 static pj_status_t  ilbc_codec_open(pjmedia_codec *codec, 
-				    const pjmedia_codec_param *attr );
+				    pjmedia_codec_param *attr );
 static pj_status_t  ilbc_codec_close(pjmedia_codec *codec );
 static pj_status_t  ilbc_codec_modify(pjmedia_codec *codec, 
 				      const pjmedia_codec_param *attr );
@@ -364,20 +364,15 @@ static pj_status_t ilbc_codec_init(pjmedia_codec *codec,
  * Open codec.
  */
 static pj_status_t ilbc_codec_open(pjmedia_codec *codec, 
-				   const pjmedia_codec_param *param_attr )
+				   pjmedia_codec_param *attr )
 {
     struct ilbc_codec *ilbc_codec = (struct ilbc_codec*)codec;
-    pjmedia_codec_param attr_copy, *attr;
     pj_status_t status;
 
     pj_assert(ilbc_codec != NULL);
     pj_assert(ilbc_codec->enc_ready == PJ_FALSE && 
 	      ilbc_codec->dec_ready == PJ_FALSE);
 
-    /* Copy param to temporary location since we need to modify fmtp_mode */
-    pj_memcpy(&attr_copy, param_attr, sizeof(*param_attr));
-    attr = &attr_copy;
-
     /* Decoder mode must be set */
     PJ_ASSERT_RETURN(attr->setting.dec_fmtp_mode==20 ||
 		     attr->setting.dec_fmtp_mode==30, PJMEDIA_CODEC_EINMODE);
@@ -392,6 +387,13 @@ static pj_status_t ilbc_codec_open(pjmedia_codec *codec,
     PJ_ASSERT_RETURN(attr->setting.enc_fmtp_mode==20 ||
 		     attr->setting.enc_fmtp_mode==30, PJMEDIA_CODEC_EINMODE);
 
+    /* Update enc_ptime in the param */
+    if (attr->setting.enc_fmtp_mode != attr->setting.dec_fmtp_mode) {
+	attr->info.enc_ptime = attr->setting.enc_fmtp_mode;
+    } else {
+	attr->info.enc_ptime = 0;
+    }
+
     /* Create enc */
     ilbc_codec->enc_frame_size = initEncode(&ilbc_codec->enc, 
 					    attr->setting.enc_fmtp_mode);
diff --git a/pjmedia/src/pjmedia-codec/l16.c b/pjmedia/src/pjmedia-codec/l16.c
index 18b86708..77d76796 100644
--- a/pjmedia/src/pjmedia-codec/l16.c
+++ b/pjmedia/src/pjmedia-codec/l16.c
@@ -59,7 +59,7 @@ static pj_status_t l16_dealloc_codec( pjmedia_codec_factory *factory,
 static pj_status_t  l16_init( pjmedia_codec *codec, 
 			       pj_pool_t *pool );
 static pj_status_t  l16_open( pjmedia_codec *codec, 
-			       const pjmedia_codec_param *attr );
+			      pjmedia_codec_param *attr );
 static pj_status_t  l16_close( pjmedia_codec *codec );
 static pj_status_t  l16_modify(pjmedia_codec *codec, 
 			       const pjmedia_codec_param *attr );
@@ -489,7 +489,7 @@ static pj_status_t l16_init( pjmedia_codec *codec, pj_pool_t *pool )
 }
 
 static pj_status_t l16_open(pjmedia_codec *codec, 
-			    const pjmedia_codec_param *attr )
+			    pjmedia_codec_param *attr )
 {
     /* Nothing to do.. */
     PJ_UNUSED_ARG(codec);
diff --git a/pjmedia/src/pjmedia-codec/speex_codec.c b/pjmedia/src/pjmedia-codec/speex_codec.c
index 6c8a1602..a0c61b94 100644
--- a/pjmedia/src/pjmedia-codec/speex_codec.c
+++ b/pjmedia/src/pjmedia-codec/speex_codec.c
@@ -59,7 +59,7 @@ static pj_status_t spx_dealloc_codec( pjmedia_codec_factory *factory,
 static pj_status_t  spx_codec_init( pjmedia_codec *codec, 
 				    pj_pool_t *pool );
 static pj_status_t  spx_codec_open( pjmedia_codec *codec, 
-				    const pjmedia_codec_param *attr );
+				    pjmedia_codec_param *attr );
 static pj_status_t  spx_codec_close( pjmedia_codec *codec );
 static pj_status_t  spx_codec_modify(pjmedia_codec *codec, 
 				     const pjmedia_codec_param *attr );
@@ -547,7 +547,7 @@ static pj_status_t spx_codec_init( pjmedia_codec *codec,
  * Open codec.
  */
 static pj_status_t spx_codec_open( pjmedia_codec *codec, 
-				   const pjmedia_codec_param *attr )
+				   pjmedia_codec_param *attr )
 {
     struct spx_private *spx;
     int id, tmp;
diff --git a/pjmedia/src/pjmedia/g711.c b/pjmedia/src/pjmedia/g711.c
index 1e9c0973..99b4ec3d 100644
--- a/pjmedia/src/pjmedia/g711.c
+++ b/pjmedia/src/pjmedia/g711.c
@@ -62,7 +62,7 @@ static pj_status_t g711_dealloc_codec( pjmedia_codec_factory *factory,
 static pj_status_t  g711_init( pjmedia_codec *codec, 
 			       pj_pool_t *pool );
 static pj_status_t  g711_open( pjmedia_codec *codec, 
-			       const pjmedia_codec_param *attr );
+			       pjmedia_codec_param *attr );
 static pj_status_t  g711_close( pjmedia_codec *codec );
 static pj_status_t  g711_modify(pjmedia_codec *codec, 
 			        const pjmedia_codec_param *attr );
@@ -397,7 +397,7 @@ static pj_status_t g711_init( pjmedia_codec *codec, pj_pool_t *pool )
 }
 
 static pj_status_t g711_open(pjmedia_codec *codec, 
-			     const pjmedia_codec_param *attr )
+			     pjmedia_codec_param *attr )
 {
     struct g711_private *priv = codec->codec_data;
     priv->pt = attr->info.pt;
diff --git a/pjmedia/src/pjmedia/stream.c b/pjmedia/src/pjmedia/stream.c
index 9c032bbf..748c222f 100644
--- a/pjmedia/src/pjmedia/stream.c
+++ b/pjmedia/src/pjmedia/stream.c
@@ -88,6 +88,17 @@ struct pjmedia_stream
 
     pjmedia_codec	    *codec;	    /**< Codec instance being used. */
     pjmedia_codec_param	     codec_param;   /**< Codec param.		    */
+    pj_int16_t		    *enc_buf;	    /**< Encoding buffer, when enc's
+						 ptime is different than dec.
+						 Otherwise it's NULL.	    */
+
+    unsigned		     enc_samples_per_frame;
+    unsigned		     enc_buf_size;  /**< Encoding buffer size, in
+						 samples.		    */
+    unsigned		     enc_buf_pos;   /**< First position in buf.	    */
+    unsigned		     enc_buf_count; /**< Number of samples in the
+						 encoding buffer.	    */
+
     unsigned		     vad_enabled;   /**< VAD enabled in param.	    */
     unsigned		     frame_size;    /**< Size of encoded base frame.*/
     pj_bool_t		     is_streaming;  /**< Currently streaming?. This
@@ -442,33 +453,87 @@ static void check_tx_rtcp(pjmedia_stream *stream, pj_uint32_t timestamp)
 
 
 /**
- * put_frame()
- *
- * This callback is called by upstream component when it has PCM frame
- * to transmit. This function encodes the PCM frame, pack it into
- * RTP packet, and transmit to peer.
+ * Rebuffer the frame when encoder and decoder has different ptime
+ * (such as when different iLBC modes are used by local and remote)
  */
-static pj_status_t put_frame( pjmedia_port *port, 
-			      const pjmedia_frame *frame )
+static void rebuffer(pjmedia_stream *stream,
+		     pjmedia_frame *frame)
+{
+    /* How many samples are needed */
+    unsigned count;
+
+    /* Normalize frame */
+    if (frame->type != PJMEDIA_FRAME_TYPE_AUDIO)
+	frame->size = 0;
+
+    /* Remove used frame from the buffer. */
+    if (stream->enc_buf_pos) {
+	if (stream->enc_buf_count) {
+	    pj_memmove(stream->enc_buf,
+		       stream->enc_buf + stream->enc_buf_pos,
+		       (stream->enc_buf_count << 1));
+	}
+	stream->enc_buf_pos = 0;
+    }
+
+    /* Make sure we have space to store the new frame */
+    pj_assert(stream->enc_buf_count + (frame->size >> 1) <
+		stream->enc_buf_size);
+
+    /* Append new frame to the buffer */
+    if (frame->size) {
+	pj_memcpy(stream->enc_buf + stream->enc_buf_count,
+		  frame->buf, frame->size);
+	stream->enc_buf_count += (frame->size >> 1);
+    }
+
+    /* How many samples are needed */
+    count = stream->codec_param.info.enc_ptime * 
+		stream->port.info.clock_rate / 1000;
+
+    /* See if we have enough samples */
+    if (stream->enc_buf_count >= count) {
+
+	frame->type = PJMEDIA_FRAME_TYPE_AUDIO;
+	frame->buf = stream->enc_buf;
+	frame->size = (count << 1);
+
+	stream->enc_buf_pos = count;
+	stream->enc_buf_count -= count;
+
+    } else {
+	/* We don't have enough samples */
+	frame->type = PJMEDIA_FRAME_TYPE_NONE;
+    }
+}
+
+
+/**
+ * put_frame_imp()
+ */
+static pj_status_t put_frame_imp( pjmedia_port *port, 
+				  const pjmedia_frame *frame )
 {
     pjmedia_stream *stream = port->port_data.pdata;
     pjmedia_channel *channel = stream->enc;
     pj_status_t status = 0;
-    struct pjmedia_frame frame_out;
+    pjmedia_frame frame_out;
     unsigned ts_len, samples_per_frame;
-    pjmedia_frame tmp_in_frame;
     void *rtphdr;
     int rtphdrlen;
 
 
     /* Don't do anything if stream is paused */
-    if (channel->paused)
+    if (channel->paused) {
+	stream->enc_buf_pos = stream->enc_buf_count = 0;
 	return PJ_SUCCESS;
-
+    }
 
     /* Number of samples in the frame */
-    //ts_len = frame->size / 2;
-    ts_len = port->info.samples_per_frame;
+    if (frame->type == PJMEDIA_FRAME_TYPE_AUDIO)
+	ts_len = (frame->size >> 1);
+    else
+	ts_len = 0;
 
     /* Increment transmit duration */
     stream->tx_duration += ts_len;
@@ -478,26 +543,7 @@ static pj_status_t put_frame( pjmedia_port *port,
     frame_out.size = 0;
 
     /* Calculate number of samples per frame */
-    samples_per_frame = stream->codec_param.info.frm_ptime *
-			stream->codec_param.info.clock_rate *
-			stream->codec_param.info.channel_cnt /
-			1000;
-
-    /* If VAD is temporarily disabled during creation, feed zero PCM frame
-     * to the codec.
-     */
-    if (stream->vad_enabled != stream->codec_param.setting.vad &&
-	stream->vad_enabled != 0 &&
-	frame->type == PJMEDIA_FRAME_TYPE_NONE &&
-	samples_per_frame <= ZERO_PCM_MAX_SIZE)
-    {
-	pj_memcpy(&tmp_in_frame, frame, sizeof(pjmedia_frame));
-	frame = &tmp_in_frame;
-
-	tmp_in_frame.buf = zero_frame;
-	tmp_in_frame.size = samples_per_frame * 2;
-	tmp_in_frame.type = PJMEDIA_FRAME_TYPE_AUDIO;
-    }
+    samples_per_frame = stream->enc_samples_per_frame;
 
 
     /* If we have DTMF digits in the queue, transmit the digits. 
@@ -628,6 +674,42 @@ static pj_status_t put_frame( pjmedia_port *port,
     /* Update stat */
     pjmedia_rtcp_tx_rtp(&stream->rtcp, frame_out.size);
 
+    return PJ_SUCCESS;
+}
+
+
+/**
+ * put_frame()
+ *
+ * This callback is called by upstream component when it has PCM frame
+ * to transmit. This function encodes the PCM frame, pack it into
+ * RTP packet, and transmit to peer.
+ */
+static pj_status_t put_frame( pjmedia_port *port, 
+			      const pjmedia_frame *frame )
+{
+    pjmedia_stream *stream = port->port_data.pdata;
+    pjmedia_frame tmp_in_frame;
+    unsigned samples_per_frame;
+
+    samples_per_frame = stream->enc_samples_per_frame;
+
+    /* If VAD is temporarily disabled during creation, feed zero PCM frame
+     * to the codec.
+     */
+    if (stream->vad_enabled != stream->codec_param.setting.vad &&
+	stream->vad_enabled != 0 &&
+	frame->type == PJMEDIA_FRAME_TYPE_NONE &&
+	samples_per_frame <= ZERO_PCM_MAX_SIZE)
+    {
+	pj_memcpy(&tmp_in_frame, frame, sizeof(pjmedia_frame));
+	frame = &tmp_in_frame;
+
+	tmp_in_frame.buf = zero_frame;
+	tmp_in_frame.size = samples_per_frame * 2;
+	tmp_in_frame.type = PJMEDIA_FRAME_TYPE_AUDIO;
+    }
+
     /* If VAD is temporarily disabled during creation, enable it
      * after transmitting for VAD_SUSPEND_SEC seconds.
      */
@@ -641,9 +723,52 @@ static pj_status_t put_frame( pjmedia_port *port,
     }
 
 
-    return PJ_SUCCESS;
+    /* If encoder has different ptime than decoder, then the frame must
+     * be passed through the encoding buffer via rebuffer() function.
+     */
+    if (stream->enc_buf != NULL) {
+	pjmedia_frame tmp_rebuffer_frame;
+	pj_status_t status = PJ_SUCCESS;
+
+	/* Copy original frame to temporary frame since we need 
+	 * to modify it.
+	 */
+	pj_memcpy(&tmp_rebuffer_frame, frame, sizeof(pjmedia_frame));
+
+	/* Loop while we have full frame in enc_buffer */
+	for (;;) {
+	    pj_status_t st;
+
+	    /* Run rebuffer() */
+	    rebuffer(stream, &tmp_rebuffer_frame);
+
+	    /* Process this frame */
+	    st = put_frame_imp(port, &tmp_rebuffer_frame);
+	    if (st != PJ_SUCCESS)
+		status = st;
+
+	    /* If we still have full frame in the buffer, re-run
+	     * rebuffer() with NULL frame.
+	     */
+	    if (stream->enc_buf_count >= stream->enc_samples_per_frame) {
+
+		tmp_rebuffer_frame.type = PJMEDIA_FRAME_TYPE_NONE;
+
+	    } else {
+
+		/* Otherwise break */
+		break;
+	    }
+	}
+
+	return status;
+
+    } else {
+	return put_frame_imp(port, frame);
+    }
 }
 
+
 #if 0
 static void dump_bin(const char *buf, unsigned len)
 {
@@ -1071,6 +1196,40 @@ PJ_DEF(pj_status_t) pjmedia_stream_create( pjmedia_endpt *endpt,
     if (status != PJ_SUCCESS)
 	goto err_cleanup;
 
+    /* If encoder and decoder's ptime are asymmetric, then we need to
+     * create buffer on the encoder side. This could happen for example
+     * with iLBC 
+     */
+    if (stream->codec_param.info.enc_ptime!=0 &&
+	stream->codec_param.info.enc_ptime!=stream->codec_param.info.frm_ptime)
+    {
+	unsigned ptime;
+
+	stream->enc_samples_per_frame = stream->codec_param.info.enc_ptime *
+					stream->port.info.clock_rate / 1000;
+
+	/* Set buffer size as twice the largest ptime value between
+	 * stream's ptime, encoder ptime, or decoder ptime.
+	 */
+
+	ptime = stream->port.info.samples_per_frame * 1000 /
+		stream->port.info.clock_rate;
+
+	if (stream->codec_param.info.enc_ptime > ptime)
+	    ptime = stream->codec_param.info.enc_ptime;
+
+	if (stream->codec_param.info.frm_ptime > ptime)
+	    ptime = stream->codec_param.info.frm_ptime;
+
+	ptime <<= 1;
+
+	/* Allocate buffer */
+	stream->enc_buf_size = stream->port.info.clock_rate * ptime / 1000;
+	stream->enc_buf = pj_pool_alloc(pool, stream->enc_buf_size * 2);
+
+    } else {
+	stream->enc_samples_per_frame = stream->port.info.samples_per_frame;
+    }
 
     /* Initially disable the VAD in the stream, to help traverse NAT better */
     stream->vad_enabled = stream->codec_param.setting.vad;
@@ -1083,9 +1242,10 @@ PJ_DEF(pj_status_t) pjmedia_stream_create( pjmedia_endpt *endpt,
 
     /* Get the frame size: */
 
-    stream->frame_size = (stream->codec_param.info.avg_bps / 8) * 
+    stream->frame_size = ((stream->codec_param.info.avg_bps + 7) / 8) * 
 			  stream->codec_param.info.frm_ptime / 1000;
 
+
     /* Init RTCP session: */
 
     pjmedia_rtcp_init(&stream->rtcp, stream->port.info.name.ptr,