From cc83068c55ebed1f62a37c567ea9730aca1be4b3 Mon Sep 17 00:00:00 2001 From: Benny Prijono Date: Sun, 23 Oct 2011 05:23:55 +0000 Subject: Fixed #1392: Immediately decode incoming frames to avoid loosing key frames when the jbuf is full and improve latency git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@3835 74dad513-b988-da41-8d7b-12977e46ad98 --- pjmedia/include/pjmedia/config.h | 10 +++ pjmedia/include/pjmedia/jbuf.h | 9 +++ pjmedia/src/pjmedia/jbuf.c | 4 + pjmedia/src/pjmedia/vid_stream.c | 153 +++++++++++++++++++++++++++++---------- 4 files changed, 138 insertions(+), 38 deletions(-) diff --git a/pjmedia/include/pjmedia/config.h b/pjmedia/include/pjmedia/config.h index 58f56e52..7e41402d 100644 --- a/pjmedia/include/pjmedia/config.h +++ b/pjmedia/include/pjmedia/config.h @@ -1044,6 +1044,16 @@ #endif +/** + * Video stream will discard old picture from the jitter buffer as soon as + * new picture is received, to reduce latency. + * + * Default: 0 + */ +#ifndef PJMEDIA_VID_STREAM_SKIP_PACKETS_TO_REDUCE_LATENCY +# define PJMEDIA_VID_STREAM_SKIP_PACKETS_TO_REDUCE_LATENCY 0 +#endif + /** * @} */ diff --git a/pjmedia/include/pjmedia/jbuf.h b/pjmedia/include/pjmedia/jbuf.h index 83ee852b..2ae4ed6c 100644 --- a/pjmedia/include/pjmedia/jbuf.h +++ b/pjmedia/include/pjmedia/jbuf.h @@ -389,6 +389,15 @@ PJ_DECL(void) pjmedia_jbuf_peek_frame(pjmedia_jbuf *jb, PJ_DECL(unsigned) pjmedia_jbuf_remove_frame(pjmedia_jbuf *jb, unsigned frame_cnt); +/** + * Check if the jitter buffer is full. + * + * @param jb The jitter buffer. + * + * @return PJ_TRUE if it is full. + */ +PJ_DECL(pj_bool_t) pjmedia_jbuf_is_full(const pjmedia_jbuf *jb); + /** * Get jitter buffer current state/settings. diff --git a/pjmedia/src/pjmedia/jbuf.c b/pjmedia/src/pjmedia/jbuf.c index 9ae84b64..52a3b0eb 100644 --- a/pjmedia/src/pjmedia/jbuf.c +++ b/pjmedia/src/pjmedia/jbuf.c @@ -635,6 +635,10 @@ PJ_DEF(pj_status_t) pjmedia_jbuf_destroy(pjmedia_jbuf *jb) return jb_framelist_destroy(&jb->jb_framelist); } +PJ_DEF(pj_bool_t) pjmedia_jbuf_is_full(const pjmedia_jbuf *jb) +{ + return jb->jb_framelist.size == jb->jb_framelist.max_count; +} static void jbuf_calculate_jitter(pjmedia_jbuf *jb) { diff --git a/pjmedia/src/pjmedia/vid_stream.c b/pjmedia/src/pjmedia/vid_stream.c index eefb1479..b22e82ed 100644 --- a/pjmedia/src/pjmedia/vid_stream.c +++ b/pjmedia/src/pjmedia/vid_stream.c @@ -68,7 +68,6 @@ # define PJMEDIA_VSTREAM_INC 1000 #endif - /** * Media channel. */ @@ -81,7 +80,6 @@ typedef struct pjmedia_vid_channel pj_bool_t paused; /**< Paused?. */ void *buf; /**< Output buffer. */ unsigned buf_size; /**< Size of output buffer. */ - unsigned buf_len; /**< Length of data in buffer. */ pjmedia_rtp_session rtp; /**< RTP session. */ } pjmedia_vid_channel; @@ -121,6 +119,9 @@ struct pjmedia_vid_stream pj_uint32_t rtcp_interval; /**< Interval, in timestamp. */ pj_bool_t initial_rr; /**< Initial RTCP RR sent */ + unsigned dec_max_size; /**< Size of decoded/raw picture*/ + pjmedia_frame dec_frame; /**< Current decoded frame. */ + unsigned frame_size; /**< Size of encoded base frame.*/ unsigned frame_ts_len; /**< Frame length in timestamp. */ @@ -149,6 +150,9 @@ struct pjmedia_vid_stream pjmedia_event_publisher epub; /**< To publish events */ }; +/* Prototypes */ +static pj_status_t decode_frame(pjmedia_vid_stream *stream, + pjmedia_frame *frame); /* * Print error. @@ -635,11 +639,47 @@ static void on_rx_rtp( void *data, goto on_return; } + pj_mutex_lock( stream->jb_mutex ); + + /* Quickly see if there may be a full picture in the jitter buffer, and + * decode them if so. More thorough check will be done in decode_frame(). + */ + if ((pj_ntohl(hdr->ts) != stream->dec_frame.timestamp.u32.lo) || hdr->m) { + if (PJMEDIA_VID_STREAM_SKIP_PACKETS_TO_REDUCE_LATENCY) { + /* Always decode whenever we have picture in jb and + * overwrite already decoded picture if necessary + */ + pj_size_t old_size = stream->dec_frame.size; + + stream->dec_frame.size = stream->dec_max_size; + if (decode_frame(stream, &stream->dec_frame) != PJ_SUCCESS) { + stream->dec_frame.size = old_size; + } + } else { + /* Only decode if we don't already have decoded one, + * unless the jb is full. + */ + pj_bool_t can_decode = PJ_FALSE; + + if (pjmedia_jbuf_is_full(stream->jb)) { + can_decode = PJ_TRUE; + } + else if (stream->dec_frame.size == 0) { + can_decode = PJ_TRUE; + } + + if (can_decode) { + stream->dec_frame.size = stream->dec_max_size; + if (decode_frame(stream, &stream->dec_frame) != PJ_SUCCESS) { + stream->dec_frame.size = 0; + } + } + } + } /* Put "good" packet to jitter buffer, or reset the jitter buffer * when RTP session is restarted. */ - pj_mutex_lock( stream->jb_mutex ); if (seq_st.status.flag.restart) { status = pjmedia_jbuf_reset(stream->jb); PJ_LOG(4,(channel->port.info.name.ptr, "Jitter buffer reset")); @@ -815,11 +855,11 @@ static pj_status_t put_frame(pjmedia_port *port, enum { COUNT_TO_REPORT = 20 }; if (stream->send_err_cnt++ == 0) { LOGERR_((channel->port.info.name.ptr, - "Transport send_rtp() error (repeated %d times)", + "Transport send_rtp() error", status)); - if (stream->send_err_cnt > COUNT_TO_REPORT) - stream->send_err_cnt = 0; } + if (stream->send_err_cnt > COUNT_TO_REPORT) + stream->send_err_cnt = 0; /* Ignore this error */ } @@ -873,10 +913,10 @@ static pj_status_t put_frame(pjmedia_port *port, return PJ_SUCCESS; } -static pj_status_t get_frame(pjmedia_port *port, - pjmedia_frame *frame) +/* Decode one image from jitter buffer */ +static pj_status_t decode_frame(pjmedia_vid_stream *stream, + pjmedia_frame *frame) { - pjmedia_vid_stream *stream = (pjmedia_vid_stream*) port->port_data.pdata; pjmedia_vid_channel *channel = stream->dec; pj_uint32_t last_ts = 0; int frm_first_seq = 0, frm_last_seq = 0; @@ -884,19 +924,9 @@ static pj_status_t get_frame(pjmedia_port *port, unsigned cnt; pj_status_t status; - /* Return no frame is channel is paused */ - if (channel->paused) { - frame->type = PJMEDIA_FRAME_TYPE_NONE; - return PJ_SUCCESS; - } - /* Repeat get payload from the jitter buffer until all payloads with same * timestamp are collected. */ - channel->buf_len = 0; - - /* Lock jitter buffer mutex first */ - pj_mutex_lock( stream->jb_mutex ); /* Check if we got a decodable frame */ for (cnt=0; ; ++cnt) { @@ -927,10 +957,8 @@ static pj_status_t get_frame(pjmedia_port *port, unsigned i; /* Generate frame bitstream from the payload */ - channel->buf_len = 0; - if (cnt > stream->rx_frame_cnt) { - PJ_LOG(1,(port->info.name.ptr, + PJ_LOG(1,(channel->port.info.name.ptr, "Discarding %u frames because array is full!", cnt - stream->rx_frame_cnt)); pjmedia_jbuf_remove_frame(stream->jb, cnt - stream->rx_frame_cnt); @@ -966,7 +994,7 @@ static pj_status_t get_frame(pjmedia_port *port, stream->rx_frames, frame->size, frame); if (status != PJ_SUCCESS) { - LOGERR_((port->info.name.ptr, "codec decode() error", + LOGERR_((channel->port.info.name.ptr, "codec decode() error", status)); frame->type = PJMEDIA_FRAME_TYPE_NONE; frame->size = 0; @@ -975,9 +1003,6 @@ static pj_status_t get_frame(pjmedia_port *port, pjmedia_jbuf_remove_frame(stream->jb, cnt); } - /* Unlock jitter buffer mutex. */ - pj_mutex_unlock( stream->jb_mutex ); - /* Learn remote frame rate after successful decoding */ if (0 && frame->type == PJMEDIA_FRAME_TYPE_VIDEO && frame->size) { @@ -1030,10 +1055,54 @@ static pj_status_t get_frame(pjmedia_port *port, stream->last_dec_ts = last_ts; } - return PJ_SUCCESS; + return got_frame ? PJ_SUCCESS : PJ_ENOTFOUND; } +static pj_status_t get_frame(pjmedia_port *port, + pjmedia_frame *frame) +{ + pjmedia_vid_stream *stream = (pjmedia_vid_stream*) port->port_data.pdata; + pjmedia_vid_channel *channel = stream->dec; + + /* Return no frame is channel is paused */ + if (channel->paused) { + frame->type = PJMEDIA_FRAME_TYPE_NONE; + frame->size = 0; + return PJ_SUCCESS; + } + + pj_mutex_lock( stream->jb_mutex ); + + if (stream->dec_frame.size == 0) { + /* Don't have frame in buffer, try to decode one */ + if (decode_frame(stream, frame) != PJ_SUCCESS) { + frame->type = PJMEDIA_FRAME_TYPE_NONE; + frame->size = 0; + } + } else { + if (frame->size < stream->dec_frame.size) { + PJ_LOG(4,(stream->dec->port.info.name.ptr, + "Error: not enough buffer for decoded frame " + "(supplied=%d, required=%d)", + (int)frame->size, (int)stream->dec_frame.size)); + frame->type = PJMEDIA_FRAME_TYPE_NONE; + frame->size = 0; + } else { + frame->type = stream->dec_frame.type; + frame->timestamp = stream->dec_frame.timestamp; + frame->size = stream->dec_frame.size; + pj_memcpy(frame->buf, stream->dec_frame.buf, frame->size); + } + + stream->dec_frame.size = 0; + } + + pj_mutex_unlock( stream->jb_mutex ); + + return PJ_SUCCESS; +} + /* * Create media channel. */ @@ -1080,19 +1149,21 @@ static pj_status_t create_channel( pj_pool_t *pool, channel->pt = pt; /* Allocate buffer for outgoing packet. */ - channel->buf_size = sizeof(pjmedia_rtp_hdr) + stream->frame_size; + if (dir == PJMEDIA_DIR_ENCODING) { + channel->buf_size = sizeof(pjmedia_rtp_hdr) + stream->frame_size; - /* It should big enough to hold (minimally) RTCP SR with an SDES. */ - min_out_pkt_size = sizeof(pjmedia_rtcp_sr_pkt) + - sizeof(pjmedia_rtcp_common) + - (4 + stream->cname.slen) + - 32; + /* It should big enough to hold (minimally) RTCP SR with an SDES. */ + min_out_pkt_size = sizeof(pjmedia_rtcp_sr_pkt) + + sizeof(pjmedia_rtcp_common) + + (4 + stream->cname.slen) + + 32; - if (channel->buf_size < min_out_pkt_size) - channel->buf_size = min_out_pkt_size; + if (channel->buf_size < min_out_pkt_size) + channel->buf_size = min_out_pkt_size; - channel->buf = pj_pool_alloc(pool, channel->buf_size); - PJ_ASSERT_RETURN(channel->buf != NULL, PJ_ENOMEM); + channel->buf = pj_pool_alloc(pool, channel->buf_size); + PJ_ASSERT_RETURN(channel->buf != NULL, PJ_ENOMEM); + } /* Create RTP and RTCP sessions: */ if (info->rtp_seq_ts_set == 0) { @@ -1156,7 +1227,7 @@ PJ_DEF(pj_status_t) pjmedia_vid_stream_create( pjmedia_vid_stream *stream; unsigned jb_init, jb_max, jb_min_pre, jb_max_pre, len; int frm_ptime, chunks_per_frm; - pjmedia_video_format_detail *vfd_enc; + pjmedia_video_format_detail *vfd_enc, *vfd_dec; char *p; unsigned mtu; pj_status_t status; @@ -1214,6 +1285,8 @@ PJ_DEF(pj_status_t) pjmedia_vid_stream_create( vfd_enc = pjmedia_format_get_video_format_detail( &info->codec_param->enc_fmt, PJ_TRUE); + vfd_dec = pjmedia_format_get_video_format_detail( + &info->codec_param->dec_fmt, PJ_TRUE); /* Init stream: */ stream->endpt = endpt; @@ -1297,6 +1370,10 @@ PJ_DEF(pj_status_t) pjmedia_vid_stream_create( if (status != PJ_SUCCESS) return status; + /* Create temporary buffer for immediate decoding */ + stream->dec_max_size = vfd_dec->size.w * vfd_dec->size.h * 4; + stream->dec_frame.buf = pj_pool_alloc(pool, stream->dec_max_size); + /* Init jitter buffer parameters: */ frm_ptime = 1000 * vfd_enc->fps.denum / vfd_enc->fps.num; chunks_per_frm = stream->frame_size / mtu; -- cgit v1.2.3