From 21bee233619f1e2187345efd4eaed85e49facc5b Mon Sep 17 00:00:00 2001 From: Benny Prijono Date: Thu, 29 Sep 2011 08:31:15 +0000 Subject: Closed #1361: codec API change. Details: - changed encode(), packetize(), unpacketize(), and decode() to encode_begin(), encode_more(), and decode() - codec has new "packing" setting - updated stream, aviplay, codec-test, and stream-util due to above - minor doxygen documentation fixes here and there git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@3776 74dad513-b988-da41-8d7b-12977e46ad98 --- pjmedia/include/pjmedia-codec/ffmpeg_codecs.h | 5 + pjmedia/include/pjmedia/vid_codec.h | 453 +++++++++----------------- pjmedia/src/pjmedia-codec/ffmpeg_codecs.c | 248 +++++++++++--- pjmedia/src/pjmedia/endpoint.c | 2 +- pjmedia/src/pjmedia/vid_stream.c | 311 +++++++++--------- pjmedia/src/test/vid_codec_test.c | 158 +++++---- 6 files changed, 611 insertions(+), 566 deletions(-) (limited to 'pjmedia') diff --git a/pjmedia/include/pjmedia-codec/ffmpeg_codecs.h b/pjmedia/include/pjmedia-codec/ffmpeg_codecs.h index 2192b31d..b089cbc1 100644 --- a/pjmedia/include/pjmedia-codec/ffmpeg_codecs.h +++ b/pjmedia/include/pjmedia-codec/ffmpeg_codecs.h @@ -26,6 +26,11 @@ PJ_BEGIN_DECL +/** + * @defgroup PJMEDIA_CODEC_VID_FFMPEG FFmpeg Codecs + * @ingroup PJMEDIA_CODEC_VID_CODECS + * @{ + */ /** * Initialize and register FFMPEG codecs factory to pjmedia endpoint. diff --git a/pjmedia/include/pjmedia/vid_codec.h b/pjmedia/include/pjmedia/vid_codec.h index dcc9de39..fbb169bb 100644 --- a/pjmedia/include/pjmedia/vid_codec.h +++ b/pjmedia/include/pjmedia/vid_codec.h @@ -35,9 +35,47 @@ PJ_BEGIN_DECL +/** + * @defgroup PJMEDIA_VID_CODEC Video Codecs + * @ingroup PJMEDIA_CODEC + * @{ + */ + #define PJMEDIA_VID_CODEC_MAX_DEC_FMT_CNT 8 #define PJMEDIA_VID_CODEC_MAX_FPS_CNT 16 +/** + * This enumeration specifies the packetization property of video encoding + * process. The value is bitmask, and smaller value will have higher priority + * to be used. + */ +typedef enum pjmedia_vid_packing +{ + /** + * This specifies that the packetization is unknown, or if nothing + * is supported. + */ + PJMEDIA_VID_PACKING_UNKNOWN, + + /** + * This specifies that the result of video encoding process will be + * segmented into packets, which is suitable for RTP transmission. + * The maximum size of the packets is set in \a enc_mtu field of + * pjmedia_vid_codec_param. + */ + PJMEDIA_VID_PACKING_PACKETS = 1, + + /** + * This specifies that video encoding function will produce a whole + * or full frame from the source frame. This is normally used for + * encoding video for offline storage such as to an AVI file. The + * maximum size of the packets is set in \a enc_mtu field of + * pjmedia_vid_codec_param. + */ + PJMEDIA_VID_PACKING_WHOLE = 2 + +} pjmedia_vid_packing; + /** * Identification used to search for codec factory that supports specific * codec specification. @@ -55,11 +93,14 @@ typedef struct pjmedia_vid_codec_info pjmedia_format_id dec_fmt_id[PJMEDIA_VID_CODEC_MAX_DEC_FMT_CNT]; /**< Supported encoding source format IDs */ + unsigned packings; /**< Supported or requested packings, + strategies, bitmask from + pjmedia_vid_packing */ unsigned fps_cnt; /**< # of supported frame-rates, can be zero (support any frame-rate) */ pjmedia_ratio fps[PJMEDIA_VID_CODEC_MAX_FPS_CNT]; /**< Supported frame-rates */ - pj_bool_t has_rtp_pack; /**< Support RTP packetization */ + } pjmedia_vid_codec_info; @@ -76,13 +117,15 @@ typedef struct pjmedia_vid_codec_info typedef struct pjmedia_vid_codec_param { pjmedia_dir dir; /**< Direction */ - pjmedia_format enc_fmt; /**< Encoded format */ - pjmedia_format dec_fmt; /**< Decoded format */ + pjmedia_vid_packing packing; /**< Packetization strategy. */ + pjmedia_format enc_fmt; /**< Encoded format */ pjmedia_codec_fmtp enc_fmtp; /**< Encoder fmtp params */ + unsigned enc_mtu; /**< MTU or max payload size setting*/ + + pjmedia_format dec_fmt; /**< Decoded format */ pjmedia_codec_fmtp dec_fmtp; /**< Decoder fmtp params */ - unsigned enc_mtu; /**< MTU or max payload size setting*/ } pjmedia_vid_codec_param; @@ -111,202 +154,63 @@ typedef struct pjmedia_vid_codec pjmedia_vid_codec; typedef struct pjmedia_vid_codec_op { /** - * Initialize codec using the specified attribute. - * - * Application should call #pjmedia_vid_codec_init() instead of - * calling this function directly. - * - * @param codec The codec instance. - * @param pool Pool to use when the codec needs to allocate - * some memory. - * - * @return PJ_SUCCESS on success. + * See #pjmedia_vid_codec_init(). */ pj_status_t (*init)(pjmedia_vid_codec *codec, pj_pool_t *pool ); /** - * Open the codec and initialize with the specified parameter. - * Upon successful initialization, the codec may modify the parameter - * and fills in the unspecified values (such as size or frame rate of - * the encoder format, as it may need to be negotiated with remote - * preferences via SDP fmtp). - * - * Application should call #pjmedia_vid_codec_open() instead of - * calling this function directly. - * - * @param codec The codec instance. - * @param param Codec initialization parameter. - * - * @return PJ_SUCCESS on success. + * See #pjmedia_vid_codec_open(). */ pj_status_t (*open)(pjmedia_vid_codec *codec, pjmedia_vid_codec_param *param ); /** - * Close and shutdown codec, releasing all resources allocated by - * this codec, if any. - * - * Application should call #pjmedia_vid_codec_close() instead of - * calling this function directly. - * - * @param codec The codec instance. - * - * @return PJ_SUCCESS on success. + * See #pjmedia_vid_codec_close(). */ pj_status_t (*close)(pjmedia_vid_codec *codec); /** - * Modify the codec parameter after the codec is open. - * Note that not all codec parameters can be modified during run-time. - * When the parameter cannot be changed, this function will return - * non-PJ_SUCCESS, and the original parameters will not be changed. - * - * Application should call #pjmedia_vid_codec_modify() instead of - * calling this function directly. - * - * @param codec The codec instance. - * @param param The new codec parameter. - * - * @return PJ_SUCCESS on success. + * See #pjmedia_vid_codec_modify(). */ pj_status_t (*modify)(pjmedia_vid_codec *codec, const pjmedia_vid_codec_param *param); /** - * Get the codec parameter after the codec is opened. - * - * Application should call #pjmedia_vid_codec_get_param() instead of - * calling this function directly. - * - * @param codec The codec instance. - * @param param The codec parameter. - * - * @return PJ_SUCCESS on success. + * See #pjmedia_vid_codec_get_param(). */ pj_status_t (*get_param)(pjmedia_vid_codec *codec, pjmedia_vid_codec_param *param); /** - * Instruct the codec to generate a payload/packet from a picture - * bitstream to be sent (via network). The maximum payload size or - * MTU is configurable via enc_mtu field of #pjmedia_vid_codec_param. - * For a long bitstream, application usually need to call this function - * multiple times until the whole bitstream is sent. Note that, for - * performance reason, the packetization will be done in-place, so the - * original bitstream may be modified by this function. - * - * Application should call #pjmedia_vid_codec_packetize() instead of - * calling this function directly. - * - * @param codec The codec instance - * @param bits The picture bitstream. - * @param bits_len The length of the bitstream. - * @param bits_pos On input, the start position of the bitstream - * to be packetized. On output, the next position for - * next packet. - * @param pkt The pointer of the generated payload. - * @param pkt_len The payload length. - * - * @return PJ_SUCCESS on success. + * See #pjmedia_vid_codec_encode_begin(). */ - pj_status_t (*packetize) (pjmedia_vid_codec *codec, - pj_uint8_t *bits, - pj_size_t bits_len, - unsigned *bits_pos, - const pj_uint8_t **pkt, - pj_size_t *pkt_len); + pj_status_t (*encode_begin)(pjmedia_vid_codec *codec, + const pjmedia_frame *input, + unsigned out_size, + pjmedia_frame *output, + pj_bool_t *has_more); /** - * Instruct the codec to parse a payload and append it into a picture - * bitstream. A picture bitstreams may need to be reconstructed from - * one or more payloads. Note that this function will not provide the - * detection of picture boundary, so application should manage the - * picture boundary detection by itself, e.g: for RTP delivery, payloads - * belong to the same picture will share the same RTP timestamp and also - * there is marker bit in the RTP header that is usually reserved for - * end-of-picture flag. Also note that in case of noticing packet lost, - * application should keep calling this function with payload pointer - * set to NULL, as the packetizer need to update its internal state. - * - * Application should call #pjmedia_vid_codec_unpacketize() instead of - * calling this function directly. - * - * @param codec The codec instance - * @param pkt The input packet. - * @param pkt_size Size of the packet. - * @param timestamp The timestamp of the first sample in the packet. - * @param frame_cnt On input, specifies the maximum number of frames - * in the array. On output, the codec must fill - * with number of frames detected in the packet. - * @param frames On output, specifies the frames that have been - * detected in the packet. - * - * @return PJ_SUCCESS on success. + * See #pjmedia_vid_codec_encode_more() */ - pj_status_t (*unpacketize)(pjmedia_vid_codec *codec, - const pj_uint8_t *payload, - pj_size_t payload_len, - pj_uint8_t *bits, - pj_size_t bits_len, - unsigned *bits_pos); + pj_status_t (*encode_more)(pjmedia_vid_codec *codec, + unsigned out_size, + pjmedia_frame *output, + pj_bool_t *has_more); - /** - * Instruct the codec to encode the specified input frame. The input - * MUST contain only one picture with appropriate format as specified - * in opening the codec. - * - * Application should call #pjmedia_vid_codec_encode() instead of - * calling this function directly. - * - * @param codec The codec instance. - * @param input The input frame. - * @param out_size The length of buffer in the output frame. - * @param output The output frame. - * - * @return PJ_SUCCESS on success; - */ - pj_status_t (*encode)(pjmedia_vid_codec *codec, - const pjmedia_frame *input, - unsigned out_size, - pjmedia_frame *output); - /** - * Instruct the codec to decode the specified input frame. The input - * frame MUST contain exactly one picture. Note that the decoded picture - * format may different to the current setting, e.g: the format specified - * in the #pjmedia_vid_codec_param when opening the codec, in this case the - * PJMEDIA_EVENT_FMT_CHANGED event will be emitted by the codec. The codec - * parameter will also be updated, and application can query the format by - * using #get_param(). - * - * Application should call #pjmedia_vid_codec_decode() instead of - * calling this function directly. - * - * @param codec The codec instance. - * @param input The input frame. - * @param out_size The length of buffer in the output frame. - * @param output The output frame. - * - * @return PJ_SUCCESS on success; + /* + * See #pjmedia_vid_codec_decode(). */ - pj_status_t (*decode)(pjmedia_vid_codec *codec, - const pjmedia_frame *input, - unsigned out_size, + pj_status_t (*decode)(pjmedia_vid_codec *codec, + pj_size_t count, + pjmedia_frame packets[], + unsigned out_size, pjmedia_frame *output); /** - * Instruct the codec to recover a missing frame. - * - * Application should call #pjmedia_vid_codec_recover() instead of - * calling this function directly. - * - * @param codec The codec instance. - * @param out_size The length of buffer in the output frame. - * @param output The output frame where generated signal - * will be placed. - * - * @return PJ_SUCCESS on success; + * See #pjmedia_vid_codec_recover() */ pj_status_t (*recover)(pjmedia_vid_codec *codec, unsigned out_size, @@ -553,11 +457,11 @@ pjmedia_vid_codec_mgr_unregister_factory( pjmedia_vid_codec_mgr *mgr, * * @return PJ_SUCCESS on success. */ -PJ_DECL(pj_status_t) pjmedia_vid_codec_mgr_enum_codecs( - pjmedia_vid_codec_mgr *mgr, - unsigned *count, - pjmedia_vid_codec_info info[], - unsigned *prio); +PJ_DECL(pj_status_t) +pjmedia_vid_codec_mgr_enum_codecs(pjmedia_vid_codec_mgr *mgr, + unsigned *count, + pjmedia_vid_codec_info info[], + unsigned *prio); /** @@ -604,9 +508,8 @@ pjmedia_vid_codec_mgr_get_codec_info2(pjmedia_vid_codec_mgr *mgr, * @return The null terminated codec info string, or NULL if * the buffer is not long enough. */ -PJ_DECL(char*) pjmedia_vid_codec_info_to_id( - const pjmedia_vid_codec_info *info, - char *id, unsigned max_len ); +PJ_DECL(char*) pjmedia_vid_codec_info_to_id(const pjmedia_vid_codec_info *info, + char *id, unsigned max_len ); /** @@ -757,9 +660,8 @@ PJ_INLINE(pj_status_t) pjmedia_vid_codec_init( pjmedia_vid_codec *codec, * * @return PJ_SUCCESS on success. */ -PJ_INLINE(pj_status_t) pjmedia_vid_codec_open( - pjmedia_vid_codec *codec, - pjmedia_vid_codec_param *param ) +PJ_INLINE(pj_status_t) pjmedia_vid_codec_open(pjmedia_vid_codec *codec, + pjmedia_vid_codec_param *param) { return (*codec->op->open)(codec, param); } @@ -790,9 +692,9 @@ PJ_INLINE(pj_status_t) pjmedia_vid_codec_close( pjmedia_vid_codec *codec ) * * @return PJ_SUCCESS on success. */ -PJ_INLINE(pj_status_t) pjmedia_vid_codec_modify( - pjmedia_vid_codec *codec, - const pjmedia_vid_codec_param *param) +PJ_INLINE(pj_status_t) +pjmedia_vid_codec_modify(pjmedia_vid_codec *codec, + const pjmedia_vid_codec_param *param) { return (*codec->op->modify)(codec, param); } @@ -806,146 +708,117 @@ PJ_INLINE(pj_status_t) pjmedia_vid_codec_modify( * * @return PJ_SUCCESS on success. */ -PJ_INLINE(pj_status_t) pjmedia_vid_codec_get_param( - pjmedia_vid_codec *codec, - pjmedia_vid_codec_param *param) +PJ_INLINE(pj_status_t) +pjmedia_vid_codec_get_param(pjmedia_vid_codec *codec, + pjmedia_vid_codec_param *param) { return (*codec->op->get_param)(codec, param); } - -/** - * Instruct the codec to generate a payload/packet from a picture - * bitstream to be sent (via network). The maximum payload size or - * MTU is configurable via enc_mtu field of #pjmedia_vid_codec_param. - * For a long bitstream, application usually need to call this function - * multiple times until the whole bitstream is sent. Note that, for - * performance reason, the packetization will be done in-place, so the - * original bitstream may be modified by this function. - * - * @param codec The codec instance - * @param bits The picture bitstream. - * @param bits_len The length of the bitstream. - * @param bits_pos On input, the start position of the bitstream - * to be packetized. On output, the next position for - * next packet. - * @param pkt The pointer of the generated payload. - * @param pkt_len The payload length. +/** + * Encode the specified input frame. The input MUST contain only one picture + * with the appropriate format as specified when opening the codec. Depending + * on the packing or packetization set in the \a packing param, the process + * may produce multiple encoded packets or payloads to represent the picture. + * This is true for example for PJMEDIA_VID_PACKING_PACKETS packing. In this + * case, the \a has_more field will be set to PJ_TRUE, and application should + * call pjmedia_vid_codec_encode_more() to get the remaining results from the + * codec. + * + * @param codec The codec instance. + * @param input The input frame. + * @param out_size The length of buffer in the output frame. This + * should be at least the same as the configured + * encoding MTU of the codec. + * @param output The output frame. + * @param has_more PJ_TRUE if more payloads are available; application + * should then call pjmedia_vid_codec_encode_more() + * to retrieve the remaining results. * - * @return PJ_SUCCESS on success. + * @return PJ_SUCCESS on success; */ -PJ_INLINE(pj_status_t) pjmedia_vid_codec_packetize( - pjmedia_vid_codec *codec, - pj_uint8_t *bits, - pj_size_t bits_len, - unsigned *bits_pos, - const pj_uint8_t **pkt, - pj_size_t *pkt_len ) +PJ_INLINE(pj_status_t) +pjmedia_vid_codec_encode_begin( pjmedia_vid_codec *codec, + const pjmedia_frame *input, + unsigned out_size, + pjmedia_frame *output, + pj_bool_t *has_more) { - return (*codec->op->packetize)(codec, bits, bits_len, bits_pos, - pkt, pkt_len); + return (*codec->op->encode_begin)(codec, input, out_size, output, + has_more); } - /** - * Instruct the codec to parse a payload and append it into a picture - * bitstream. A picture bitstreams may need to be reconstructed from - * one or more payloads. Note that this function will not provide the - * detection of picture boundary, so application should manage the - * picture boundary detection by itself, e.g: for RTP delivery, payloads - * belong to the same picture will share the same RTP timestamp and also - * there is marker bit in the RTP header that is usually reserved for - * end-of-picture flag. Also note that in case of noticing packet lost, - * application should keep calling this function with payload pointer - * set to NULL, as the packetizer need to update its internal state. - * - * @param codec The codec instance - * @param pkt The input packet. - * @param pkt_size Size of the packet. - * @param timestamp The timestamp of the first sample in the packet. - * @param frame_cnt On input, specifies the maximum number of frames - * in the array. On output, the codec must fill - * with number of frames detected in the packet. - * @param frames On output, specifies the frames that have been - * detected in the packet. - * - * @return PJ_SUCCESS on success. - */ -PJ_INLINE(pj_status_t) pjmedia_vid_codec_unpacketize( - pjmedia_vid_codec *codec, - const pj_uint8_t *payload, - pj_size_t payload_len, - pj_uint8_t *bits, - pj_size_t bits_len, - unsigned *bits_pos ) -{ - return (*codec->op->unpacketize)(codec, payload, payload_len, bits, - bits_len, bits_pos); -} - - -/** - * Instruct the codec to encode the specified input frame. The input - * MUST contain only one picture with appropriate format as specified - * in opening the codec. - * - * @param codec The codec instance. - * @param input The input frame. - * @param out_size The length of buffer in the output frame. + * Retrieve more encoded packets/payloads from the codec. Application + * should call this function repeatedly until \a has_more flag is set + * to PJ_FALSE. + * + * @param codec The codec instance. + * @param out_size The length of buffer in the output frame. This + * should be at least the same as as the configured + * encoding MTU of the codec. * @param output The output frame. + * @param has_more PJ_TRUE if more payloads are available, which in + * this case application should call \a encode_more() + * to retrieve them. * * @return PJ_SUCCESS on success; */ -PJ_INLINE(pj_status_t) pjmedia_vid_codec_encode( - pjmedia_vid_codec *codec, - const pjmedia_frame *input, - unsigned out_size, - pjmedia_frame *output) +PJ_INLINE(pj_status_t) +pjmedia_vid_codec_encode_more( pjmedia_vid_codec *codec, + unsigned out_size, + pjmedia_frame *output, + pj_bool_t *has_more) { - return (*codec->op->encode)(codec, input, out_size, output); + return (*codec->op->encode_more)(codec, out_size, output, has_more); } - /** - * Instruct the codec to decode the specified input frame. The input - * frame MUST contain exactly one picture. Note that the decoded picture - * format may different to the current setting, e.g: the format specified - * in the #pjmedia_vid_codec_param when opening the codec, in this case the - * PJMEDIA_EVENT_FMT_CHANGED event will be emitted by the codec. The codec - * parameter will also be updated, and application can query the format by - * using #get_param(). - * - * @param codec The codec instance. - * @param input The input frame. + * Decode the input packets into one picture. If the packing is set to + * PJMEDIA_VID_PACKING_PACKETS when opening the codec, the codec is set + * to decode multiple encoded packets into one picture. These encoded + * packets are typically retrieved from the jitter buffer. If the packing + * is set to PJMEDIA_VID_PACKING_WHOLE, then this decode function can only + * accept one frame as the input. + * + * Note that the decoded picture format may different to the configured + * setting (i.e. the format specified in the #pjmedia_vid_codec_param when + * opening the codec), in this case the PJMEDIA_EVENT_FMT_CHANGED event will + * be emitted by the codec to notify the event. The codec parameter will + * also be updated, and application can query the format by using + * pjmedia_vid_codec_get_param(). + * + * @param codec The codec instance. + * @param pkt_count Number of packets in the input. + * @param packets Array of input packets, each containing an encoded + * frame. * @param out_size The length of buffer in the output frame. * @param output The output frame. * * @return PJ_SUCCESS on success; */ -PJ_INLINE(pj_status_t) pjmedia_vid_codec_decode( - pjmedia_vid_codec *codec, - const pjmedia_frame *input, - unsigned out_size, - pjmedia_frame *output) +PJ_INLINE(pj_status_t) pjmedia_vid_codec_decode(pjmedia_vid_codec *codec, + pj_size_t pkt_count, + pjmedia_frame packets[], + unsigned out_size, + pjmedia_frame *output) { - return (*codec->op->decode)(codec, input, out_size, output); + return (*codec->op->decode)(codec, pkt_count, packets, out_size, output); } - /** - * Instruct the codec to recover a missing frame. + * Recover a missing frame. * - * @param codec The codec instance. + * @param codec The codec instance. * @param out_size The length of buffer in the output frame. * @param output The output frame where generated signal * will be placed. * * @return PJ_SUCCESS on success; */ -PJ_INLINE(pj_status_t) pjmedia_vid_codec_recover( - pjmedia_vid_codec *codec, - unsigned out_size, - pjmedia_frame *output) +PJ_INLINE(pj_status_t) pjmedia_vid_codec_recover(pjmedia_vid_codec *codec, + unsigned out_size, + pjmedia_frame *output) { if (codec->op && codec->op->recover) return (*codec->op->recover)(codec, out_size, output); @@ -960,13 +833,7 @@ PJ_INLINE(pj_status_t) pjmedia_vid_codec_recover( /** * @defgroup PJMEDIA_CODEC_VID_CODECS Supported video codecs - * @ingroup PJMEDIA_CODEC - * @brief Documentation about individual video codec supported by PJMEDIA - * @{ - * Please see the APIs provided by the individual codecs below. - */ -/** - * @} + * @ingroup PJMEDIA_VID_CODEC */ diff --git a/pjmedia/src/pjmedia-codec/ffmpeg_codecs.c b/pjmedia/src/pjmedia-codec/ffmpeg_codecs.c index 16afce33..94610068 100644 --- a/pjmedia/src/pjmedia-codec/ffmpeg_codecs.c +++ b/pjmedia/src/pjmedia-codec/ffmpeg_codecs.c @@ -70,26 +70,20 @@ static pj_status_t ffmpeg_codec_modify(pjmedia_vid_codec *codec, const pjmedia_vid_codec_param *attr ); static pj_status_t ffmpeg_codec_get_param(pjmedia_vid_codec *codec, pjmedia_vid_codec_param *param); -static pj_status_t ffmpeg_packetize ( pjmedia_vid_codec *codec, - pj_uint8_t *buf, - pj_size_t buf_len, - unsigned *pos, - const pj_uint8_t **payload, - pj_size_t *payload_len); -static pj_status_t ffmpeg_unpacketize(pjmedia_vid_codec *codec, - const pj_uint8_t *payload, - pj_size_t payload_len, - pj_uint8_t *buf, - pj_size_t buf_len, - unsigned *pos); -static pj_status_t ffmpeg_codec_encode( pjmedia_vid_codec *codec, - const pjmedia_frame *input, - unsigned output_buf_len, - pjmedia_frame *output); -static pj_status_t ffmpeg_codec_decode( pjmedia_vid_codec *codec, - const pjmedia_frame *input, - unsigned output_buf_len, - pjmedia_frame *output); +static pj_status_t ffmpeg_codec_encode_begin( pjmedia_vid_codec *codec, + const pjmedia_frame *input, + unsigned out_size, + pjmedia_frame *output, + pj_bool_t *has_more); +static pj_status_t ffmpeg_codec_encode_more(pjmedia_vid_codec *codec, + unsigned out_size, + pjmedia_frame *output, + pj_bool_t *has_more); +static pj_status_t ffmpeg_codec_decode( pjmedia_vid_codec *codec, + pj_size_t pkt_count, + pjmedia_frame packets[], + unsigned out_size, + pjmedia_frame *output); /* Definition for FFMPEG codecs operations. */ static pjmedia_vid_codec_op ffmpeg_op = @@ -99,9 +93,8 @@ static pjmedia_vid_codec_op ffmpeg_op = &ffmpeg_codec_close, &ffmpeg_codec_modify, &ffmpeg_codec_get_param, - &ffmpeg_packetize, - &ffmpeg_unpacketize, - &ffmpeg_codec_encode, + &ffmpeg_codec_encode_begin, + &ffmpeg_codec_encode_more, &ffmpeg_codec_decode, NULL }; @@ -145,6 +138,15 @@ typedef struct ffmpeg_private const pjmedia_video_format_info *dec_vfi; pjmedia_video_apply_fmt_param dec_vafp; + /* Buffers, only needed for multi-packets */ + pj_bool_t whole; + void *enc_buf; + unsigned enc_buf_size; + unsigned enc_frame_len; + unsigned enc_processed; + void *dec_buf; + unsigned dec_buf_size; + /* The ffmpeg codec states. */ AVCodec *enc; AVCodec *dec; @@ -235,7 +237,7 @@ static FUNC_UNPACKETIZE(h263_unpacketize); /* Internal codec info */ -ffmpeg_codec_desc codec_desc[] = +static ffmpeg_codec_desc codec_desc[] = { #if ENABLE_H264 { @@ -477,7 +479,8 @@ static const ffmpeg_codec_desc* find_codec_desc_by_info( if (desc->enabled && (desc->info.fmt_id == info->fmt_id) && ((desc->info.dir & info->dir) == info->dir) && - (desc->info.pt == info->pt)) + (desc->info.pt == info->pt) && + (desc->info.packings & info->packings)) { return desc; } @@ -539,8 +542,7 @@ PJ_DEF(pj_status_t) pjmedia_codec_ffmpeg_init(pjmedia_vid_codec_mgr *mgr, av_log_set_level(AV_LOG_ERROR); /* Enum FFMPEG codecs */ - for (c=av_codec_next(NULL); c; c=av_codec_next(c)) - { + for (c=av_codec_next(NULL); c; c=av_codec_next(c)) { ffmpeg_codec_desc *desc; pjmedia_format_id fmt_id; int codec_info_idx; @@ -658,9 +660,11 @@ PJ_DEF(pj_status_t) pjmedia_codec_ffmpeg_init(pjmedia_vid_codec_mgr *mgr, if (desc->info.clock_rate == 0) desc->info.clock_rate = 90000; - /* Set RTP packetization support flag in the codec info */ - desc->info.has_rtp_pack = (desc->packetize != NULL) && - (desc->unpacketize != NULL); + /* Set supported packings */ + desc->info.packings |= PJMEDIA_VID_PACKING_WHOLE; + if (desc->packetize && desc->unpacketize) + desc->info.packings |= PJMEDIA_VID_PACKING_PACKETS; + } /* Review all codecs for applying base format, registering format match for @@ -706,8 +710,11 @@ PJ_DEF(pj_status_t) pjmedia_codec_ffmpeg_init(pjmedia_vid_codec_mgr *mgr, desc->info.dir |= copied_dir; desc->enabled = (desc->info.dir != PJMEDIA_DIR_NONE); - desc->info.has_rtp_pack = (desc->packetize != NULL) && - (desc->unpacketize != NULL); + + /* Set supported packings */ + desc->info.packings |= PJMEDIA_VID_PACKING_WHOLE; + if (desc->packetize && desc->unpacketize) + desc->info.packings |= PJMEDIA_VID_PACKING_PACKETS; if (copied_dir != PJMEDIA_DIR_NONE) { const char *dir_name[] = {NULL, "encoder", "decoder", "codec"}; @@ -801,6 +808,7 @@ static pj_status_t ffmpeg_default_attr( pjmedia_vid_codec_factory *factory, pjmedia_vid_codec_param *attr ) { const ffmpeg_codec_desc *desc; + unsigned i; PJ_ASSERT_RETURN(factory==&ffmpeg_factory.base, PJ_EINVAL); PJ_ASSERT_RETURN(info && attr, PJ_EINVAL); @@ -812,6 +820,20 @@ static pj_status_t ffmpeg_default_attr( pjmedia_vid_codec_factory *factory, pj_bzero(attr, sizeof(pjmedia_vid_codec_param)); + /* Scan the requested packings and use the lowest number */ + attr->packing = 0; + for (i=0; i<15; ++i) { + unsigned packing = (1 << i); + if ((desc->info.packings & info->packings) & packing) { + attr->packing = (pjmedia_vid_packing)packing; + break; + } + } + if (attr->packing == 0) { + /* No supported packing in info */ + return PJMEDIA_CODEC_EUNSUP; + } + /* Direction */ attr->dir = desc->info.dir; @@ -1152,6 +1174,16 @@ static pj_status_t ffmpeg_codec_open( pjmedia_vid_codec *codec, goto on_error; } + /* Alloc buffers if needed */ + ff->whole = (ff->param.packing == PJMEDIA_VID_PACKING_WHOLE); + if (!ff->whole) { + ff->enc_buf_size = ff->enc_vafp.framebytes; + ff->enc_buf = pj_pool_alloc(ff->pool, ff->enc_buf_size); + + ff->dec_buf_size = ff->dec_vafp.framebytes; + ff->dec_buf = pj_pool_alloc(ff->pool, ff->dec_buf_size); + } + /* Update codec attributes, e.g: encoding format may be changed by * SDP fmtp negotiation. */ @@ -1259,10 +1291,10 @@ static pj_status_t ffmpeg_unpacketize(pjmedia_vid_codec *codec, /* * Encode frames. */ -static pj_status_t ffmpeg_codec_encode( pjmedia_vid_codec *codec, - const pjmedia_frame *input, - unsigned output_buf_len, - pjmedia_frame *output) +static pj_status_t ffmpeg_codec_encode_whole(pjmedia_vid_codec *codec, + const pjmedia_frame *input, + unsigned output_buf_len, + pjmedia_frame *output) { ffmpeg_private *ff = (ffmpeg_private*)codec->codec_data; pj_uint8_t *p = (pj_uint8_t*)input->buf; @@ -1311,13 +1343,96 @@ static pj_status_t ffmpeg_codec_encode( pjmedia_vid_codec *codec, return PJ_SUCCESS; } +static pj_status_t ffmpeg_codec_encode_begin( pjmedia_vid_codec *codec, + const pjmedia_frame *input, + unsigned out_size, + pjmedia_frame *output, + pj_bool_t *has_more) +{ + ffmpeg_private *ff = (ffmpeg_private*)codec->codec_data; + pj_status_t status; + + *has_more = PJ_FALSE; + + if (ff->whole) { + status = ffmpeg_codec_encode_whole(codec, input, out_size, output); + } else { + pjmedia_frame whole_frm; + const pj_uint8_t *payload; + pj_size_t payload_len; + + pj_bzero(&whole_frm, sizeof(whole_frm)); + whole_frm.buf = ff->enc_buf; + whole_frm.size = ff->enc_buf_size; + status = ffmpeg_codec_encode_whole(codec, input, + whole_frm.size, &whole_frm); + if (status != PJ_SUCCESS) + return status; + + ff->enc_frame_len = (unsigned)whole_frm.size; + ff->enc_processed = 0; + status = ffmpeg_packetize(codec, (pj_uint8_t*)whole_frm.buf, + whole_frm.size, &ff->enc_processed, + &payload, &payload_len); + if (status != PJ_SUCCESS) + return status; + + if (out_size < payload_len) + return PJMEDIA_CODEC_EFRMTOOSHORT; + + output->type = PJMEDIA_FRAME_TYPE_VIDEO; + pj_memcpy(output->buf, payload, payload_len); + output->size = payload_len; + + *has_more = (ff->enc_processed < ff->enc_frame_len); + } + + return status; +} + +static pj_status_t ffmpeg_codec_encode_more(pjmedia_vid_codec *codec, + unsigned out_size, + pjmedia_frame *output, + pj_bool_t *has_more) +{ + ffmpeg_private *ff = (ffmpeg_private*)codec->codec_data; + const pj_uint8_t *payload; + pj_size_t payload_len; + pj_status_t status; + + *has_more = PJ_FALSE; + + if (ff->enc_processed >= ff->enc_frame_len) { + /* No more frame */ + return PJ_EEOF; + } + + status = ffmpeg_packetize(codec, (pj_uint8_t*)ff->enc_buf, + ff->enc_frame_len, &ff->enc_processed, + &payload, &payload_len); + if (status != PJ_SUCCESS) + return status; + + if (out_size < payload_len) + return PJMEDIA_CODEC_EFRMTOOSHORT; + + output->type = PJMEDIA_FRAME_TYPE_VIDEO; + pj_memcpy(output->buf, payload, payload_len); + output->size = payload_len; + + *has_more = (ff->enc_processed < ff->enc_frame_len); + + return PJ_SUCCESS; +} + + /* * Decode frame. */ -static pj_status_t ffmpeg_codec_decode( pjmedia_vid_codec *codec, - const pjmedia_frame *input, - unsigned output_buf_len, - pjmedia_frame *output) +static pj_status_t ffmpeg_codec_decode_whole(pjmedia_vid_codec *codec, + const pjmedia_frame *input, + unsigned output_buf_len, + pjmedia_frame *output) { ffmpeg_private *ff = (ffmpeg_private*)codec->codec_data; AVFrame avframe; @@ -1416,6 +1531,15 @@ static pj_status_t ffmpeg_codec_decode( pjmedia_vid_codec *codec, if (status != PJ_SUCCESS) return status; + /* Realloc buffer if necessary */ + if (ff->dec_vafp.framebytes > ff->dec_buf_size) { + PJ_LOG(5,(THIS_FILE, "Reallocating decoding buffer %u --> %u", + (unsigned)ff->dec_buf_size, + (unsigned)ff->dec_vafp.framebytes)); + ff->dec_buf_size = ff->dec_vafp.framebytes; + ff->dec_buf = pj_pool_alloc(ff->pool, ff->dec_buf_size); + } + /* Broadcast event */ if (pjmedia_event_publisher_has_sub(&codec->epub)) { pjmedia_event event; @@ -1475,6 +1599,50 @@ static pj_status_t ffmpeg_codec_decode( pjmedia_vid_codec *codec, return PJ_SUCCESS; } +static pj_status_t ffmpeg_codec_decode( pjmedia_vid_codec *codec, + pj_size_t pkt_count, + pjmedia_frame packets[], + unsigned out_size, + pjmedia_frame *output) +{ + ffmpeg_private *ff = (ffmpeg_private*)codec->codec_data; + pj_status_t status; + + PJ_ASSERT_RETURN(codec && pkt_count > 0 && packets && output, + PJ_EINVAL); + + if (ff->whole) { + pj_assert(pkt_count==1); + return ffmpeg_codec_decode_whole(codec, &packets[0], out_size, output); + } else { + pjmedia_frame whole_frm; + unsigned whole_len = 0; + unsigned i; + + for (i=0; i ff->dec_buf_size) { + PJ_LOG(5,(THIS_FILE, "Decoding buffer overflow")); + break; + } + + status = ffmpeg_unpacketize(codec, packets[i].buf, packets[i].size, + ff->dec_buf, ff->dec_buf_size, + &whole_len); + if (status != PJ_SUCCESS) { + PJ_PERROR(5,(THIS_FILE, status, "Unpacketize error")); + continue; + } + } + + whole_frm.buf = ff->dec_buf; + whole_frm.size = whole_len; + whole_frm.timestamp = output->timestamp = packets[i].timestamp; + whole_frm.bit_info = 0; + + return ffmpeg_codec_decode_whole(codec, &whole_frm, out_size, output); + } +} + #ifdef _MSC_VER # pragma comment( lib, "avcodec.lib") diff --git a/pjmedia/src/pjmedia/endpoint.c b/pjmedia/src/pjmedia/endpoint.c index 385850ba..535e0c5c 100644 --- a/pjmedia/src/pjmedia/endpoint.c +++ b/pjmedia/src/pjmedia/endpoint.c @@ -584,7 +584,7 @@ PJ_DEF(pj_status_t) pjmedia_endpt_create_video_sdp(pjmedia_endpt *endpt, } /* Must support RTP packetization and bidirectional */ - if (!codec_info[i].has_rtp_pack || + if ((codec_info[i].packings & PJMEDIA_VID_PACKING_PACKETS) == 0 || codec_info[i].dir != PJMEDIA_DIR_ENCODING_DECODING) { continue; diff --git a/pjmedia/src/pjmedia/vid_stream.c b/pjmedia/src/pjmedia/vid_stream.c index 4a4fa18c..dc28fc9a 100644 --- a/pjmedia/src/pjmedia/vid_stream.c +++ b/pjmedia/src/pjmedia/vid_stream.c @@ -123,6 +123,10 @@ struct pjmedia_vid_stream unsigned frame_size; /**< Size of encoded base frame.*/ unsigned frame_ts_len; /**< Frame length in timestamp. */ + unsigned rx_frame_cnt; /**< # of array in rx_frames */ + pjmedia_frame *rx_frames; /**< Temp. buffer for incoming + frame assembly. */ + #if defined(PJMEDIA_STREAM_ENABLE_KA) && PJMEDIA_STREAM_ENABLE_KA!=0 pj_bool_t use_ka; /**< Stream keep-alive with non- codec-VAD mechanism is @@ -731,7 +735,8 @@ static pj_status_t put_frame(pjmedia_port *port, unsigned rtp_ts_len; void *rtphdr; int rtphdrlen; - unsigned processed = 0; + pj_bool_t has_more_data = PJ_FALSE; + pj_size_t total_sent = 0; #if defined(PJMEDIA_STREAM_ENABLE_KA) && PJMEDIA_STREAM_ENABLE_KA != 0 @@ -766,70 +771,73 @@ static pj_status_t put_frame(pjmedia_port *port, frame_out.size = 0; /* Encode! */ - status = pjmedia_vid_codec_encode(stream->codec, frame, - channel->buf_size - - sizeof(pjmedia_rtp_hdr), - &frame_out); + status = pjmedia_vid_codec_encode_begin(stream->codec, frame, + channel->buf_size - + sizeof(pjmedia_rtp_hdr), + &frame_out, + &has_more_data); if (status != PJ_SUCCESS) { - LOGERR_((channel->port.info.name.ptr, - "Codec encode() error", status)); + LOGERR_((channel->port.info.name.ptr, + "Codec encode_begin() error", status)); /* Update RTP timestamp */ pjmedia_rtp_encode_rtp(&channel->rtp, channel->pt, 1, 0, - rtp_ts_len, (const void**)&rtphdr, &rtphdrlen); - return status; + rtp_ts_len, (const void**)&rtphdr, + &rtphdrlen); + return status; } - - while (processed < frame_out.size) { - pj_uint8_t *payload; - pj_uint8_t *rtp_pkt; - pj_size_t payload_len; - - /* Generate RTP payload */ - status = pjmedia_vid_codec_packetize(stream->codec, - (pj_uint8_t*)frame_out.buf, - frame_out.size, - &processed, - (const pj_uint8_t**)&payload, - &payload_len); - if (status != PJ_SUCCESS) { - LOGERR_((channel->port.info.name.ptr, - "Codec pack() error", status)); - - /* Update RTP timestamp */ - pjmedia_rtp_encode_rtp(&channel->rtp, channel->pt, 1, 0, - rtp_ts_len, (const void**)&rtphdr, - &rtphdrlen); - return status; - } - - /* Encapsulate. */ - status = pjmedia_rtp_encode_rtp( &channel->rtp, - channel->pt, - (processed==frame_out.size?1:0), - payload_len, - rtp_ts_len, - (const void**)&rtphdr, - &rtphdrlen); - - if (status != PJ_SUCCESS) { - LOGERR_((channel->port.info.name.ptr, + /* Loop while we have frame to send */ + for (;;) { + status = pjmedia_rtp_encode_rtp(&channel->rtp, + channel->pt, + (has_more_data == PJ_FALSE ? 1 : 0), + frame_out.size, + rtp_ts_len, + (const void**)&rtphdr, + &rtphdrlen); + if (status != PJ_SUCCESS) { + LOGERR_((channel->port.info.name.ptr, "RTP encode_rtp() error", status)); return status; - } + } - /* Next packets use same timestamp */ - rtp_ts_len = 0; + // Copy RTP header to the beginning of packet + pj_memcpy(channel->buf, rtphdr, sizeof(pjmedia_rtp_hdr)); - rtp_pkt = payload - sizeof(pjmedia_rtp_hdr); + // Send the RTP packet to the transport. + status = pjmedia_transport_send_rtp(stream->transport, + (char*)channel->buf, + frame_out.size + + sizeof(pjmedia_rtp_hdr)); + if (status != PJ_SUCCESS) { + LOGERR_((channel->port.info.name.ptr, + "Transport send_rtp() error", status)); + /* Ignore this error */ + } - /* Copy RTP header to the beginning of packet */ - pj_memcpy(rtp_pkt, rtphdr, sizeof(pjmedia_rtp_hdr)); + total_sent += frame_out.size; - /* Send the RTP packet to the transport. */ - pjmedia_transport_send_rtp(stream->transport, rtp_pkt, - payload_len + sizeof(pjmedia_rtp_hdr)); + if (!has_more_data) + break; + + /* Next packets use same timestamp */ + rtp_ts_len = 0; + + frame_out.size = 0; + + /* Encode more! */ + status = pjmedia_vid_codec_encode_more(stream->codec, + channel->buf_size - + sizeof(pjmedia_rtp_hdr), + &frame_out, + &has_more_data); + if (status != PJ_SUCCESS) { + LOGERR_((channel->port.info.name.ptr, + "Codec encode_more() error", status)); + /* Ignore this error (?) */ + break; + } } /* Check if now is the time to transmit RTCP SR/RR report. @@ -841,12 +849,12 @@ static pj_status_t put_frame(pjmedia_port *port, } /* Do nothing if we have nothing to transmit */ - if (frame_out.size == 0) { + if (total_sent == 0) { return PJ_SUCCESS; } /* Update stat */ - pjmedia_rtcp_tx_rtp(&stream->rtcp, frame_out.size); + pjmedia_rtcp_tx_rtp(&stream->rtcp, total_sent); stream->rtcp.stat.rtp_tx_last_ts = pj_ntohl(stream->enc->rtp.out_hdr.ts); stream->rtcp.stat.rtp_tx_last_seq = pj_ntohs(stream->enc->rtp.out_hdr.seq); @@ -863,9 +871,10 @@ static pj_status_t get_frame(pjmedia_port *port, { pjmedia_vid_stream *stream = (pjmedia_vid_stream*) port->port_data.pdata; pjmedia_vid_channel *channel = stream->dec; - pjmedia_frame frame_in; pj_uint32_t last_ts = 0; int frm_first_seq = 0, frm_last_seq = 0; + pj_bool_t got_frame = PJ_FALSE; + unsigned cnt; pj_status_t status; /* Return no frame is channel is paused */ @@ -875,107 +884,93 @@ static pj_status_t get_frame(pjmedia_port *port, } /* Repeat get payload from the jitter buffer until all payloads with same - * timestamp are collected (a complete frame unpacketized). + * timestamp are collected. */ - { - pj_bool_t got_frame; - unsigned cnt; - - channel->buf_len = 0; - got_frame = PJ_FALSE; + channel->buf_len = 0; - /* Lock jitter buffer mutex first */ - pj_mutex_lock( stream->jb_mutex ); + /* Lock jitter buffer mutex first */ + pj_mutex_lock( stream->jb_mutex ); - /* Check if we got a decodable frame */ - for (cnt=0; ; ++cnt) { - char ptype; - pj_uint32_t ts; - int seq; - - /* Peek frame from jitter buffer. */ - pjmedia_jbuf_peek_frame(stream->jb, cnt, NULL, NULL, - &ptype, NULL, &ts, &seq); - if (ptype == PJMEDIA_JB_NORMAL_FRAME) { - if (last_ts == 0) { - last_ts = ts; - frm_first_seq = seq; - } - if (ts != last_ts) { - got_frame = PJ_TRUE; - break; - } - frm_last_seq = seq; - } else if (ptype == PJMEDIA_JB_ZERO_EMPTY_FRAME) { - /* No more packet in the jitter buffer */ + /* Check if we got a decodable frame */ + for (cnt=0; ; ++cnt) { + char ptype; + pj_uint32_t ts; + int seq; + + /* Peek frame from jitter buffer. */ + pjmedia_jbuf_peek_frame(stream->jb, cnt, NULL, NULL, + &ptype, NULL, &ts, &seq); + if (ptype == PJMEDIA_JB_NORMAL_FRAME) { + if (last_ts == 0) { + last_ts = ts; + frm_first_seq = seq; + } + if (ts != last_ts) { + got_frame = PJ_TRUE; break; } + frm_last_seq = seq; + } else if (ptype == PJMEDIA_JB_ZERO_EMPTY_FRAME) { + /* No more packet in the jitter buffer */ + break; } + } - if (got_frame) { - unsigned i; - - /* Generate frame bitstream from the payload */ - channel->buf_len = 0; - for (i = 0; i < cnt; ++i) { - const pj_uint8_t *p; - pj_size_t psize; - char ptype; - - /* We use jbuf_peek_frame() as it will returns the pointer of - * the payload (no buffer and memcpy needed), just as we need. - */ - pjmedia_jbuf_peek_frame(stream->jb, i, (const void**)&p, - &psize, &ptype, NULL, NULL, NULL); - - if (ptype != PJMEDIA_JB_NORMAL_FRAME) { - /* Packet lost, must set payload to NULL and keep going */ - p = NULL; - psize = 0; - } + if (got_frame) { + unsigned i; - status = pjmedia_vid_codec_unpacketize( - stream->codec, - p, psize, - (pj_uint8_t*)channel->buf, - channel->buf_size, - &channel->buf_len); - if (status != PJ_SUCCESS) { - LOGERR_((channel->port.info.name.ptr, - "Codec unpack() error", status)); - /* Just ignore this unpack error */ - } - } + /* Generate frame bitstream from the payload */ + channel->buf_len = 0; - pjmedia_jbuf_remove_frame(stream->jb, cnt); + if (cnt > stream->rx_frame_cnt) { + PJ_LOG(1,(port->info.name.ptr, + "Discarding %u frames because array is full!", + cnt - stream->rx_frame_cnt)); + pjmedia_jbuf_remove_frame(stream->jb, cnt - stream->rx_frame_cnt); + cnt = stream->rx_frame_cnt; } - /* Unlock jitter buffer mutex. */ - pj_mutex_unlock( stream->jb_mutex ); + for (i = 0; i < cnt; ++i) { + char ptype; - if (!got_frame) { + stream->rx_frames[i].type = PJMEDIA_FRAME_TYPE_VIDEO; + stream->rx_frames[i].timestamp.u64 = last_ts; + stream->rx_frames[i].bit_info = 0; + + /* We use jbuf_peek_frame() as it will returns the pointer of + * the payload (no buffer and memcpy needed), just as we need. + */ + pjmedia_jbuf_peek_frame(stream->jb, i, + (const void**)&stream->rx_frames[i].buf, + &stream->rx_frames[i].size, &ptype, + NULL, NULL, NULL); + + if (ptype != PJMEDIA_JB_NORMAL_FRAME) { + /* Packet lost, must set payload to NULL and keep going */ + stream->rx_frames[i].buf = NULL; + stream->rx_frames[i].size = 0; + stream->rx_frames[i].type = PJMEDIA_FRAME_TYPE_NONE; + continue; + } + } + + /* Decode */ + status = pjmedia_vid_codec_decode(stream->codec, cnt, + stream->rx_frames, + frame->size, frame); + if (status != PJ_SUCCESS) { + LOGERR_((port->info.name.ptr, "codec decode() error", + status)); frame->type = PJMEDIA_FRAME_TYPE_NONE; frame->size = 0; - return PJ_SUCCESS; } - } - - /* Decode */ - frame_in.buf = channel->buf; - frame_in.size = channel->buf_len; - frame_in.bit_info = 0; - frame_in.type = PJMEDIA_FRAME_TYPE_VIDEO; - frame_in.timestamp.u64 = last_ts; - status = pjmedia_vid_codec_decode(stream->codec, &frame_in, - frame->size, frame); - if (status != PJ_SUCCESS) { - LOGERR_((port->info.name.ptr, "codec decode() error", - status)); - frame->type = PJMEDIA_FRAME_TYPE_NONE; - frame->size = 0; + pjmedia_jbuf_remove_frame(stream->jb, cnt); } + /* Unlock jitter buffer mutex. */ + pj_mutex_unlock( stream->jb_mutex ); + /* Learn remote frame rate after successful decoding */ if (0 && frame->type == PJMEDIA_FRAME_TYPE_VIDEO && frame->size) { @@ -1013,7 +1008,7 @@ static pj_status_t get_frame(pjmedia_port *port, dump_port_info(stream->dec, "changed"); pjmedia_event_init(&event, PJMEDIA_EVENT_FMT_CHANGED, - &frame_in.timestamp, &stream->epub); + &frame->timestamp, &stream->epub); event.data.fmt_changed.dir = PJMEDIA_DIR_DECODING; pj_memcpy(&event.data.fmt_changed.new_fmt, &stream->info.codec_param->dec_fmt, @@ -1156,6 +1151,7 @@ PJ_DEF(pj_status_t) pjmedia_vid_stream_create( int frm_ptime, chunks_per_frm; pjmedia_video_format_detail *vfd_enc; char *p; + unsigned mtu; pj_status_t status; if (!pool) { @@ -1187,7 +1183,6 @@ PJ_DEF(pj_status_t) pjmedia_vid_stream_create( if (status != PJ_SUCCESS) return status; - /* Get codec param: */ if (!info->codec_param) { pjmedia_vid_codec_param def_param; @@ -1202,6 +1197,14 @@ PJ_DEF(pj_status_t) pjmedia_vid_stream_create( pj_assert(info->codec_param); } + /* Init codec param and adjust MTU */ + info->codec_param->dir = info->dir; + info->codec_param->enc_mtu -= (sizeof(pjmedia_rtp_hdr) + + PJMEDIA_STREAM_RESV_PAYLOAD_LEN); + if (info->codec_param->enc_mtu > PJMEDIA_MAX_MTU) + info->codec_param->enc_mtu = PJMEDIA_MAX_MTU; + mtu = info->codec_param->enc_mtu; + vfd_enc = pjmedia_format_get_video_format_detail( &info->codec_param->enc_fmt, PJ_TRUE); @@ -1235,11 +1238,6 @@ PJ_DEF(pj_status_t) pjmedia_vid_stream_create( if (status != PJ_SUCCESS) return status; - /* Init codec param */ - info->codec_param->dir = info->dir; - info->codec_param->enc_mtu = PJMEDIA_MAX_MTU - sizeof(pjmedia_rtp_hdr) - - PJMEDIA_STREAM_RESV_PAYLOAD_LEN; - /* Init and open the codec. */ status = pjmedia_vid_codec_init(stream->codec, pool); if (status != PJ_SUCCESS) @@ -1294,7 +1292,8 @@ PJ_DEF(pj_status_t) pjmedia_vid_stream_create( /* Init jitter buffer parameters: */ frm_ptime = 1000 * vfd_enc->fps.denum / vfd_enc->fps.num; - chunks_per_frm = stream->frame_size / PJMEDIA_MAX_MTU; + chunks_per_frm = stream->frame_size / mtu; + if (chunks_per_frm == 0) chunks_per_frm = 1; /* JB max count, default 500ms */ if (info->jb_max >= frm_ptime) @@ -1320,9 +1319,16 @@ PJ_DEF(pj_status_t) pjmedia_vid_stream_create( else jb_init = 0; + /* Allocate array for temporary storage for assembly of incoming + * frames. Add more just in case. + */ + stream->rx_frame_cnt = chunks_per_frm * 2; + stream->rx_frames = pj_pool_calloc(pool, stream->rx_frame_cnt, + sizeof(stream->rx_frames[0])); + /* Create jitter buffer */ status = pjmedia_jbuf_create(pool, &stream->dec->port.info.name, - PJMEDIA_MAX_MTU, + mtu + PJMEDIA_STREAM_RESV_PAYLOAD_LEN, 1000 * vfd_enc->fps.denum / vfd_enc->fps.num, jb_max, &stream->jb); if (status != PJ_SUCCESS) @@ -1460,8 +1466,10 @@ PJ_DEF(pj_status_t) pjmedia_vid_stream_destroy( pjmedia_vid_stream *stream ) } /* Destroy jitter buffer */ - if (stream->jb) + if (stream->jb) { pjmedia_jbuf_destroy(stream->jb); + stream->jb = NULL; + } #if TRACE_JB if (TRACE_JB_OPENED(stream)) { @@ -1707,7 +1715,10 @@ static pj_status_t get_video_codec_info_param(pjmedia_vid_stream_info *si, return PJMEDIA_EMISSINGRTPMAP; } - + + /* Request for codec with the correct packing for streaming */ + si->codec_info.packings = PJMEDIA_VID_PACKING_PACKETS; + /* Now that we have codec info, get the codec param. */ si->codec_param = PJ_POOL_ALLOC_T(pool, pjmedia_vid_codec_param); status = pjmedia_vid_codec_mgr_get_default_param(mgr, diff --git a/pjmedia/src/test/vid_codec_test.c b/pjmedia/src/test/vid_codec_test.c index 4d222564..a6c62469 100644 --- a/pjmedia/src/test/vid_codec_test.c +++ b/pjmedia/src/test/vid_codec_test.c @@ -28,9 +28,6 @@ #define THIS_FILE "vid_codec.c" -#define BYPASS_CODEC 0 -#define BYPASS_PACKETIZER 0 - /* * Capture device setting: * -1 = colorbar, @@ -80,69 +77,62 @@ static pj_status_t codec_on_event(pjmedia_event_subscription *esub, static pj_status_t codec_put_frame(pjmedia_port *port, pjmedia_frame *frame) { + enum { MAX_PACKETS = 50 }; codec_port_data_t *port_data = (codec_port_data_t*)port->port_data.pdata; pj_status_t status; + pjmedia_vid_codec *codec = port_data->codec; + unsigned enc_cnt = 0; + pj_uint8_t *enc_buf; + unsigned enc_size_left; + pjmedia_frame enc_frames[MAX_PACKETS]; + pj_bool_t has_more = PJ_FALSE; + + enc_buf = port_data->enc_buf; + enc_size_left = port_data->enc_buf_size; + + /* + * Encode + */ + enc_frames[enc_cnt].buf = enc_buf; + enc_frames[enc_cnt].size = enc_size_left; + + status = pjmedia_vid_codec_encode_begin(codec, frame, enc_size_left, + &enc_frames[enc_cnt], &has_more); + if (status != PJ_SUCCESS) goto on_error; -#if !BYPASS_CODEC - { - pjmedia_vid_codec *codec = port_data->codec; - pjmedia_frame enc_frame; - - enc_frame.buf = port_data->enc_buf; - enc_frame.size = port_data->enc_buf_size; - - status = pjmedia_vid_codec_encode(codec, frame, enc_frame.size, - &enc_frame); - if (status != PJ_SUCCESS) goto on_error; - -#if !BYPASS_PACKETIZER - if (enc_frame.size) { - unsigned pos = 0; - pj_bool_t packetized = PJ_FALSE; - unsigned unpack_pos = 0; - - while (pos < enc_frame.size) { - pj_uint8_t *payload; - pj_size_t payload_len; - - status = pjmedia_vid_codec_packetize( - codec, - (pj_uint8_t*)enc_frame.buf, - enc_frame.size, &pos, - (const pj_uint8_t**)&payload, - &payload_len); - if (status == PJ_ENOTSUP) - break; - if (status != PJ_SUCCESS) - goto on_error; - - status = pjmedia_vid_codec_unpacketize( - codec, payload, payload_len, - port_data->pack_buf, - port_data->pack_buf_size, - &unpack_pos); - if (status != PJ_SUCCESS) - goto on_error; - - // what happen if the bitstream is broken? - //if (i++ != 1) unpack_pos -= 10; - - packetized = PJ_TRUE; - } + enc_buf += enc_frames[enc_cnt].size; + enc_size_left -= enc_frames[enc_cnt].size; - if (packetized) { - enc_frame.buf = port_data->pack_buf; - enc_frame.size = unpack_pos; - } - } -#endif + ++enc_cnt; + while (has_more) { + enc_frames[enc_cnt].buf = enc_buf; + enc_frames[enc_cnt].size = enc_size_left; + + status = pjmedia_vid_codec_encode_more(codec, enc_size_left, + &enc_frames[enc_cnt], + &has_more); + if (status != PJ_SUCCESS) + break; + + enc_buf += enc_frames[enc_cnt].size; + enc_size_left -= enc_frames[enc_cnt].size; - status = pjmedia_vid_codec_decode(codec, &enc_frame, - frame->size, frame); - if (status != PJ_SUCCESS) goto on_error; + ++enc_cnt; + + if (enc_cnt >= MAX_PACKETS) { + assert(!"Too many packets!"); + break; + } } -#endif + /* + * Decode + */ + status = pjmedia_vid_codec_decode(codec, enc_cnt, enc_frames, + frame->size, frame); + if (status != PJ_SUCCESS) goto on_error; + + /* Display */ status = pjmedia_port_put_frame( pjmedia_vid_port_get_passive_port(port_data->rdr_port), frame); @@ -195,7 +185,8 @@ static int enum_codecs() return PJ_SUCCESS; } -static int encode_decode_test(pj_pool_t *pool, const char *codec_id) +static int encode_decode_test(pj_pool_t *pool, const char *codec_id, + pjmedia_vid_packing packing) { const pj_str_t port_name = {"codec", 5}; @@ -204,16 +195,30 @@ static int encode_decode_test(pj_pool_t *pool, const char *codec_id) codec_port_data_t codec_port_data; pjmedia_vid_codec_param codec_param; const pjmedia_vid_codec_info *codec_info; - + const char *packing_name; pjmedia_vid_dev_index cap_idx, rdr_idx; pjmedia_vid_port *capture=NULL, *renderer=NULL; pjmedia_vid_port_param vport_param; pjmedia_video_format_detail *vfd; pjmedia_event_subscription esub; + char codec_name[5]; pj_status_t status; int rc = 0; - PJ_LOG(3, (THIS_FILE, " encode decode test")); + switch (packing) { + case PJMEDIA_VID_PACKING_PACKETS: + packing_name = "framed"; + break; + case PJMEDIA_VID_PACKING_WHOLE: + packing_name = "whole"; + break; + default: + packing_name = "unknown"; + break; + } + + PJ_LOG(3, (THIS_FILE, " encode decode test: codec=%s, packing=%s", + codec_id, packing_name)); /* Lookup codec */ { @@ -293,7 +298,7 @@ static int encode_decode_test(pj_pool_t *pool, const char *codec_id) rc = 246; goto on_return; } -#if !BYPASS_CODEC + codec_param.packing = packing; /* Open codec */ status = pjmedia_vid_codec_mgr_alloc_codec(NULL, codec_info, @@ -321,7 +326,6 @@ static int encode_decode_test(pj_pool_t *pool, const char *codec_id) pjmedia_event_subscription_init(&esub, &codec_on_event, &codec_port_data); pjmedia_event_subscribe(&codec->epub, &esub); -#endif /* !BYPASS_CODEC */ } pjmedia_vid_port_param_default(&vport_param); @@ -392,27 +396,13 @@ static int encode_decode_test(pj_pool_t *pool, const char *codec_id) rc = 270; goto on_return; } -#if BYPASS_CODEC - PJ_LOG(3, (THIS_FILE, " starting loopback test: %c%c%c%c %dx%d", - ((codec_param.dec_fmt.id & 0x000000FF) >> 0), - ((codec_param.dec_fmt.id & 0x0000FF00) >> 8), - ((codec_param.dec_fmt.id & 0x00FF0000) >> 16), - ((codec_param.dec_fmt.id & 0xFF000000) >> 24), - codec_param.dec_fmt.det.vid.size.w, - codec_param.dec_fmt.det.vid.size.h - )); -#else - PJ_LOG(3, (THIS_FILE, " starting codec test: %c%c%c%c<->%.*s %dx%d", - ((codec_param.dec_fmt.id & 0x000000FF) >> 0), - ((codec_param.dec_fmt.id & 0x0000FF00) >> 8), - ((codec_param.dec_fmt.id & 0x00FF0000) >> 16), - ((codec_param.dec_fmt.id & 0xFF000000) >> 24), + PJ_LOG(3, (THIS_FILE, " starting codec test: %s<->%.*s %dx%d", + pjmedia_fourcc_name(codec_param.dec_fmt.id, codec_name), codec_info->encoding_name.slen, codec_info->encoding_name.ptr, codec_param.dec_fmt.det.vid.size.w, codec_param.dec_fmt.det.vid.size.h )); -#endif /* Start streaming.. */ status = pjmedia_vid_port_start(renderer); @@ -455,7 +445,7 @@ int vid_codec_test(void) int orig_log_level; orig_log_level = pj_log_get_level(); - pj_log_set_level(6); + pj_log_set_level(3); PJ_LOG(3, (THIS_FILE, "Performing video codec tests..")); @@ -475,7 +465,11 @@ int vid_codec_test(void) if (rc != 0) goto on_return; - rc = encode_decode_test(pool, "h263-1998"); + rc = encode_decode_test(pool, "h263-1998", PJMEDIA_VID_PACKING_WHOLE); + if (rc != 0) + goto on_return; + + rc = encode_decode_test(pool, "h263-1998", PJMEDIA_VID_PACKING_PACKETS); if (rc != 0) goto on_return; -- cgit v1.2.3