From 97f362db7fa1fad9a0258bdb5e218b80dab93031 Mon Sep 17 00:00:00 2001 From: Liong Sauw Ming Date: Thu, 25 Jun 2015 08:17:52 +0000 Subject: Re #1861: Initial implementation of video orientation support - Utility to resize and rotate video frame - Support for iOS + sample - pjsua API to set video device's orientation git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@5118 74dad513-b988-da41-8d7b-12977e46ad98 --- pjmedia/src/pjmedia-videodev/ios_dev.m | 174 ++++++++++++++++++++++++++++----- 1 file changed, 151 insertions(+), 23 deletions(-) (limited to 'pjmedia/src/pjmedia-videodev/ios_dev.m') diff --git a/pjmedia/src/pjmedia-videodev/ios_dev.m b/pjmedia/src/pjmedia-videodev/ios_dev.m index 79ece95a..ac4166c7 100644 --- a/pjmedia/src/pjmedia-videodev/ios_dev.m +++ b/pjmedia/src/pjmedia-videodev/ios_dev.m @@ -16,6 +16,7 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#include "util.h" #include #include #include @@ -37,6 +38,11 @@ #define DEFAULT_HEIGHT 288 #define DEFAULT_FPS 15 +/* Define whether we should maintain the aspect ratio when rotating the image. + * For more details, please refer to vid_util.h. + */ +#define MAINTAIN_ASPECT_RATIO PJ_TRUE + typedef struct ios_fmt_info { pjmedia_format_id pjmedia_format; @@ -108,6 +114,10 @@ struct ios_stream pj_bool_t is_planar; NSLock *frame_lock; void *capture_buf; + void *frame_buf; + + pjmedia_vid_dev_conv conv; + pjmedia_rect_size vid_size; AVCaptureSession *cap_session; AVCaptureDeviceInput *dev_input; @@ -313,12 +323,20 @@ static pj_status_t ios_factory_init(pjmedia_vid_dev_factory *f) if ([dev supportsAVCaptureSessionPreset: ios_sizes[m].preset_str]) { + /* Landscape video */ fmt = &qdi->info.fmt[qdi->info.fmt_cnt++]; pjmedia_format_init_video(fmt, ios_fmts[l].pjmedia_format, ios_sizes[m].supported_size_w, ios_sizes[m].supported_size_h, DEFAULT_FPS, 1); + /* Portrait video */ + fmt = &qdi->info.fmt[qdi->info.fmt_cnt++]; + pjmedia_format_init_video(fmt, + ios_fmts[l].pjmedia_format, + ios_sizes[m].supported_size_h, + ios_sizes[m].supported_size_w, + DEFAULT_FPS, 1); } } } @@ -435,6 +453,8 @@ static pj_status_t ios_factory_default_param(pj_pool_t *pool, fromConnection:(AVCaptureConnection *)connection { CVImageBufferRef img; + pj_status_t status; + void *frame_buf; /* Refrain from calling pjlib functions which require thread registration * here, since according to the doc, dispatch queue cannot guarantee @@ -451,7 +471,6 @@ static pj_status_t ios_factory_default_param(pj_pool_t *pool, /* Lock the base address of the pixel buffer */ CVPixelBufferLockBaseAddress(img, kCVPixelBufferLock_ReadOnly); - [stream->frame_lock lock]; if (stream->is_planar && stream->capture_buf) { @@ -463,10 +482,19 @@ static pj_status_t ios_factory_default_param(pj_pool_t *pool, * air, at 352*288 the image stride is 384. */ pj_size_t stride = CVPixelBufferGetBytesPerRowOfPlane(img, 0); - pj_bool_t need_clip = (stride != stream->size.w); + pj_size_t height = CVPixelBufferGetHeight(img); + pj_bool_t need_clip; + + /* Auto detect rotation */ + if (height != stream->vid_size.h) { + stream->vid_size.w = stream->vid_size.h; + stream->vid_size.h = height; + } + + need_clip = (stride != stream->vid_size.w); p = (pj_uint8_t*)CVPixelBufferGetBaseAddressOfPlane(img, 0); - p_len = stream->size.w * stream->size.h; + p_len = stream->vid_size.w * stream->vid_size.h; Y = (pj_uint8_t*)stream->capture_buf; U = Y + p_len; V = U + p_len/4; @@ -475,9 +503,10 @@ static pj_status_t ios_factory_default_param(pj_pool_t *pool, pj_memcpy(Y, p, p_len); } else { int i = 0; - for (;isize.h;++i) { - pj_memcpy(Y+(i*stream->size.w), p+(i*stride), - stream->size.w); + for (; i < stream->vid_size.h; ++i) { + pj_memcpy(Y, p, stream->vid_size.w); + Y += stream->vid_size.w; + p += stride; } } @@ -492,13 +521,13 @@ static pj_status_t ios_factory_default_param(pj_pool_t *pool, } } else { int i = 0; - for (;i<(stream->size.h)/2;++i) { + for (;i<(stream->vid_size.h)/2;++i) { int y=0; - for (;y<(stream->size.w)/2;++y) { + for (;y<(stream->vid_size.w)/2;++y) { *U++ = *p++; *V++ = *p++; } - p += (stride - stream->size.w); + p += (stride - stream->vid_size.w); } } } @@ -506,6 +535,13 @@ static pj_status_t ios_factory_default_param(pj_pool_t *pool, pj_memcpy(stream->capture_buf, CVPixelBufferGetBaseAddress(img), stream->frame_size); } + + status = pjmedia_vid_dev_conv_resize_and_rotate(&stream->conv, + stream->capture_buf, + &frame_buf); + if (status == PJ_SUCCESS) { + stream->frame_buf = frame_buf; + } stream->frame_ts.u64 += stream->ts_inc; [stream->frame_lock unlock]; @@ -527,7 +563,7 @@ static pj_status_t ios_stream_get_frame(pjmedia_vid_dev_stream *strm, frame->timestamp.u64 = stream->frame_ts.u64; [stream->frame_lock lock]; - pj_memcpy(frame->buf, stream->capture_buf, stream->frame_size); + pj_memcpy(frame->buf, stream->frame_buf, stream->frame_size); [stream->frame_lock unlock]; return PJ_SUCCESS; @@ -645,8 +681,10 @@ static pj_status_t ios_factory_create_stream( AVCaptureDevice *dev = qf->dev_info[param->cap_id].dev; for (i = PJ_ARRAY_SIZE(ios_sizes)-1; i > 0; --i) { - if ((vfd->size.w == ios_sizes[i].supported_size_w) && - (vfd->size.h == ios_sizes[i].supported_size_h)) + if (((vfd->size.w == ios_sizes[i].supported_size_w) && + (vfd->size.h == ios_sizes[i].supported_size_h)) || + ((vfd->size.w == ios_sizes[i].supported_size_h) && + (vfd->size.h == ios_sizes[i].supported_size_w))) { break; } @@ -654,9 +692,18 @@ static pj_status_t ios_factory_create_stream( strm->cap_session.sessionPreset = ios_sizes[i].preset_str; - vfd->size.w = ios_sizes[i].supported_size_w; - vfd->size.h = ios_sizes[i].supported_size_h; + /* If the requested size is portrait (or landscape), we make + * our natural orientation portrait (or landscape) as well. + */ + if (vfd->size.w > vfd->size.h) { + vfd->size.w = ios_sizes[i].supported_size_w; + vfd->size.h = ios_sizes[i].supported_size_h; + } else { + vfd->size.h = ios_sizes[i].supported_size_w; + vfd->size.w = ios_sizes[i].supported_size_h; + } strm->size = vfd->size; + strm->vid_size = vfd->size; strm->bytes_per_row = strm->size.w * vfi->bpp / 8; strm->frame_size = strm->bytes_per_row * strm->size.h; @@ -723,6 +770,7 @@ static pj_status_t ios_factory_create_stream( } strm->capture_buf = pj_pool_alloc(strm->pool, strm->frame_size); + strm->frame_buf = strm->capture_buf; strm->frame_lock = [[NSLock alloc]init]; /* Native preview */ @@ -730,6 +778,19 @@ static pj_status_t ios_factory_create_stream( ios_stream_set_cap(&strm->base, PJMEDIA_VID_DEV_CAP_INPUT_PREVIEW, ¶m->native_preview); } + + /* Video orientation. + * If we send in portrait, we need to set up orientation converter + * as well. + */ + if ((param->flags & PJMEDIA_VID_DEV_CAP_ORIENTATION) || + (vfd->size.h > vfd->size.w)) + { + if (param->orient == PJMEDIA_ORIENT_UNKNOWN) + param->orient = PJMEDIA_ORIENT_NATURAL; + ios_stream_set_cap(&strm->base, PJMEDIA_VID_DEV_CAP_ORIENTATION, + ¶m->orient); + } } else if (param->dir & PJMEDIA_DIR_RENDER) { @@ -987,19 +1048,84 @@ static pj_status_t ios_stream_set_cap(pjmedia_vid_dev_stream *s, return PJ_SUCCESS; } - /* TODO: orientation for capture device */ case PJMEDIA_VID_DEV_CAP_ORIENTATION: { + pjmedia_orient orient = *(pjmedia_orient *)pval; + + pj_assert(orient >= PJMEDIA_ORIENT_UNKNOWN && + orient <= PJMEDIA_ORIENT_ROTATE_270DEG); + + if (orient == PJMEDIA_ORIENT_UNKNOWN) + return PJ_EINVAL; + pj_memcpy(&strm->param.orient, pval, sizeof(strm->param.orient)); - if (strm->param.orient == PJMEDIA_ORIENT_UNKNOWN) - return PJ_SUCCESS; - - dispatch_async(dispatch_get_main_queue(), ^{ - strm->render_view.transform = - CGAffineTransformMakeRotation( - ((int)strm->param.orient-1) * -M_PI_2); - }); + + if (strm->param.dir == PJMEDIA_DIR_RENDER) { + dispatch_async(dispatch_get_main_queue(), ^{ + strm->render_view.transform = + CGAffineTransformMakeRotation( + ((int)strm->param.orient-1) * -M_PI_2); + }); + + return PJ_SUCCESS; + } + + const AVCaptureVideoOrientation cap_ori[4] = + { + AVCaptureVideoOrientationLandscapeLeft, /* NATURAL */ + AVCaptureVideoOrientationPortrait, /* 90DEG */ + AVCaptureVideoOrientationLandscapeRight, /* 180DEG */ + AVCaptureVideoOrientationPortraitUpsideDown, /* 270DEG */ + }; + AVCaptureConnection *vidcon; + pj_bool_t support_ori = PJ_TRUE; + + pj_assert(strm->param.dir == PJMEDIA_DIR_CAPTURE); + + if (!strm->video_output) + return PJMEDIA_EVID_NOTREADY; + + vidcon = [strm->video_output + connectionWithMediaType:AVMediaTypeVideo]; + if ([vidcon isVideoOrientationSupported]) { + vidcon.videoOrientation = cap_ori[strm->param.orient-1]; + } else { + support_ori = PJ_FALSE; + } + + if (!strm->conv.conv) { + pj_status_t status; + pjmedia_rect_size orig_size; + + /* Original native size of device is landscape */ + orig_size.w = (strm->size.w > strm->size.h? strm->size.w : + strm->size.h); + orig_size.h = (strm->size.w > strm->size.h? strm->size.h : + strm->size.w); + + if (!support_ori) { + PJ_LOG(4, (THIS_FILE, "Native video capture orientation " + "unsupported, will use converter's " + "rotation.")); + } + + status = pjmedia_vid_dev_conv_create_converter( + &strm->conv, strm->pool, + &strm->param.fmt, + orig_size, strm->size, + (support_ori?PJ_FALSE:PJ_TRUE), + MAINTAIN_ASPECT_RATIO); + + if (status != PJ_SUCCESS) + return status; + } + + pjmedia_vid_dev_conv_set_rotation(&strm->conv, strm->param.orient); + + PJ_LOG(5, (THIS_FILE, "Video capture orientation set to %d", + strm->param.orient)); + return PJ_SUCCESS; } @@ -1140,6 +1266,8 @@ static pj_status_t ios_stream_destroy(pjmedia_vid_dev_stream *strm) [stream->frame_lock release]; stream->frame_lock = nil; } + + pjmedia_vid_dev_conv_destroy_converter(&stream->conv); pj_pool_release(stream->pool); -- cgit v1.2.3