From dbaa384af07319b4f7bffe6909e3d470a0e0b672 Mon Sep 17 00:00:00 2001 From: Liong Sauw Ming Date: Thu, 30 Jul 2015 06:23:35 +0000 Subject: Fixed #1861: Add support for video capture orientation on Android git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@5138 74dad513-b988-da41-8d7b-12977e46ad98 --- pjlib/include/pj/config_site_sample.h | 3 + pjmedia/src/pjmedia-videodev/android_dev.c | 98 ++++++++++++++++++++-- pjmedia/src/pjmedia-videodev/ios_dev.m | 4 +- pjmedia/src/pjmedia-videodev/util.c | 6 +- .../src/swig/java/android/AndroidManifest.xml | 1 + .../src/org/pjsip/pjsua2/app/CallActivity.java | 52 ++++++++++++ pjsip-apps/src/swig/symbols.i | 2 + pjsip-apps/src/swig/symbols.lst | 2 +- pjsip/include/pjsua2/media.hpp | 33 ++++++++ pjsip/src/pjsua2/media.cpp | 22 +++++ 10 files changed, 210 insertions(+), 13 deletions(-) diff --git a/pjlib/include/pj/config_site_sample.h b/pjlib/include/pj/config_site_sample.h index b879f372..4a56f70a 100644 --- a/pjlib/include/pj/config_site_sample.h +++ b/pjlib/include/pj/config_site_sample.h @@ -391,6 +391,9 @@ /* Fine tune Speex's default settings for best performance/quality */ #define PJMEDIA_CODEC_SPEEX_DEFAULT_QUALITY 5 + + /* Increase number of video device's supported formats */ + #define PJMEDIA_VID_DEV_INFO_FMT_CNT 128 /* * PJSIP settings. diff --git a/pjmedia/src/pjmedia-videodev/android_dev.c b/pjmedia/src/pjmedia-videodev/android_dev.c index 4878b072..dca0d338 100644 --- a/pjmedia/src/pjmedia-videodev/android_dev.c +++ b/pjmedia/src/pjmedia-videodev/android_dev.c @@ -16,6 +16,7 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#include "util.h" #include #include #include @@ -38,6 +39,10 @@ #define DEFAULT_FPS 15 #define ALIGN16(x) ((((x)+15) >> 4) << 4) +/* Define whether we should maintain the aspect ratio when rotating the image. + * For more details, please refer to util.h. + */ +#define MAINTAIN_ASPECT_RATIO PJ_TRUE /* Format map info */ typedef struct and_fmt_map @@ -118,6 +123,10 @@ typedef struct and_stream /** NV21/YV12 -> I420 Conversion buffer */ pj_uint8_t *convert_buf; + pjmedia_rect_size cam_size; + + /** Converter to rotate frame */ + pjmedia_vid_dev_conv conv; /** Frame format param for NV21/YV12 -> I420 conversion */ pjmedia_video_apply_fmt_param @@ -511,7 +520,8 @@ static pj_status_t and_factory_refresh(pjmedia_vid_dev_factory *ff) vdi->id = f->dev_count; vdi->dir = PJMEDIA_DIR_CAPTURE; vdi->has_callback = PJ_TRUE; - vdi->caps = PJMEDIA_VID_DEV_CAP_SWITCH; + vdi->caps = PJMEDIA_VID_DEV_CAP_SWITCH | + PJMEDIA_VID_DEV_CAP_ORIENTATION; /* Set driver & name info */ pj_ansi_strncpy(vdi->driver, "Android", sizeof(vdi->driver)); @@ -578,13 +588,20 @@ static pj_status_t and_factory_refresh(pjmedia_vid_dev_factory *ff) else if (fmt == PJMEDIA_FORMAT_NV21) adi->has_nv21 = PJ_TRUE; for (k = 0; k < adi->sup_size_cnt && - vdi->fmt_cnt < max_fmt_cnt; k++) + vdi->fmt_cnt < max_fmt_cnt-1; k++) { + /* Landscape video */ pjmedia_format_init_video(&vdi->fmt[vdi->fmt_cnt++], fmt, adi->sup_size[k].w, adi->sup_size[k].h, DEFAULT_FPS, 1); + /* Portrait video */ + pjmedia_format_init_video(&vdi->fmt[vdi->fmt_cnt++], + fmt, + adi->sup_size[k].h, + adi->sup_size[k].w, + DEFAULT_FPS, 1); } } (*jni_env)->ReleaseIntArrayElements(jni_env, jiarray, fmts, @@ -598,13 +615,18 @@ static pj_status_t and_factory_refresh(pjmedia_vid_dev_factory *ff) int k; adi->forced_i420 = PJ_TRUE; for (k = 0; k < adi->sup_size_cnt && - vdi->fmt_cnt < max_fmt_cnt; k++) + vdi->fmt_cnt < max_fmt_cnt-1; k++) { pjmedia_format_init_video(&vdi->fmt[vdi->fmt_cnt++], PJMEDIA_FORMAT_I420, adi->sup_size[k].w, adi->sup_size[k].h, DEFAULT_FPS, 1); + pjmedia_format_init_video(&vdi->fmt[vdi->fmt_cnt++], + PJMEDIA_FORMAT_I420, + adi->sup_size[k].h, + adi->sup_size[k].w, + DEFAULT_FPS, 1); } } } else { @@ -636,7 +658,7 @@ static pj_status_t and_factory_refresh(pjmedia_vid_dev_factory *ff) f->dev_count)); for (i = 0; i < f->dev_count; i++) { and_dev_info *adi = &f->dev_info[i]; - char tmp_str[1024], *p; + char tmp_str[2048], *p; int j, plen, slen; PJ_LOG(4, (THIS_FILE, "%2d: %s", i, f->dev_info[i].info.name)); @@ -792,10 +814,12 @@ static pj_status_t and_factory_create_stream( with_attach = jni_get_env(&jni_env); /* Instantiate PjCamera */ + strm->cam_size.w = (vfd->size.w > vfd->size.h? vfd->size.w: vfd->size.h); + strm->cam_size.h = (vfd->size.w > vfd->size.h? vfd->size.h: vfd->size.w); jcam = (*jni_env)->NewObject(jni_env, jobjs.cam.cls, jobjs.cam.m_init, adi->dev_idx, /* idx */ - vfd->size.w, /* w */ - vfd->size.h, /* h */ + strm->cam_size.w, /* w */ + strm->cam_size.h, /* h */ and_fmt, /* fmt */ vfd->fps.num*1000/ vfd->fps.denum, /* fps */ @@ -814,6 +838,19 @@ static pj_status_t and_factory_create_stream( status = PJMEDIA_EVID_SYSERR; goto on_return; } + + /* Video orientation. + * If we send in portrait, we need to set up orientation converter + * as well. + */ + if ((param->flags & PJMEDIA_VID_DEV_CAP_ORIENTATION) || + (vfd->size.h > vfd->size.w)) + { + if (param->orient == PJMEDIA_ORIENT_UNKNOWN) + param->orient = PJMEDIA_ORIENT_NATURAL; + and_stream_set_cap(&strm->base, PJMEDIA_VID_DEV_CAP_ORIENTATION, + ¶m->orient); + } on_return: jni_detach_env(with_attach); @@ -921,6 +958,40 @@ static pj_status_t and_stream_set_cap(pjmedia_vid_dev_stream *s, break; } + case PJMEDIA_VID_DEV_CAP_ORIENTATION: + { + pjmedia_orient orient = *(pjmedia_orient *)pval; + + pj_assert(orient >= PJMEDIA_ORIENT_UNKNOWN && + orient <= PJMEDIA_ORIENT_ROTATE_270DEG); + + if (orient == PJMEDIA_ORIENT_UNKNOWN) + return PJ_EINVAL; + + pj_memcpy(&strm->param.orient, pval, + sizeof(strm->param.orient)); + + if (!strm->conv.conv) { + status = pjmedia_vid_dev_conv_create_converter( + &strm->conv, strm->pool, + &strm->param.fmt, + strm->cam_size, + strm->param.fmt.det.vid.size, + PJ_TRUE, + MAINTAIN_ASPECT_RATIO); + + if (status != PJ_SUCCESS) + return status; + } + + pjmedia_vid_dev_conv_set_rotation(&strm->conv, strm->param.orient); + + PJ_LOG(4, (THIS_FILE, "Video capture orientation set to %d", + strm->param.orient)); + + break; + } + default: status = PJMEDIA_EVID_INVCAP; break; @@ -1005,6 +1076,8 @@ static pj_status_t and_stream_destroy(pjmedia_vid_dev_stream *s) jni_detach_env(with_attach); + pjmedia_vid_dev_conv_destroy_converter(&strm->conv); + if (strm->pool) pj_pool_release(strm->pool); @@ -1020,6 +1093,8 @@ static void JNICALL OnGetFrame(JNIEnv *env, jobject obj, and_stream *strm = *(and_stream**)&user_data; pjmedia_frame f; pj_uint8_t *Y, *U, *V; + pj_status_t status; + void *frame_buf, *data_buf; strm->frame_ts.u64 += strm->ts_inc; if (!strm->vid_cb.capture_cb) @@ -1039,7 +1114,7 @@ static void JNICALL OnGetFrame(JNIEnv *env, jobject obj, f.type = PJMEDIA_FRAME_TYPE_VIDEO; f.size = length; f.timestamp.u64 = strm->frame_ts.u64; - f.buf = (*env)->GetByteArrayElements(env, data, 0); + f.buf = data_buf = (*env)->GetByteArrayElements(env, data, 0); Y = (pj_uint8_t*)f.buf; U = Y + strm->vafp.plane_bytes[0]; @@ -1115,9 +1190,16 @@ static void JNICALL OnGetFrame(JNIEnv *env, jobject obj, } } + + status = pjmedia_vid_dev_conv_resize_and_rotate(&strm->conv, + f.buf, + &frame_buf); + if (status == PJ_SUCCESS) { + f.buf = frame_buf; + } (*strm->vid_cb.capture_cb)(&strm->base, strm->user_data, &f); - (*env)->ReleaseByteArrayElements(env, data, f.buf, JNI_ABORT); + (*env)->ReleaseByteArrayElements(env, data, data_buf, JNI_ABORT); } #endif /* PJMEDIA_VIDEO_DEV_HAS_ANDROID */ diff --git a/pjmedia/src/pjmedia-videodev/ios_dev.m b/pjmedia/src/pjmedia-videodev/ios_dev.m index ac4166c7..4ae06e70 100644 --- a/pjmedia/src/pjmedia-videodev/ios_dev.m +++ b/pjmedia/src/pjmedia-videodev/ios_dev.m @@ -39,9 +39,9 @@ #define DEFAULT_FPS 15 /* Define whether we should maintain the aspect ratio when rotating the image. - * For more details, please refer to vid_util.h. + * For more details, please refer to util.h. */ -#define MAINTAIN_ASPECT_RATIO PJ_TRUE +#define MAINTAIN_ASPECT_RATIO PJ_TRUE typedef struct ios_fmt_info { diff --git a/pjmedia/src/pjmedia-videodev/util.c b/pjmedia/src/pjmedia-videodev/util.c index d66a90a8..b86c22b0 100644 --- a/pjmedia/src/pjmedia-videodev/util.c +++ b/pjmedia/src/pjmedia-videodev/util.c @@ -168,11 +168,13 @@ pjmedia_vid_dev_conv_create_converter(pjmedia_vid_dev_conv *conv, pjmedia_vid_dev_conv_set_rotation(conv, PJMEDIA_ORIENT_NATURAL); - PJ_LOG(4, (THIS_FILE, "Orientation converter created: %dx%d to %dx%d", + PJ_LOG(4, (THIS_FILE, "Orientation converter created: %dx%d to %dx%d, " + "maintain aspect ratio=%s", conv_param.src.det.vid.size.w, conv_param.src.det.vid.size.h, conv_param.dst.det.vid.size.w, - conv_param.dst.det.vid.size.h)); + conv_param.dst.det.vid.size.h, + maintain_aspect_ratio? "yes": "no")); return PJ_SUCCESS; } diff --git a/pjsip-apps/src/swig/java/android/AndroidManifest.xml b/pjsip-apps/src/swig/java/android/AndroidManifest.xml index bb2eb28a..9a7463a0 100644 --- a/pjsip-apps/src/swig/java/android/AndroidManifest.xml +++ b/pjsip-apps/src/swig/java/android/AndroidManifest.xml @@ -40,6 +40,7 @@ diff --git a/pjsip-apps/src/swig/java/android/src/org/pjsip/pjsua2/app/CallActivity.java b/pjsip-apps/src/swig/java/android/src/org/pjsip/pjsua2/app/CallActivity.java index 91123be7..6df2b7e7 100644 --- a/pjsip-apps/src/swig/java/android/src/org/pjsip/pjsua2/app/CallActivity.java +++ b/pjsip-apps/src/swig/java/android/src/org/pjsip/pjsua2/app/CallActivity.java @@ -21,12 +21,17 @@ package org.pjsip.pjsua2.app; import android.os.Bundle; import android.os.Handler; import android.os.Message; +import android.view.Display; +import android.view.Surface; import android.view.SurfaceHolder; import android.view.SurfaceView; import android.view.View; +import android.view.WindowManager; import android.widget.Button; import android.widget.TextView; import android.app.Activity; +import android.content.Context; +import android.content.res.Configuration; import org.pjsip.pjsua2.*; @@ -130,6 +135,49 @@ public class CallActivity extends Activity } } + @Override + public void onConfigurationChanged(Configuration newConfig) { + super.onConfigurationChanged(newConfig); + + WindowManager wm; + Display display; + int rotation; + pjmedia_orient orient; + + wm = (WindowManager)this.getSystemService(Context.WINDOW_SERVICE); + display = wm.getDefaultDisplay(); + rotation = display.getRotation(); + System.out.println("Device orientation changed: " + rotation); + + switch (rotation) { + case Surface.ROTATION_0: // Portrait + orient = pjmedia_orient.PJMEDIA_ORIENT_ROTATE_270DEG; + break; + case Surface.ROTATION_90: // Landscape, home button on the right + orient = pjmedia_orient.PJMEDIA_ORIENT_NATURAL; + break; + case Surface.ROTATION_180: + orient = pjmedia_orient.PJMEDIA_ORIENT_ROTATE_90DEG; + break; + case Surface.ROTATION_270: // Landscape, home button on the left + orient = pjmedia_orient.PJMEDIA_ORIENT_ROTATE_180DEG; + break; + default: + orient = pjmedia_orient.PJMEDIA_ORIENT_UNKNOWN; + } + + if (MyApp.ep != null && MainActivity.account != null) { + try { + AccountConfig cfg = MainActivity.account.cfg; + int cap_dev = cfg.getVideoConfig().getDefaultCaptureDevice(); + MyApp.ep.vidDevManager().setCaptureOrient(cap_dev, orient, + true); + } catch (Exception e) { + System.out.println(e); + } + } + } + @Override protected void onDestroy() { @@ -255,6 +303,10 @@ public class CallActivity extends Activity } else if (m.what == MainActivity.MSG_TYPE.CALL_MEDIA_STATE) { if (MainActivity.currentCall.vidWin != null) { + /* Set capture orientation according to current + * device orientation. + */ + onConfigurationChanged(getResources().getConfiguration()); /* If there's incoming video, display it. */ setupVideoSurface(); } diff --git a/pjsip-apps/src/swig/symbols.i b/pjsip-apps/src/swig/symbols.i index 7f32ff7d..4a59a1a3 100644 --- a/pjsip-apps/src/swig/symbols.i +++ b/pjsip-apps/src/swig/symbols.i @@ -93,6 +93,8 @@ typedef enum pjmedia_dir {PJMEDIA_DIR_NONE = 0, PJMEDIA_DIR_ENCODING = 1, PJMEDI typedef enum pjmedia_tp_proto {PJMEDIA_TP_PROTO_NONE = 0, PJMEDIA_TP_PROTO_RTP_AVP, PJMEDIA_TP_PROTO_RTP_SAVP, PJMEDIA_TP_PROTO_UNKNOWN} pjmedia_tp_proto; +typedef enum pjmedia_orient {PJMEDIA_ORIENT_UNKNOWN, PJMEDIA_ORIENT_NATURAL, PJMEDIA_ORIENT_ROTATE_90DEG, PJMEDIA_ORIENT_ROTATE_180DEG, PJMEDIA_ORIENT_ROTATE_270DEG} pjmedia_orient; + typedef enum pjmedia_format_id {PJMEDIA_FORMAT_L16 = 0, PJMEDIA_FORMAT_PCM = PJMEDIA_FORMAT_L16, PJMEDIA_FORMAT_PCMA = ((('W' << 24) | ('A' << 16)) | ('L' << 8)) | 'A', PJMEDIA_FORMAT_ALAW = PJMEDIA_FORMAT_PCMA, PJMEDIA_FORMAT_PCMU = ((('W' << 24) | ('A' << 16)) | ('L' << 8)) | 'u', PJMEDIA_FORMAT_ULAW = PJMEDIA_FORMAT_PCMU, PJMEDIA_FORMAT_AMR = ((('R' << 24) | ('M' << 16)) | ('A' << 8)) | ' ', PJMEDIA_FORMAT_G729 = ((('9' << 24) | ('2' << 16)) | ('7' << 8)) | 'G', PJMEDIA_FORMAT_ILBC = ((('C' << 24) | ('B' << 16)) | ('L' << 8)) | 'I', PJMEDIA_FORMAT_RGB24 = ((('3' << 24) | ('B' << 16)) | ('G' << 8)) | 'R', PJMEDIA_FORMAT_RGBA = ((('A' << 24) | ('B' << 16)) | ('G' << 8)) | 'R', PJMEDIA_FORMAT_BGRA = ((('A' << 24) | ('R' << 16)) | ('G' << 8)) | 'B', PJMEDIA_FORMAT_RGB32 = PJMEDIA_FORMAT_RGBA, PJMEDIA_FORMAT_DIB = (((' ' << 24) | ('B' << 16)) | ('I' << 8)) | 'D', PJMEDIA_FORMAT_GBRP = ((('P' << 24) | ('R' << 16)) | ('B' << 8)) | 'G', PJMEDIA_FORMAT_AYUV = ((('V' << 24) | ('U' << 16)) | ('Y' << 8)) | 'A', PJMEDIA_FORMAT_YUY2 = ((('2' << 24) | ('Y' << 16)) | ('U' << 8)) | 'Y', PJMEDIA_FORMAT_UYVY = ((('Y' << 24) | ('V' << 16)) | ('Y' << 8)) | 'U', PJMEDIA_FORMAT_YVYU = ((('U' << 24) | ('Y' << 16)) | ('V' << 8)) | 'Y', PJMEDIA_FORMAT_I420 = ((('0' << 24) | ('2' << 16)) | ('4' << 8)) | 'I', PJMEDIA_FORMAT_IYUV = PJMEDIA_FORMAT_I420, PJMEDIA_FORMAT_YV12 = ((('2' << 24) | ('1' << 16)) | ('V' << 8)) | 'Y', PJMEDIA_FORMAT_NV21 = ((('1' << 24) | ('2' << 16)) | ('V' << 8)) | 'N', PJMEDIA_FORMAT_I422 = ((('2' << 24) | ('2' << 16)) | ('4' << 8)) | 'I', PJMEDIA_FORMAT_I420JPEG = ((('0' << 24) | ('2' << 16)) | ('4' << 8)) | 'J', PJMEDIA_FORMAT_I422JPEG = ((('2' << 24) | ('2' << 16)) | ('4' << 8)) | 'J', PJMEDIA_FORMAT_H261 = ((('1' << 24) | ('6' << 16)) | ('2' << 8)) | 'H', PJMEDIA_FORMAT_H263 = ((('3' << 24) | ('6' << 16)) | ('2' << 8)) | 'H', PJMEDIA_FORMAT_H263P = ((('3' << 24) | ('6' << 16)) | ('2' << 8)) | 'P', PJMEDIA_FORMAT_H264 = ((('4' << 24) | ('6' << 16)) | ('2' << 8)) | 'H', PJMEDIA_FORMAT_MJPEG = ((('G' << 24) | ('P' << 16)) | ('J' << 8)) | 'M', PJMEDIA_FORMAT_MPEG1VIDEO = ((('V' << 24) | ('1' << 16)) | ('P' << 8)) | 'M', PJMEDIA_FORMAT_MPEG2VIDEO = ((('V' << 24) | ('2' << 16)) | ('P' << 8)) | 'M', PJMEDIA_FORMAT_MPEG4 = ((('4' << 24) | ('G' << 16)) | ('P' << 8)) | 'M'} pjmedia_format_id; typedef enum pjsip_cred_data_type {PJSIP_CRED_DATA_PLAIN_PASSWD = 0, PJSIP_CRED_DATA_DIGEST = 1, PJSIP_CRED_DATA_EXT_AKA = 16} pjsip_cred_data_type; diff --git a/pjsip-apps/src/swig/symbols.lst b/pjsip-apps/src/swig/symbols.lst index a3e5eae6..ce9f0a36 100644 --- a/pjsip-apps/src/swig/symbols.lst +++ b/pjsip-apps/src/swig/symbols.lst @@ -14,7 +14,7 @@ pjmedia-videodev/videodev.h pjmedia_vid_dev_index pjmedia_vid_dev_std_index pjme pjmedia-audiodev/audiodev.h pjmedia_aud_dev_route pjmedia_aud_dev_cap pjmedia/wav_port.h pjmedia_file_writer_option pjmedia_file_player_option pjmedia/tonegen.h pjmedia_tone_digit pjmedia_tone_digit_map pjmedia_tone_desc -pjmedia/types.h pjmedia_type pjmedia_dir pjmedia_tp_proto +pjmedia/types.h pjmedia_type pjmedia_dir pjmedia_tp_proto pjmedia_orient pjmedia/format.h pjmedia_format_id pjsip/sip_auth.h pjsip_cred_data_type diff --git a/pjsip/include/pjsua2/media.hpp b/pjsip/include/pjsua2/media.hpp index c3bf6e2d..1f45e54e 100644 --- a/pjsip/include/pjsua2/media.hpp +++ b/pjsip/include/pjsua2/media.hpp @@ -1673,6 +1673,39 @@ public: */ const VideoDevInfoVector &enumDev() throw(Error); + /** + * Check whether the video capture device is currently active, i.e. if + * a video preview has been started or there is a video call using + * the device. + * + * @param dev_id The video device id + * + * @return True if it's active. + */ + bool isCaptureActive(int dev_id) const; + + /** + * This will configure video orientation of the video capture device. + * If the device is currently active (i.e. if there is a video call + * using the device or a video preview has been started), the method + * will forward the setting to the video device instance to be applied + * immediately, if it supports it. + * + * The setting will be saved for future opening of the video device, + * if the "keep" argument is set to true. If the video device is + * currently inactive, and the "keep" argument is false, this method + * will throw Error. + * + * @param dev_id The video device id + * @param orient The video orientation. + * @param keep Specify whether the setting is to be kept for + * future use. + * + */ + void setCaptureOrient(pjmedia_vid_dev_index dev_id, + pjmedia_orient orient, + bool keep=true) throw(Error); + private: VideoDevInfoVector videoDevList; diff --git a/pjsip/src/pjsua2/media.cpp b/pjsip/src/pjsua2/media.cpp index 46c60a46..6c8dbf0e 100644 --- a/pjsip/src/pjsua2/media.cpp +++ b/pjsip/src/pjsua2/media.cpp @@ -1306,6 +1306,28 @@ void VidDevManager::clearVideoDevList() #endif } +bool VidDevManager::isCaptureActive(int dev_id) const +{ +#if PJSUA_HAS_VIDEO + return (pjsua_vid_dev_is_active(dev_id) == PJ_TRUE? true: false); +#else + PJ_UNUSED_ARG(dev_id); + + return false; +#endif +} + +void VidDevManager::setCaptureOrient(pjmedia_vid_dev_index dev_id, + pjmedia_orient orient, + bool keep) throw(Error) +{ +#if PJSUA_HAS_VIDEO + PJSUA2_CHECK_EXPR(pjsua_vid_dev_set_setting(dev_id, + PJMEDIA_VID_DEV_CAP_ORIENTATION, &orient, keep)); +#endif +} + + VidDevManager::VidDevManager() { } -- cgit v1.2.3