Closed #1757: iOS OpenGL renderer

* Optimize rendering speed and CPU usage by avoiding buffer copy from frame's buffer
* Add various video dev capabilities



git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@4821 74dad513-b988-da41-8d7b-12977e46ad98

3 files changed, 175 insertions, 90 deletions

diff --git a/pjmedia/include/pjmedia-videodev/opengl_dev.h b/pjmedia/include/pjmedia-videodev/opengl_dev.h
index f5eb67b0..750df8e5 100644
--- a/pjmedia/include/pjmedia-videodev/opengl_dev.h
+++ b/pjmedia/include/pjmedia-videodev/opengl_dev.h
@@ -21,9 +21,13 @@
 
 #include <pjmedia-videodev/videodev_imp.h>
 
-/* OpenGL implementation on each platform needs to implement this and
- * stream operations.
+/* OpenGL implementation on each platform needs to implement these functions
+ * and stream operations.
  */
+/* Get capabilities of the implementation */
+int pjmedia_vid_dev_opengl_imp_get_cap(void);
+
+/* Create OpenGL stream */
 pj_status_t
 pjmedia_vid_dev_opengl_imp_create_stream(pj_pool_t *pool,
                                          pjmedia_vid_dev_param *param,
@@ -31,6 +35,7 @@ pjmedia_vid_dev_opengl_imp_create_stream(pj_pool_t *pool,
                                          void *user_data,
                                          pjmedia_vid_dev_stream **p_vid_strm);
 
+/****************************************************************************/
 /* OpenGL buffers opaque structure. */
 typedef struct gl_buffers gl_buffers;
 
diff --git a/pjmedia/src/pjmedia-videodev/ios_opengl_dev.m b/pjmedia/src/pjmedia-videodev/ios_opengl_dev.m
index bf608e3e..8b751337 100644
--- a/pjmedia/src/pjmedia-videodev/ios_opengl_dev.m
+++ b/pjmedia/src/pjmedia-videodev/ios_opengl_dev.m
@@ -62,12 +62,14 @@ struct iosgl_stream
     pj_status_t             status;
     pj_timestamp            frame_ts;
     unsigned                ts_inc;
+    pjmedia_rect_size       vid_size;
     
     gl_buffers                  *gl_buf;
     GLView                      *gl_view;
     EAGLContext                 *ogl_context;
     CVOpenGLESTextureCacheRef    vid_texture;
     CVImageBufferRef             pb;
+    void                        *pb_addr;
     CVOpenGLESTextureRef         texture;
 };
 
@@ -100,6 +102,15 @@ static pjmedia_vid_dev_stream_op stream_op =
     &iosgl_stream_destroy
 };
 
+int pjmedia_vid_dev_opengl_imp_get_cap(void)
+{
+    return PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW |
+           PJMEDIA_VID_DEV_CAP_OUTPUT_RESIZE |
+           PJMEDIA_VID_DEV_CAP_OUTPUT_POSITION |
+           PJMEDIA_VID_DEV_CAP_OUTPUT_HIDE |
+           PJMEDIA_VID_DEV_CAP_ORIENTATION;
+}
+
 static iosgl_fmt_info* get_iosgl_format_info(pjmedia_format_id id)
 {
     unsigned i;
@@ -174,10 +185,8 @@ pjmedia_vid_dev_opengl_imp_create_stream(pj_pool_t *pool,
     struct iosgl_stream *strm;
     const pjmedia_video_format_detail *vfd;
     pj_status_t status = PJ_SUCCESS;
-    iosgl_fmt_info *ifi;
-    
-    if (!(ifi = get_iosgl_format_info(param->fmt.id)))
-        return PJMEDIA_EVID_BADFORMAT;
+    CGRect rect;
+    CVReturn err;
     
     strm = PJ_POOL_ZALLOC_T(pool, struct iosgl_stream);
     pj_memcpy(&strm->param, param, sizeof(*param));
@@ -188,58 +197,63 @@ pjmedia_vid_dev_opengl_imp_create_stream(pj_pool_t *pool,
     vfd = pjmedia_format_get_video_format_detail(&strm->param.fmt, PJ_TRUE);
     strm->ts_inc = PJMEDIA_SPF2(param->clock_rate, &vfd->fps, 1);
     
-    if (param->dir & PJMEDIA_DIR_RENDER) {
-        CVReturn err;
-        UIWindow *window;
-        CGRect rect;
-        
-	if ((param->flags & PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW) &&
-            param->window.info.ios.window)
-        {
-            /* Get output window handle provided by the application */
-	    window = (UIWindow *)param->window.info.ios.window;
-            rect = window.bounds;
-        } else {
-            rect = CGRectMake(0, 0, strm->param.disp_size.w,
-                              strm->param.disp_size.h);
-        }
-        
-	strm->gl_view = [[GLView alloc] initWithFrame:rect];
-	if (!strm->gl_view)
-	    return PJ_ENOMEM;
-        strm->gl_view->stream = strm;
-
-        /* Perform OpenGL buffer initializations in the main thread. */
-        strm->status = PJ_SUCCESS;
-        [strm->gl_view performSelectorOnMainThread:@selector(init_buffers)
-                       withObject:nil waitUntilDone:YES];
-        if ((status = strm->status) != PJ_SUCCESS) {
-            PJ_LOG(3, (THIS_FILE, "Unable to create and init OpenGL buffers"));
-            goto on_error;
-        }
-        
-        /*  Create a new CVOpenGLESTexture cache */
-        err = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL,
-                                           strm->ogl_context, NULL,
-                                           &strm->vid_texture);
-        if (err) {
-            PJ_LOG(3, (THIS_FILE, "Unable to create OpenGL texture cache %d",
-                       err));
-            status = PJMEDIA_EVID_SYSERR;
-            goto on_error;
-        }
-
-        PJ_LOG(4, (THIS_FILE, "iOS OpenGL ES renderer successfully created"));
+    /* If OUTPUT_RESIZE flag is not used, set display size to default */
+    if (!(param->flags & PJMEDIA_VID_DEV_CAP_OUTPUT_RESIZE)) {
+        pj_bzero(&strm->param.disp_size, sizeof(strm->param.disp_size));
+    }
+    
+    /* Set video format */
+    status = iosgl_stream_set_cap(&strm->base, PJMEDIA_VID_DEV_CAP_FORMAT,
+                                  &param->fmt);
+    if (status != PJ_SUCCESS)
+        goto on_error;
+    
+    rect = CGRectMake(0, 0, strm->param.disp_size.w, strm->param.disp_size.h);
+    strm->gl_view = [[GLView alloc] initWithFrame:rect];
+    if (!strm->gl_view)
+        return PJ_ENOMEM;
+    strm->gl_view->stream = strm;
+    
+    /* Perform OpenGL buffer initializations in the main thread. */
+    strm->status = PJ_SUCCESS;
+    [strm->gl_view performSelectorOnMainThread:@selector(init_buffers)
+                                    withObject:nil waitUntilDone:YES];
+    if ((status = strm->status) != PJ_SUCCESS) {
+        PJ_LOG(3, (THIS_FILE, "Unable to create and init OpenGL buffers"));
+        goto on_error;
+    }
+    
+    /*  Create a new CVOpenGLESTexture cache */
+    err = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL,
+                                       strm->ogl_context, NULL,
+                                       &strm->vid_texture);
+    if (err) {
+        PJ_LOG(3, (THIS_FILE, "Unable to create OpenGL texture cache %d",
+                   err));
+        status = PJMEDIA_EVID_SYSERR;
+        goto on_error;
     }
     
     /* Apply the remaining settings */
-    /*
-     if (param->flags & PJMEDIA_VID_DEV_CAP_INPUT_SCALE) {
-     iosgl_stream_set_cap(&strm->base,
-     PJMEDIA_VID_DEV_CAP_INPUT_SCALE,
-     &param->fmt);
-     }
-     */
+    if (param->flags & PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW) {
+        iosgl_stream_set_cap(&strm->base, PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW,
+                             param->window.info.ios.window);
+    }
+    if (param->flags & PJMEDIA_VID_DEV_CAP_OUTPUT_POSITION) {
+        iosgl_stream_set_cap(&strm->base, PJMEDIA_VID_DEV_CAP_OUTPUT_POSITION,
+                             &param->window_pos);
+    }
+    if (param->flags & PJMEDIA_VID_DEV_CAP_OUTPUT_HIDE) {
+        iosgl_stream_set_cap(&strm->base, PJMEDIA_VID_DEV_CAP_OUTPUT_HIDE,
+                             &param->window_hide);
+    }
+    if (param->flags & PJMEDIA_VID_DEV_CAP_ORIENTATION) {
+        iosgl_stream_set_cap(&strm->base, PJMEDIA_VID_DEV_CAP_ORIENTATION,
+                             &param->orient);
+    }
+    
+    PJ_LOG(4, (THIS_FILE, "iOS OpenGL ES renderer successfully created"));
+                    
     /* Done */
     strm->base.op = &stream_op;
     *p_vid_strm = &strm->base;
@@ -282,9 +296,7 @@ static pj_status_t iosgl_stream_get_cap(pjmedia_vid_dev_stream *s,
     
     PJ_ASSERT_RETURN(s && pval, PJ_EINVAL);
     
-    if (cap==PJMEDIA_VID_DEV_CAP_INPUT_SCALE) {
-        return PJMEDIA_EVID_INVCAP;
-    } else if (cap == PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW) {
+    if (cap == PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW) {
         pjmedia_vid_dev_hwnd *wnd = (pjmedia_vid_dev_hwnd *)pval;
         wnd->info.ios.window = strm->gl_view;
         return PJ_SUCCESS;
@@ -304,8 +316,71 @@ static pj_status_t iosgl_stream_set_cap(pjmedia_vid_dev_stream *s,
     
     PJ_ASSERT_RETURN(s && pval, PJ_EINVAL);
     
-    if (cap==PJMEDIA_VID_DEV_CAP_INPUT_SCALE) {
+    if (cap==PJMEDIA_VID_DEV_CAP_FORMAT) {
+        const pjmedia_video_format_info *vfi;
+        pjmedia_video_format_detail *vfd;
+        pjmedia_format *fmt = (pjmedia_format *)pval;
+        iosgl_fmt_info *ifi;
+        
+        if (!(ifi = get_iosgl_format_info(fmt->id)))
+            return PJMEDIA_EVID_BADFORMAT;
+        
+        vfi = pjmedia_get_video_format_info(pjmedia_video_format_mgr_instance(),
+                                            fmt->id);
+        if (!vfi)
+            return PJMEDIA_EVID_BADFORMAT;
+        
+        pjmedia_format_copy(&strm->param.fmt, fmt);
+        
+        vfd = pjmedia_format_get_video_format_detail(fmt, PJ_TRUE);
+        pj_memcpy(&strm->vid_size, &vfd->size, sizeof(vfd->size));
+        if (strm->param.disp_size.w == 0 || strm->param.disp_size.h == 0)
+            pj_memcpy(&strm->param.disp_size, &vfd->size, sizeof(vfd->size));
+        
+        /* Invalidate the buffer */
+        if (strm->pb) {
+            CVPixelBufferRelease(strm->pb);
+            strm->pb = NULL;
+        }
+        
 	return PJ_SUCCESS;
+    } else if (cap == PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW) {
+        UIView *view = (UIView *)pval;
+        strm->param.window.info.ios.window = (void *)pval;
+        dispatch_async(dispatch_get_main_queue(),
+                       ^{[view addSubview:strm->gl_view];});
+        return PJ_SUCCESS;
+    } else if (cap == PJMEDIA_VID_DEV_CAP_OUTPUT_RESIZE) {
+        pj_memcpy(&strm->param.disp_size, pval, sizeof(strm->param.disp_size));
+        dispatch_async(dispatch_get_main_queue(), ^{
+            strm->gl_view.bounds = CGRectMake(0, 0, strm->param.disp_size.w,
+                                              strm->param.disp_size.h);
+        });
+        return PJ_SUCCESS;
+    } else if (cap == PJMEDIA_VID_DEV_CAP_OUTPUT_POSITION) {
+        pj_memcpy(&strm->param.window_pos, pval, sizeof(strm->param.window_pos));
+        dispatch_async(dispatch_get_main_queue(), ^{
+            strm->gl_view.center = CGPointMake(strm->param.window_pos.x +
+                                               strm->param.disp_size.w/2.0,
+                                               strm->param.window_pos.y +
+                                               strm->param.disp_size.h/2.0);
+        });
+        return PJ_SUCCESS;
+    } else if (cap == PJMEDIA_VID_DEV_CAP_OUTPUT_HIDE) {
+        dispatch_async(dispatch_get_main_queue(), ^{
+            strm->gl_view.hidden = (BOOL)(*((pj_bool_t *)pval));
+        });
+        return PJ_SUCCESS;
+    } else if (cap == PJMEDIA_VID_DEV_CAP_ORIENTATION) {
+        pj_memcpy(&strm->param.orient, pval, sizeof(strm->param.orient));
+        if (strm->param.orient == PJMEDIA_ORIENT_UNKNOWN)
+            return PJ_SUCCESS;
+        dispatch_async(dispatch_get_main_queue(), ^{
+            strm->gl_view.transform =
+                CGAffineTransformMakeRotation(((int)strm->param.orient-1) *
+                                              -M_PI_2);
+        });
+        return PJ_SUCCESS;
     }
     
     return PJMEDIA_EVID_INVCAP;
@@ -330,16 +405,26 @@ static pj_status_t iosgl_stream_put_frame(pjmedia_vid_dev_stream *strm,
     struct iosgl_stream *stream = (struct iosgl_stream*)strm;
     CVReturn err;
 
-    err = CVPixelBufferCreateWithBytes(kCFAllocatorDefault,
-                                       stream->param.disp_size.w,
-                                       stream->param.disp_size.h,
-                                       kCVPixelFormatType_32BGRA,
-                                       frame->buf,
-                                       stream->param.disp_size.w * 4,
-                                       NULL, NULL, NULL, &stream->pb);
-    if (err) {
-        PJ_LOG(3, (THIS_FILE, "Unable to create pixel buffer %d", err));
-        return PJMEDIA_EVID_SYSERR;
+    /* Pixel buffer will only create a wrapper for the frame's buffer,
+     * so if the frame buffer changes, we have to recreate pb
+     */
+    if (!stream->pb || (frame->buf && stream->pb_addr != frame->buf)) {
+        if (stream->pb) {
+            CVPixelBufferRelease(stream->pb);
+            stream->pb = NULL;
+        }
+        err = CVPixelBufferCreateWithBytes(kCFAllocatorDefault,
+                                           stream->vid_size.w,
+                                           stream->vid_size.h,
+                                           kCVPixelFormatType_32BGRA,
+                                           frame->buf,
+                                           stream->vid_size.w * 4,
+                                           NULL, NULL, NULL, &stream->pb);
+        if (err) {
+            PJ_LOG(3, (THIS_FILE, "Unable to create pixel buffer %d", err));
+            return PJMEDIA_EVID_SYSERR;
+        }
+        stream->pb_addr = frame->buf;
     }
 
     /* Create a CVOpenGLESTexture from the CVImageBuffer */
@@ -347,14 +432,13 @@ static pj_status_t iosgl_stream_put_frame(pjmedia_vid_dev_stream *strm,
                                                      stream->vid_texture,
                                                      stream->pb, NULL,
                                                      GL_TEXTURE_2D, GL_RGBA,
-                                                     stream->param.disp_size.w,
-                                                     stream->param.disp_size.h,
+                                                     stream->vid_size.w,
+                                                     stream->vid_size.h,
                                                      GL_BGRA,
                                                      GL_UNSIGNED_BYTE,
                                                      0, &stream->texture);
     if (!stream->texture || err) {
         PJ_LOG(3, (THIS_FILE, "Unable to create OpenGL texture %d", err));
-        CVPixelBufferRelease(stream->pb);
         return PJMEDIA_EVID_SYSERR;
     }
     
@@ -369,7 +453,6 @@ static pj_status_t iosgl_stream_put_frame(pjmedia_vid_dev_stream *strm,
     /* Flush the CVOpenGLESTexture cache and release the texture */
     CVOpenGLESTextureCacheFlush(stream->vid_texture, 0);
     CFRelease(stream->texture);
-    CVPixelBufferRelease(stream->pb);
     
     return PJ_SUCCESS;
 }
@@ -396,6 +479,11 @@ static pj_status_t iosgl_stream_destroy(pjmedia_vid_dev_stream *strm)
     
     iosgl_stream_stop(strm);
     
+    if (stream->pb) {
+        CVPixelBufferRelease(stream->pb);
+        stream->pb = NULL;
+    }
+    
     if (stream->vid_texture) {
         CFRelease(stream->vid_texture);
         stream->vid_texture = NULL;
diff --git a/pjmedia/src/pjmedia-videodev/opengl_dev.c b/pjmedia/src/pjmedia-videodev/opengl_dev.c
index a25090ef..7db7dba4 100644
--- a/pjmedia/src/pjmedia-videodev/opengl_dev.c
+++ b/pjmedia/src/pjmedia-videodev/opengl_dev.c
@@ -336,7 +336,7 @@ static pj_status_t opengl_factory_init(pjmedia_vid_dev_factory *f)
 {
     struct opengl_factory *qf = (struct opengl_factory*)f;
     struct opengl_dev_info *qdi;
-    unsigned i, l;
+    unsigned l;
     
     /* Initialize input and output devices here */
     qf->dev_info = (struct opengl_dev_info*)
@@ -349,25 +349,17 @@ static pj_status_t opengl_factory_init(pjmedia_vid_dev_factory *f)
     strcpy(qdi->info.driver, "OpenGL");
     qdi->info.dir = PJMEDIA_DIR_RENDER;
     qdi->info.has_callback = PJ_FALSE;
-    qdi->info.caps = PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW;
-
-    for (i = 0; i < qf->dev_count; i++) {
-	qdi = &qf->dev_info[i];
-	qdi->info.fmt_cnt = PJ_ARRAY_SIZE(opengl_fmts);
-	qdi->info.caps |= PJMEDIA_VID_DEV_CAP_FORMAT;
+    qdi->info.caps = PJMEDIA_VID_DEV_CAP_FORMAT;
+    qdi->info.fmt_cnt = PJ_ARRAY_SIZE(opengl_fmts);
+    qdi->info.caps |= pjmedia_vid_dev_opengl_imp_get_cap();
 	
-	for (l = 0; l < PJ_ARRAY_SIZE(opengl_fmts); l++) {
-	    pjmedia_format *fmt = &qdi->info.fmt[l];
-	    pjmedia_format_init_video(fmt,
-				      opengl_fmts[l],
-				      DEFAULT_WIDTH,
-				      DEFAULT_HEIGHT,
-				      DEFAULT_FPS, 1);
-	}
+    for (l = 0; l < PJ_ARRAY_SIZE(opengl_fmts); l++) {
+        pjmedia_format *fmt = &qdi->info.fmt[l];
+        pjmedia_format_init_video(fmt, opengl_fmts[l], DEFAULT_WIDTH,
+                                  DEFAULT_HEIGHT, DEFAULT_FPS, 1);
     }
     
-    PJ_LOG(4, (THIS_FILE, "OpenGL initialized with %d devices",
-	       qf->dev_count));
+    PJ_LOG(4, (THIS_FILE, "OpenGL device initialized"));
     
     return PJ_SUCCESS;
 }