iomx-dr: split IOMXHWBuffer_Setup function

In HwBuffer, split Setup into Setup, GetMinUndequeued and SetBufferCount since
we want to control the buffer count logic from omxil.c.

Some OMX components (like OMX.TI.*.Decoder) may have nBufferCountActual that is
greater than nBufferCountMin + min_undequeued. In that case we decreased the
number of buffer wanted by the component and had an undefined behavior.

In order to fix it, we need to increase nBufferCountActual value from the
component only when it's smaller than nBufferCountMin + min_undequeued.
Signed-off-by: Martin Storsjö <martin@martin.st>

modules/codec/omxil/iomx_hwbuffer.c

@@ -100,8 +100,7 @@ int IOMXHWBuffer_Disconnect( void *window )
 }
 
 
-int IOMXHWBuffer_Setup( void *window, int w, int h, int hal_format, int hw_usage,
-                        unsigned int *num_frames, unsigned int *min_undequeued )
+int IOMXHWBuffer_Setup( void *window, int w, int h, int hal_format, int hw_usage )
 {
     ANativeWindow *anw = (ANativeWindow *)window;
     int usage = 0;
@@ -134,19 +133,38 @@ int IOMXHWBuffer_Setup( void *window, int w, int h, int hal_format, int hw_usage
     CHECK_ERR();
 #endif
 
+    return 0;
+}
+
+int IOMXHWBuffer_GetMinUndequeued( void *window, unsigned int *min_undequeued )
+{
+    ANativeWindow *anw = (ANativeWindow *)window;
+    status_t err;
+
+    CHECK_ANW();
 #if ANDROID_API >= 11
-    if( *min_undequeued == 0 )
-    {
-        anw->query( anw, NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, min_undequeued );
-        CHECK_ERR();
-    }
+    err = anw->query( anw, NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, min_undequeued );
+    CHECK_ERR();
 #endif
     /* set a minimum value of min_undequeued in case query fails */
     if( *min_undequeued == 0 )
         *min_undequeued = 2;
-    *num_frames += *min_undequeued;
 
-    err = native_window_set_buffer_count( anw, *num_frames );
+    LOGD( "IOMXHWBuffer_GetMinUndequeued: %p %u", anw, *min_undequeued );
+
+    return 0;
+}
+
+int IOMXHWBuffer_SetBufferCount(void *window, unsigned int count )
+{
+    ANativeWindow *anw = (ANativeWindow *)window;
+    status_t err;
+
+    CHECK_ANW();
+
+    LOGD( "IOMXHWBuffer_SetBufferCount: %p %u", anw, count );
+
+    err = native_window_set_buffer_count( anw, count );
     CHECK_ERR();
 
     return 0;

modules/codec/omxil/omxil.c

@@ -2017,7 +2017,8 @@ static void HwBuffer_Init( decoder_t *p_dec, OmxPort *p_port )
 
     if( !(pf_enable_graphic_buffers && pf_get_graphic_buffer_usage &&
           pf_omx_hwbuffer_connect && pf_omx_hwbuffer_disconnect &&
-          pf_omx_hwbuffer_setup && pf_omx_hwbuffer_setcrop &&
+          pf_omx_hwbuffer_setup && pf_omx_hwbuffer_get_min_undequeued &&
+          pf_omx_hwbuffer_set_buffer_count && pf_omx_hwbuffer_setcrop &&
           pf_omx_hwbuffer_dequeue && pf_omx_hwbuffer_lock &&
           pf_omx_hwbuffer_queue && pf_omx_hwbuffer_cancel &&
           ((OMX_COMPONENTTYPE*)p_port->omx_handle)->UseBuffer) )
@@ -2114,7 +2115,6 @@ static int HwBuffer_AllocateBuffers( decoder_t *p_dec, OmxPort *p_port )
     decoder_sys_t *p_sys = p_dec->p_sys;
     OMX_PARAM_PORTDEFINITIONTYPE *def = &p_port->definition;
     unsigned int min_undequeued = 0;
-    unsigned int num_frames = def->nBufferCountMin;
     unsigned int i = 0;
     int colorFormat = def->format.video.eColorFormat;
     OMX_ERRORTYPE omx_error;
@@ -2150,19 +2150,27 @@ static int HwBuffer_AllocateBuffers( decoder_t *p_dec, OmxPort *p_port )
                                def->format.video.nFrameWidth,
                                def->format.video.nFrameHeight,
                                colorFormat,
-                               (int) i_hw_usage,
-                               &num_frames, &min_undequeued) != 0 )
+                               (int) i_hw_usage ) != 0 )
     {
         msg_Err( p_dec, "can't setup OMXHWBuffer" );
         goto error;
     }
 
-    if( num_frames != p_port->definition.nBufferCountActual )
+    if( pf_omx_hwbuffer_get_min_undequeued( p_port->p_hwbuf->window,
+                                            &min_undequeued ) != 0 )
     {
+        msg_Err( p_dec, "can't get min_undequeued" );
+        goto error;
+    }
+
+    if( def->nBufferCountActual < def->nBufferCountMin + min_undequeued )
+    {
+        unsigned int new_frames_num = def->nBufferCountMin + min_undequeued;
+
         OMX_DBG( "AllocateBuffers: video out wants more frames: %lu vs %u",
-                 p_port->definition.nBufferCountActual, num_frames);
+                 p_port->definition.nBufferCountActual, new_frames_num );
 
-        p_port->definition.nBufferCountActual = num_frames;
+        p_port->definition.nBufferCountActual = new_frames_num;
         omx_error = OMX_SetParameter( p_dec->p_sys->omx_handle,
                                       OMX_IndexParamPortDefinition,
                                       &p_port->definition );
@@ -2170,6 +2178,13 @@ static int HwBuffer_AllocateBuffers( decoder_t *p_dec, OmxPort *p_port )
                      omx_error, ErrorToString(omx_error) );
     }
 
+    if( pf_omx_hwbuffer_set_buffer_count( p_port->p_hwbuf->window,
+                                          def->nBufferCountActual ) != 0 )
+    {
+        msg_Err( p_dec, "can't set buffer_count" );
+        goto error;
+    }
+
     jni_SetAndroidSurfaceSize( def->format.video.nFrameWidth,
                                def->format.video.nFrameHeight,
                                def->format.video.nFrameWidth,

modules/codec/omxil/omxil_core.c

@@ -92,8 +92,9 @@ OMX_ERRORTYPE (*pf_get_graphic_buffer_usage)(OMX_HANDLETYPE, OMX_U32, OMX_U32*);
 
 int (*pf_omx_hwbuffer_connect) (void *);
 int (*pf_omx_hwbuffer_disconnect) (void *);
-int (*pf_omx_hwbuffer_setup) (void *, int, int, int, int, unsigned int *,
-                              unsigned int *);
+int (*pf_omx_hwbuffer_setup) (void *, int, int, int, int );
+int (*pf_omx_hwbuffer_get_min_undequeued) (void *, unsigned int *);
+int (*pf_omx_hwbuffer_set_buffer_count) (void *, unsigned int );
 int (*pf_omx_hwbuffer_setcrop) (void *, int, int, int, int);
 int (*pf_omx_hwbuffer_dequeue) (void *, void **);
 int (*pf_omx_hwbuffer_lock) (void *, void *);
@@ -179,6 +180,8 @@ int InitOmxCore(vlc_object_t *p_this)
     pf_omx_hwbuffer_connect = dlsym( dll_handle, "OMXHWBuffer_Connect" );
     pf_omx_hwbuffer_disconnect = dlsym( dll_handle, "OMXHWBuffer_Disconnect" );
     pf_omx_hwbuffer_setup = dlsym( dll_handle, "OMXHWBuffer_Setup" );
+    pf_omx_hwbuffer_get_min_undequeued = dlsym( dll_handle, "OMXHWBuffer_GetMinUndequeued" );
+    pf_omx_hwbuffer_set_buffer_count = dlsym( dll_handle, "OMXHWBuffer_SetBufferCount" );
     pf_omx_hwbuffer_setcrop = dlsym( dll_handle, "OMXHWBuffer_Setcrop" );
     pf_omx_hwbuffer_dequeue = dlsym( dll_handle, "OMXHWBuffer_Dequeue" );
     pf_omx_hwbuffer_lock = dlsym( dll_handle, "OMXHWBuffer_Lock" );

modules/codec/omxil/omxil_core.h

@@ -40,8 +40,9 @@ OMX_ERRORTYPE (*pf_get_graphic_buffer_usage)(OMX_HANDLETYPE, OMX_U32, OMX_U32*);
 /* OMXHWBuffer functions */
 int (*pf_omx_hwbuffer_connect) (void *);
 int (*pf_omx_hwbuffer_disconnect) (void *);
-int (*pf_omx_hwbuffer_setup) (void *, int, int, int, int, unsigned int *,
-                              unsigned int *);
+int (*pf_omx_hwbuffer_setup) (void *, int, int, int, int );
+int (*pf_omx_hwbuffer_get_min_undequeued) (void *, unsigned int *);
+int (*pf_omx_hwbuffer_set_buffer_count) (void *, unsigned int );
 int (*pf_omx_hwbuffer_setcrop) (void *, int, int, int, int);
 int (*pf_omx_hwbuffer_dequeue) (void *, void **);
 int (*pf_omx_hwbuffer_lock) (void *, void *);