avcodec: rationalize hwaccel surface allocation callback

modules/codec/avcodec/dxva2.c

@@ -425,8 +425,9 @@ static int Extract(vlc_va_t *external, picture_t *picture, AVFrame *ff)
     IDirect3DSurface9_UnlockRect(d3d);
     return VLC_SUCCESS;
 }
+
 /* FIXME it is nearly common with VAAPI */
-static int Get(vlc_va_t *external, AVFrame *ff)
+static int Get(vlc_va_t *external, void **opaque, uint8_t **data)
 {
     vlc_va_dxva2_t *va = vlc_va_dxva2_Get(external);
 
@@ -459,16 +460,8 @@ static int Get(vlc_va_t *external, AVFrame *ff)
 
     surface->refcount = 1;
     surface->order = va->surface_order++;
-
-    /* */
-    for (int i = 0; i < 4; i++) {
-        ff->data[i] = NULL;
-        ff->linesize[i] = 0;
-
-        if (i == 0 || i == 3)
-            ff->data[i] = (void*)surface->d3d;/* Yummie */
-    }
-    ff->opaque = surface;
+    *data = (void *)surface->d3d;
+    *opaque = surface;
     return VLC_SUCCESS;
 }
 

modules/codec/avcodec/hwdummy.c

@@ -56,17 +56,11 @@ struct vlc_va_sys_t
     AVVDPAUContext context;
 };
 
-static int Lock(vlc_va_t *va, AVFrame *ff)
+static int Lock(vlc_va_t *va, void **opaque, uint8_t **data)
 {
-    for (unsigned i = 0; i < AV_NUM_DATA_POINTERS; i++)
-    {
-        ff->data[i] = NULL;
-        ff->linesize[i] = 0;
-    }
-
-    ff->data[0] = (void *)(uintptr_t)DATA_MAGIC; /* must be non-NULL */
-    ff->data[3] = (void *)(uintptr_t)DATA_MAGIC;
-    ff->opaque = (void *)(uintptr_t)OPAQUE_MAGIC;
+    *data = (void *)(uintptr_t)DATA_MAGIC;
+    *opaque = (void *)(uintptr_t)OPAQUE_MAGIC;
+    (void) va;
     return VLC_SUCCESS;
 }
 

modules/codec/avcodec/va.h

@@ -37,7 +37,7 @@ struct vlc_va_t {
 
     int  (*setup)(vlc_va_t *, void **hw, vlc_fourcc_t *output,
                   int width, int height);
-    int  (*get)(vlc_va_t *, AVFrame *frame);
+    int  (*get)(vlc_va_t *, void **opaque, uint8_t **data);
     void (*release)(void *opaque, uint8_t *surface);
     int  (*extract)(vlc_va_t *, picture_t *dst, AVFrame *src);
 };
@@ -70,6 +70,9 @@ static inline int vlc_va_Setup(vlc_va_t *va, void **hw, vlc_fourcc_t *output,
  * The surface will be used as output for the hardware decoder, and possibly
  * also as a reference frame to decode other surfaces.
  *
+ * @param opaque pointer to storage space for surface internal data [OUT]
+ * @param data pointer to the AVFrame data[0] and data[3] pointers [OUT]
+ *
  * @note This function needs not be reentrant. However it may be called
  * concurrently with vlc_va_Extract() and/or vlc_va_Release() from other
  * threads and other frames.
@@ -77,9 +80,9 @@ static inline int vlc_va_Setup(vlc_va_t *va, void **hw, vlc_fourcc_t *output,
  * @param frame libavcodec frame [IN/OUT]
  * @return VLC_SUCCESS on success, otherwise an error code.
  */
-static inline int vlc_va_Get(vlc_va_t *va, AVFrame *frame)
+static inline int vlc_va_Get(vlc_va_t *va, void **opaque, uint8_t **data)
 {
-    return va->get(va, frame);
+    return va->get(va, opaque, data);
 }
 
 /**

modules/codec/avcodec/vaapi.c

@@ -477,7 +477,7 @@ static int Extract( vlc_va_t *va, picture_t *p_picture, AVFrame *p_ff )
     return VLC_SUCCESS;
 }
 
-static int Get( vlc_va_t *va, AVFrame *p_ff )
+static int Get( vlc_va_t *va, void **opaque, uint8_t **data )
 {
     vlc_va_sys_t *sys = va->sys;
     int i_old;
@@ -504,17 +504,8 @@ static int Get( vlc_va_t *va, AVFrame *p_ff )
 
     p_surface->i_refcount = 1;
     p_surface->i_order = sys->i_surface_order++;
-
-    /* */
-    for( int i = 0; i < 4; i++ )
-    {
-        p_ff->data[i] = NULL;
-        p_ff->linesize[i] = 0;
-
-        if( i == 0 || i == 3 )
-            p_ff->data[i] = (void*)(uintptr_t)p_surface->i_id;/* Yummie */
-    }
-    p_ff->opaque = p_surface;
+    *data = (void *)(uintptr_t)p_surface->i_id;
+    *opaque = p_surface;
     return VLC_SUCCESS;
 }
 

modules/codec/avcodec/vda.c

@@ -203,20 +203,12 @@ ok:
     return VLC_SUCCESS;
 }
 
-static int Get( vlc_va_t *external, AVFrame *p_ff )
+static int Get( vlc_va_t *external, void **opaque, uint8_t **data )
 {
     VLC_UNUSED( external );
 
-    /* */
-    for( int i = 0; i < 4; i++ )
-    {
-        p_ff->data[i] = NULL;
-        p_ff->linesize[i] = 0;
-
-        if( i == 0 || i == 3 )
-        p_ff->data[i] = (uint8_t *)1; // dummy
-    }
-
+    *data = (uint8_t *)1; // dummy
+    (void) opaque;
     return VLC_SUCCESS;
 }
 
@@ -257,14 +249,13 @@ static int Extract( vlc_va_t *external, picture_t *p_picture, AVFrame *p_ff )
 
 static void Release( void *opaque, uint8_t *data )
 {
-    assert( opaque == NULL );
 #if 0
     CVPixelBufferRef cv_buffer = ( CVPixelBufferRef )p_ff->data[3];
 
     if ( cv_buffer )
         CVPixelBufferRelease( cv_buffer );
 #endif
-    (void) data;
+    (void) opaque; (void) data;
 }
 
 static void Close( vlc_va_t *external )

modules/codec/avcodec/video.c

@@ -907,11 +907,14 @@ static int lavc_va_GetFrame(struct AVCodecContext *ctx, AVFrame *frame,
         msg_Err(dec, "hardware acceleration setup failed");
         return -1;
     }
-    if (vlc_va_Get(va, frame))
+    if (vlc_va_Get(va, &frame->opaque, &frame->data[0]))
     {
         msg_Err(dec, "hardware acceleration picture allocation failed");
         return -1;
     }
+    /* data[0] must be non-NULL for libavcodec internal checks.
+     * data[3] actually contains the format-specific surface handle. */
+    frame->data[3] = frame->data[0];
 
     frame->buf[0] = av_buffer_create(frame->data[0], 0, va->release,
                                      frame->opaque, 0);
@@ -921,7 +924,6 @@ static int lavc_va_GetFrame(struct AVCodecContext *ctx, AVFrame *frame,
         return -1;
     }
     assert(frame->data[0] != NULL);
-    assert(frame->data[3] != NULL);
     (void) flags;
     return 0;
 }
@@ -1097,12 +1099,13 @@ static int ffmpeg_va_GetFrameBuf( struct AVCodecContext *p_context, AVFrame *p_f
         return -1;
     }
 
-    if( vlc_va_Get( p_va, p_ff_pic ) )
+    if( vlc_va_Get( p_va, &p_ff_pic->opaque, &p_ff_pic->data[0] ) )
     {
         msg_Err( p_dec, "vlc_va_Get failed" );
         return -1;
     }
 
+    p_ff_pic->data[3] = p_ff_pic->data[0];
     p_ff_pic->type = FF_BUFFER_TYPE_USER;
     return 0;
 }

modules/hw/vdpau/avcodec.c

@@ -60,18 +60,12 @@ struct vlc_va_sys_t
     uint16_t height;
 };
 
-static int Lock(vlc_va_t *va, AVFrame *ff)
+static int Lock(vlc_va_t *va, void **opaque, uint8_t **data)
 {
     vlc_va_sys_t *sys = va->sys;
     VdpVideoSurface surface;
     VdpStatus err;
 
-    for (unsigned i = 0; i < AV_NUM_DATA_POINTERS; i++)
-    {
-        ff->data[i] = NULL;
-        ff->linesize[i] = 0;
-    }
-
     err = vdp_video_surface_create(sys->vdp, sys->device, VDP_CHROMA_TYPE_420,
                                    sys->width, sys->height, &surface);
     if (err != VDP_STATUS_OK)
@@ -85,9 +79,8 @@ static int Lock(vlc_va_t *va, AVFrame *ff)
     if (unlikely(field == NULL))
         return VLC_ENOMEM;
 
-    ff->data[0] = (void *)sys->vdp; /* must be non-NULL */
-    ff->data[3] = (void *)(uintptr_t)surface;
-    ff->opaque = field;
+    *data = (void *)(uintptr_t)surface;
+    *opaque = field;
     return VLC_SUCCESS;
 }