avcodec: pass a dummy picture_sys_t to the va decoder Open()

may share some resources between the decoder pool pictures and the va.
Signed-off-by: Rémi Denis-Courmont <remi@remlab.net>

modules/codec/avcodec/dxva2.c

@@ -54,7 +54,7 @@
 #include "../h264_nal.h"
 
 static int Open(vlc_va_t *, AVCodecContext *, enum PixelFormat,
-                const es_format_t *);
+                const es_format_t *, picture_sys_t *p_sys);
 static void Close(vlc_va_t *, AVCodecContext *);
 
 vlc_module_begin()
@@ -575,11 +575,13 @@ static void Close(vlc_va_t *va, AVCodecContext *ctx)
 }
 
 static int Open(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
-                const es_format_t *fmt)
+                const es_format_t *fmt, picture_sys_t *p_sys)
 {
     if (pix_fmt != AV_PIX_FMT_DXVA2_VLD)
         return VLC_EGENERIC;
 
+    (void) p_sys;
+
     vlc_va_sys_t *sys = calloc(1, sizeof (*sys));
     if (unlikely(sys == NULL))
         return VLC_ENOMEM;

modules/codec/avcodec/va.c

@@ -75,10 +75,11 @@ static int vlc_va_Start(void *func, va_list ap)
     AVCodecContext *ctx = va_arg(ap, AVCodecContext *);
     enum PixelFormat pix_fmt = va_arg(ap, enum PixelFormat);
     const es_format_t *fmt = va_arg(ap, const es_format_t *);
+    picture_sys_t *p_sys = va_arg(ap, picture_sys_t *);
     int (*open)(vlc_va_t *, AVCodecContext *, enum PixelFormat,
-                const es_format_t *) = func;
+                const es_format_t *, picture_sys_t *) = func;
 
-    return open(va, ctx, pix_fmt, fmt);
+    return open(va, ctx, pix_fmt, fmt, p_sys);
 }
 
 static void vlc_va_Stop(void *func, va_list ap)
@@ -91,14 +92,15 @@ static void vlc_va_Stop(void *func, va_list ap)
 }
 
 vlc_va_t *vlc_va_New(vlc_object_t *obj, AVCodecContext *avctx,
-                     enum PixelFormat pix_fmt, const es_format_t *fmt)
+                     enum PixelFormat pix_fmt, const es_format_t *fmt,
+                     picture_sys_t *p_sys)
 {
     vlc_va_t *va = vlc_object_create(obj, sizeof (*va));
     if (unlikely(va == NULL))
         return NULL;
 
     va->module = vlc_module_load(va, "hw decoder", "$avcodec-hw", true,
-                                 vlc_va_Start, va, avctx, pix_fmt, fmt);
+                                 vlc_va_Start, va, avctx, pix_fmt, fmt, p_sys);
     if (va->module == NULL)
     {
         vlc_object_release(va);

modules/codec/avcodec/va.h

@@ -58,7 +58,8 @@ vlc_fourcc_t vlc_va_GetChroma(enum PixelFormat hwfmt, enum PixelFormat swfmt);
  * @return a new VLC object on success, NULL on error.
  */
 vlc_va_t *vlc_va_New(vlc_object_t *obj, AVCodecContext *,
-                     enum PixelFormat, const es_format_t *fmt);
+                     enum PixelFormat, const es_format_t *fmt,
+                     picture_sys_t *p_sys);
 
 /**
  * Initializes the acceleration video decoding back-end for libavcodec.

modules/codec/avcodec/vaapi.c

@@ -291,12 +291,13 @@ static int FindFormat(vlc_va_sys_t *sys)
 }
 
 static int Create( vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
-                   const es_format_t *fmt )
+                   const es_format_t *fmt, picture_sys_t *p_sys )
 {
     if( pix_fmt != AV_PIX_FMT_VAAPI_VLD )
         return VLC_EGENERIC;
 
     (void) fmt;
+    (void) p_sys;
 #ifdef VLC_VA_BACKEND_XLIB
     if( !vlc_xlib_init( VLC_OBJECT(va) ) )
     {

modules/codec/avcodec/vda.c

@@ -45,7 +45,7 @@
 #pragma mark prototypes and definitions
 
 static int Open( vlc_va_t *, AVCodecContext *, enum PixelFormat,
-                 const es_format_t * );
+                 const es_format_t *, picture_sys_t * );
 static void Close( vlc_va_t * , AVCodecContext *);
 static int Setup( vlc_va_t *, AVCodecContext *, vlc_fourcc_t *);
 static int Get( vlc_va_t *, picture_t *, uint8_t ** );
@@ -117,11 +117,13 @@ static vlc_va_vda_t *vlc_va_vda_Get( vlc_va_t *va )
 #pragma mark - module handling
 
 static int Open( vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
-                 const es_format_t *fmt )
+                 const es_format_t *fmt, picture_sys_t *p_sys )
 {
     if( pix_fmt != AV_PIX_FMT_VDA_VLD )
         return VLC_EGENERIC;
 
+    (void) p_sys;
+
     msg_Dbg( va, "opening VDA module" );
     if( ctx->codec_id != AV_CODEC_ID_H264 )
     {
@@ -312,7 +314,7 @@ vlc_module_begin ()
 vlc_module_end ()
 
 static int Open( vlc_va_t *va, AVCodecContext *avctx,
-                 enum PixelFormat pix_fmt, const es_format_t *fmt )
+                 enum PixelFormat pix_fmt, const es_format_t *fmt, picture_sys_t *p_sys )
 {
     if( pix_fmt != AV_PIX_FMT_VDA )
         return VLC_EGENERIC;

modules/codec/avcodec/video.c

@@ -1134,8 +1134,13 @@ static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
         if (lavc_UpdateVideoFormat(p_dec, p_context, true))
             continue; /* Unsupported brand of hardware acceleration */
 
+        picture_t *test_pic = decoder_NewPicture(p_dec);
+        assert(!test_pic || test_pic->format.i_chroma == p_dec->fmt_out.video.i_chroma);
         vlc_va_t *va = vlc_va_New(VLC_OBJECT(p_dec), p_context, hwfmt,
-                                  &p_dec->fmt_in);
+                                  &p_dec->fmt_in,
+                                  test_pic ? test_pic->p_sys : NULL);
+        if (test_pic)
+            picture_Release(test_pic);
         if (va == NULL)
             continue; /* Unsupported codec profile or such */
 

modules/hw/vdpau/avcodec.c

@@ -172,11 +172,12 @@ static int Setup(vlc_va_t *va, AVCodecContext *avctx, vlc_fourcc_t *chromap)
 }
 
 static int Open(vlc_va_t *va, AVCodecContext *avctx, enum PixelFormat pix_fmt,
-                const es_format_t *fmt)
+                const es_format_t *fmt, picture_sys_t *p_sys)
 {
     if (pix_fmt != AV_PIX_FMT_VDPAU)
         return VLC_EGENERIC;
 
+    (void) p_sys;
     void *func;
     VdpStatus err;
 #if (LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 2, 0))