avcodec: pass PixelFormat as parameter to VA activation callback

This makes checking for video output support before activation a lot
simpler (especially avoiding callbacks).

This effectively reverts commit b667100b.

modules/codec/avcodec/dxva2.c

@@ -52,7 +52,8 @@
 #include "va.h"
 #include "../../video_chroma/copy.h"
 
-static int Open(vlc_va_t *, AVCodecContext *, const es_format_t *);
+static int Open(vlc_va_t *, AVCodecContext *, enum PixelFormat,
+                const es_format_t *);
 static void Close(vlc_va_t *, AVCodecContext *);
 
 vlc_module_begin()
@@ -544,8 +545,12 @@ static void Close(vlc_va_t *va, AVCodecContext *ctx)
     free(sys);
 }
 
-static int Open(vlc_va_t *va, AVCodecContext *ctx, const es_format_t *fmt)
+static int Open(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
+                const es_format_t *fmt)
 {
+    if (pix_fmt != AV_PIX_FMT_DXVA2_VLD)
+        return VLC_EGENERIC;
+
     vlc_va_sys_t *sys = calloc(1, sizeof (*sys));
     if (unlikely(sys == NULL))
         return VLC_ENOMEM;
@@ -594,7 +599,6 @@ static int Open(vlc_va_t *va, AVCodecContext *ctx, const es_format_t *fmt)
 
     /* TODO print the hardware name/vendor for debugging purposes */
     va->description = DxDescribe(sys);
-    va->pix_fmt = PIX_FMT_DXVA2_VLD;
     va->setup   = Setup;
     va->get     = Get;
     va->release = Release;

modules/codec/avcodec/va.c

@@ -32,10 +32,12 @@ static int vlc_va_Start(void *func, va_list ap)
 {
     vlc_va_t *va = va_arg(ap, vlc_va_t *);
     AVCodecContext *ctx = va_arg(ap, AVCodecContext *);
+    enum PixelFormat pix_fmt = va_arg(ap, enum PixelFormat);
     const es_format_t *fmt = va_arg(ap, const es_format_t *);
-    int (*open)(vlc_va_t *, AVCodecContext *, const es_format_t *) = func;
+    int (*open)(vlc_va_t *, AVCodecContext *, enum PixelFormat,
+                const es_format_t *) = func;
 
-    return open(va, ctx, fmt);
+    return open(va, ctx, pix_fmt, fmt);
 }
 
 static void vlc_va_Stop(void *func, va_list ap)
@@ -48,14 +50,14 @@ static void vlc_va_Stop(void *func, va_list ap)
 }
 
 vlc_va_t *vlc_va_New(vlc_object_t *obj, AVCodecContext *avctx,
-                     const es_format_t *fmt)
+                     enum PixelFormat pix_fmt, const es_format_t *fmt)
 {
     vlc_va_t *va = vlc_object_create(obj, sizeof (*va));
     if (unlikely(va == NULL))
         return NULL;
 
     va->module = vlc_module_load(va, "hw decoder", "$avcodec-hw", true,
-                                 vlc_va_Start, va, avctx, fmt);
+                                 vlc_va_Start, va, avctx, pix_fmt, fmt);
     if (va->module == NULL)
     {
         vlc_object_release(va);

modules/codec/avcodec/va.h

@@ -35,7 +35,6 @@ struct vlc_va_t {
     vlc_va_sys_t *sys;
     module_t *module;
     const char *description;
-    int pix_fmt;
 
     int  (*setup)(vlc_va_t *, AVCodecContext *, vlc_fourcc_t *output);
     int  (*get)(vlc_va_t *, picture_t *pic, uint8_t **data);
@@ -49,7 +48,8 @@ struct vlc_va_t {
  * @param fmt VLC format of the content to decode
  * @return a new VLC object on success, NULL on error.
  */
-vlc_va_t *vlc_va_New(vlc_object_t *obj, AVCodecContext *, const es_format_t *fmt);
+vlc_va_t *vlc_va_New(vlc_object_t *obj, AVCodecContext *,
+                     enum PixelFormat, const es_format_t *fmt);
 
 /**
  * Initializes the acceleration video decoding back-end for libavcodec.

modules/codec/avcodec/vaapi.c

@@ -439,8 +439,12 @@ static void Delete( vlc_va_t *va, AVCodecContext *avctx )
     free( sys );
 }
 
-static int Create( vlc_va_t *va, AVCodecContext *ctx, const es_format_t *fmt )
+static int Create( vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
+                   const es_format_t *fmt )
 {
+    if( pix_fmt != AV_PIX_FMT_VAAPI_VLD )
+        return VLC_EGENERIC;
+
     (void) fmt;
 #ifdef VLC_VA_BACKEND_XLIB
     if( !vlc_xlib_init( VLC_OBJECT(va) ) )
@@ -579,7 +583,6 @@ static int Create( vlc_va_t *va, AVCodecContext *ctx, const es_format_t *fmt )
 
     va->sys = sys;
     va->description = vaQueryVendorString( sys->p_display );
-    va->pix_fmt = PIX_FMT_VAAPI_VLD; /* Only VLD supported */
     va->setup = Setup;
     va->get = Get;
     va->release = Release;

modules/codec/avcodec/vda.c

@@ -115,9 +115,12 @@ static vlc_va_vda_t *vlc_va_vda_Get( vlc_va_t *va )
 
 #pragma mark - module handling
 
-static int Open( vlc_va_t *va, AVCodecContext *ctx,
+static int Open( vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
                  const es_format_t *fmt )
 {
+    if( pix_fmt != AV_PIX_FMT_VDA_VLD )
+        return VLC_EGENERIC;
+
     msg_Dbg( va, "opening VDA module" );
     if( ctx->codec_id != AV_CODEC_ID_H264 )
     {
@@ -140,7 +143,6 @@ static int Open( vlc_va_t *va, AVCodecContext *ctx,
 
     va->sys = p_vda;
     va->description = "VDA";
-    va->pix_fmt = PIX_FMT_VDA_VLD;
     va->setup = Setup;
     va->get = Get;
     va->release = NULL;
@@ -308,12 +310,15 @@ vlc_module_begin ()
     set_callbacks( Open, Close )
 vlc_module_end ()
 
-static int Open( vlc_va_t *va, AVCodecContext *avctx, const es_format_t *fmt )
+static int Open( vlc_va_t *va, AVCodecContext *avctx,
+                 enum PixelFormat pix_fmt, const es_format_t *fmt )
 {
+    if( pix_fmt != AV_PIX_FMT_VDA )
+        return VLC_EGENERIC;
+
     msg_Dbg( va, "VDA decoder Open");
 
     va->description = (char *)"VDA";
-    va->pix_fmt = AV_PIX_FMT_VDA;
     va->setup = Setup;
     va->get = Get;
     va->release = NULL;

modules/codec/avcodec/video.c

@@ -1096,10 +1096,12 @@ static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
 {
     decoder_t *p_dec = p_context->opaque;
     decoder_sys_t *p_sys = p_dec->p_sys;
-    vlc_va_t *p_va = p_sys->p_va;
 
-    if( p_va != NULL )
-        vlc_va_Delete( p_va, p_context );
+    if (p_sys->p_va != NULL)
+    {
+        vlc_va_Delete(p_sys->p_va, p_context);
+        p_sys->p_va = NULL;
+    }
 
     /* Enumerate available formats */
     bool can_hwaccel = false;
@@ -1119,41 +1121,36 @@ static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
     if (!can_hwaccel)
         goto end;
 
-    p_va = vlc_va_New( VLC_OBJECT(p_dec), p_context, &p_dec->fmt_in );
-    if( p_va == NULL )
-        goto end;
-
     for( size_t i = 0; pi_fmt[i] != PIX_FMT_NONE; i++ )
     {
-        if( p_va->pix_fmt != pi_fmt[i] )
+        vlc_va_t *va = vlc_va_New(VLC_OBJECT(p_dec), p_context, pi_fmt[i],
+                                  &p_dec->fmt_in);
+        if (va == NULL)
             continue;
 
         /* We try to call vlc_va_Setup when possible to detect errors when
          * possible (later is too late) */
         if( p_context->width > 0 && p_context->height > 0
-         && vlc_va_Setup( p_va, p_context, &p_dec->fmt_out.video.i_chroma ) )
+         && vlc_va_Setup(va, p_context, &p_dec->fmt_out.video.i_chroma))
         {
             msg_Err( p_dec, "acceleration setup failure" );
-            break;
+            vlc_va_Delete(va, p_context);
+            continue;
         }
 
-        if( p_va->description )
-            msg_Info( p_dec, "Using %s for hardware decoding.",
-                      p_va->description );
+        if (va->description != NULL)
+            msg_Info(p_dec, "Using %s for hardware decoding", va->description);
 
         /* FIXME this will disable direct rendering
          * even if a new pixel format is renegotiated
          */
         p_sys->b_direct_rendering = false;
-        p_sys->p_va = p_va;
+        p_sys->p_va = va;
         p_context->draw_horiz_band = NULL;
         return pi_fmt[i];
     }
 
-    vlc_va_Delete( p_va, p_context );
-
 end:
     /* Fallback to default behaviour */
-    p_sys->p_va = NULL;
     return avcodec_default_get_format( p_context, pi_fmt );
 }

modules/hw/vdpau/avcodec.c

@@ -240,8 +240,12 @@ static int Setup(vlc_va_t *va, AVCodecContext *avctx, vlc_fourcc_t *chromap)
     return VLC_SUCCESS;
 }
 
-static int Open(vlc_va_t *va, AVCodecContext *avctx, const es_format_t *fmt)
+static int Open(vlc_va_t *va, AVCodecContext *avctx, enum PixelFormat pix_fmt,
+                const es_format_t *fmt)
 {
+    if (pix_fmt != AV_PIX_FMT_VDPAU)
+        return VLC_EGENERIC;
+
     void *func;
     VdpStatus err;
 #if (LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 2, 0))
@@ -367,7 +371,6 @@ static int Open(vlc_va_t *va, AVCodecContext *avctx, const es_format_t *fmt)
 
     va->sys = sys;
     va->description = infos;
-    va->pix_fmt = AV_PIX_FMT_VDPAU;
     va->setup = Setup;
     va->get = Lock;
     va->release = NULL;