avcodec: probe acceleration module only once

The acceleration plugin now returns the expected pixel format instead
of checking it. That way, the avcodec module can check it directly.

This works as long as each acceleration plugin supports only one pixel
format (otherwise a table would be required).

modules/codec/avcodec/dxva2.c

@@ -51,7 +51,7 @@
 #include "va.h"
 #include "copy.h"
 
-static int Open(vlc_va_t *, int, int, const es_format_t *);
+static int Open(vlc_va_t *, int, const es_format_t *);
 static void Close(vlc_va_t *);
 
 vlc_module_begin()
@@ -501,12 +501,8 @@ static void Close(vlc_va_t *external)
     free(va);
 }
 
-static int Open(vlc_va_t *external, int pixfmt, int codec_id,
-                const es_format_t *fmt)
+static int Open(vlc_va_t *external, int codec_id, const es_format_t *fmt)
 {
-    if( pixfmt != PIX_FMT_DXVA2_VLD )
-        return NULL;
-
     vlc_va_dxva2_t *va = calloc(1, sizeof(*va));
     if (!va)
         return NULL;
@@ -555,6 +551,7 @@ static int Open(vlc_va_t *external, int pixfmt, int codec_id,
 
     /* TODO print the hardware name/vendor for debugging purposes */
     external->description = DxDescribe(va);
+    external->pix_fmt = PIX_FMT_DXVA2_VLD;
     external->setup   = Setup;
     external->get     = Get;
     external->release = Release;

modules/codec/avcodec/va.h

@@ -33,6 +33,7 @@ struct vlc_va_t {
     vlc_va_sys_t *sys;
     module_t *module;
     char *description;
+    int pix_fmt;
 
     int  (*setup)(vlc_va_t *, void **hw, vlc_fourcc_t *output,
                   int width, int height);
@@ -59,7 +60,7 @@ static inline int vlc_va_Extract(vlc_va_t *va, picture_t *dst, AVFrame *src)
     return va->extract(va, dst, src);
 }
 
-static vlc_va_t *vlc_va_New(vlc_object_t *, int, int, const es_format_t *);
+static vlc_va_t *vlc_va_New(vlc_object_t *, int, const es_format_t *);
 static void vlc_va_Delete(vlc_va_t *va);
 
 #endif

modules/codec/avcodec/vaapi.c

@@ -41,7 +41,7 @@
 #include "va.h"
 #include "copy.h"
 
-static int Create( vlc_va_t *, int, int, const es_format_t * );
+static int Create( vlc_va_t *, int, const es_format_t * );
 static void Delete( vlc_va_t * );
 
 vlc_module_begin ()
@@ -507,13 +507,8 @@ static void Delete( vlc_va_t *p_external )
     free( p_va );
 }
 
-static int Create( vlc_va_t *p_va, int pixfmt, int i_codec_id,
-                const es_format_t *fmt )
+static int Create( vlc_va_t *p_va, int i_codec_id, const es_format_t *fmt )
 {
-    /* Only VLD supported */
-    if( pixfmt != PIX_FMT_VAAPI_VLD )
-        return VLC_EGENERIC;
-
     if( !vlc_xlib_init( VLC_OBJECT(p_va) ) )
     {
         msg_Warn( p_va, "Ignoring VA API" );
@@ -526,7 +521,8 @@ static int Create( vlc_va_t *p_va, int pixfmt, int i_codec_id,
     if( err )
         return err;
 
-    /* */
+    /* Only VLD supported */
+    p_va->pix_fmt = PIX_FMT_VAAPI_VLD;
     p_va->setup = Setup;
     p_va->get = Get;
     p_va->release = Release;

modules/codec/avcodec/vda.c

@@ -39,7 +39,7 @@
 #include <libavcodec/vda.h>
 #include <VideoDecodeAcceleration/VDADecoder.h>
 
-static int Open( vlc_va_t *, int, int, const es_format_t * );
+static int Open( vlc_va_t *, int, const es_format_t * );
 static void Close( vlc_va_t * );
 
 static const int  nvda_pix_fmt_list[] = { 0, 1 };
@@ -261,10 +261,9 @@ static void Close( vlc_va_t *p_external )
     free( p_va );
 }
 
-static int Open( vlc_va_t *external, int pixfmt, int i_codec_id,
-                  const es_format_t *fmt )
+static int Open( vlc_va_t *external, int i_codec_id, const es_format_t *fmt )
 {
-    if( pixfmt != PIX_FMT_VDA_VLD || i_codec_id != CODEC_ID_H264 )
+    if( i_codec_id != CODEC_ID_H264 )
         return NULL;
 
     if( fmt->p_extra == NULL || fmt->i_extra < 7 )
@@ -283,6 +282,7 @@ static int Open( vlc_va_t *external, int pixfmt, int i_codec_id,
 
     external->sys = p_va;
     external->description = (char *)"VDA";
+    external->pix_fmt = PIX_FMT_VDA_VLD;
     external->setup = Setup;
     external->get = Get;
     external->release = Release;

modules/codec/avcodec/video.c

@@ -1121,15 +1121,14 @@ static void ffmpeg_ReleaseFrameBuf( struct AVCodecContext *p_context,
 static int ffmpeg_va_Start( void *func, va_list ap )
 {
     vlc_va_t *va = va_arg( ap, vlc_va_t * );
-    int pix = va_arg( ap, int );
     int codec = va_arg( ap, int );
     const es_format_t *fmt = va_arg( ap, const es_format_t * );
-    int (*open)( vlc_va_t *, int, int, const es_format_t * ) = func;
+    int (*open)( vlc_va_t *, int, const es_format_t * ) = func;
 
-    return open( va, pix, codec, fmt );
+    return open( va, codec, fmt );
 }
 
-static vlc_va_t *vlc_va_New( vlc_object_t *parent, int pixfmt, int codec_id,
+static vlc_va_t *vlc_va_New( vlc_object_t *parent, int codec_id,
                              const es_format_t *fmt )
 {
     vlc_va_t *p_va = vlc_object_create( parent, sizeof( *p_va ) );
@@ -1137,7 +1136,7 @@ static vlc_va_t *vlc_va_New( vlc_object_t *parent, int pixfmt, int codec_id,
         return NULL;
 
     p_va->module = vlc_module_load( p_va, "hw decoder", "$avcodec-hw",
-                                    true, ffmpeg_va_Start, p_va, pixfmt,
+                                    true, ffmpeg_va_Start, p_va,
                                     codec_id, fmt );
     if( p_va->module == NULL )
     {
@@ -1166,53 +1165,51 @@ static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
 {
     decoder_t *p_dec = p_context->opaque;
     decoder_sys_t *p_sys = p_dec->p_sys;
+    vlc_va_t *p_va = p_sys->p_va;
 
-    if( p_sys->p_va )
-    {
-        vlc_va_Delete( p_sys->p_va );
-        p_sys->p_va = NULL;
-    }
+    if( p_va != NULL )
+        vlc_va_Delete( p_va );
 
-    /* Try too look for a supported hw acceleration */
-    for( int i = 0; pi_fmt[i] != PIX_FMT_NONE; i++ )
+    p_va = vlc_va_New( VLC_OBJECT(p_dec), p_sys->i_codec_id, &p_dec->fmt_in );
+    if( p_va != NULL )
     {
-        const char *name = av_get_pix_fmt_name(pi_fmt[i]);
-        msg_Dbg( p_dec, "Available decoder output format %d (%s)", pi_fmt[i],
-                 name ? name : "unknown" );
-
-        vlc_va_t *p_va = vlc_va_New( VLC_OBJECT(p_dec), pi_fmt[i],
-                                     p_sys->i_codec_id, &p_dec->fmt_in );
-        if( p_va == NULL )
-        {
-            msg_Dbg( p_dec, "acceleration not available" );
-            continue;
-        }
-        if( p_context->width > 0 && p_context->height > 0 )
+        /* Try too look for a supported hw acceleration */
+        for( size_t i = 0; pi_fmt[i] != PIX_FMT_NONE; i++ )
         {
+            const char *name = av_get_pix_fmt_name(pi_fmt[i]);
+            msg_Dbg( p_dec, "Available decoder output format %d (%s)",
+                     pi_fmt[i], name ? name : "unknown" );
+            if( p_va->pix_fmt != pi_fmt[i] )
+                continue;
+
             /* We try to call vlc_va_Setup when possible to detect errors when
              * possible (later is too late) */
-            if( vlc_va_Setup( p_va, &p_context->hwaccel_context,
+            if( p_context->width > 0 && p_context->height > 0
+             && vlc_va_Setup( p_va, &p_context->hwaccel_context,
                               &p_dec->fmt_out.video.i_chroma,
                               p_context->width, p_context->height ) )
             {
-                msg_Err( p_dec, "vlc_va_Setup failed" );
-                vlc_va_Delete( p_va );
-                continue;
+                msg_Err( p_dec, "acceleration setup failure" );
+                break;
             }
-        }
 
-        p_sys->p_va = p_va;
-        if( p_va->description )
-            msg_Info( p_dec, "Using %s for hardware decoding.",
-                      p_va->description );
+            if( p_va->description )
+                msg_Info( p_dec, "Using %s for hardware decoding.",
+                          p_va->description );
+
+            /* FIXME this will disable direct rendering
+             * even if a new pixel format is renegotiated
+             */
+            p_sys->b_direct_rendering = false;
+            p_sys->p_va = p_va;
+            p_context->draw_horiz_band = NULL;
+            return pi_fmt[i];
+        }
 
-        /* FIXME this will disable direct rendering
-         * even if a new pixel format is renegotiated
-         */
-        p_sys->b_direct_rendering = false;
-        p_context->draw_horiz_band = NULL;
-        return pi_fmt[i];
+        msg_Err( p_dec, "acceleration not available" );
+        vlc_va_Delete( p_va );
     }
+    p_sys->p_va = NULL;
 
     /* Fallback to default behaviour */
     return avcodec_default_get_format( p_context, pi_fmt );