avcodec: probe acceleration module only once

The acceleration plugin now returns the expected pixel format instead
of checking it. That way, the avcodec module can check it directly.

This works as long as each acceleration plugin supports only one pixel
format (otherwise a table would be required).
(cherry picked from commit b667100bcf0cbd916db4fbb6ce6f8a51ff971ad8)

Conflicts:
	modules/codec/avcodec/dxva2.c
	modules/codec/avcodec/va.h
	modules/codec/avcodec/vaapi.c
	modules/codec/avcodec/vda.c
	modules/codec/avcodec/video.c

modules/codec/avcodec/dxva2.c

@@ -54,7 +54,7 @@
 #include "va.h"
 #include "copy.h"
 
-static int Open(vlc_va_t *, int, int, const es_format_t *);
+static int Open(vlc_va_t *, int, const es_format_t *);
 static void Close(vlc_va_t *);
 
 vlc_module_begin()
@@ -509,13 +509,8 @@ static void Close(vlc_va_t *external)
     free(va);
 }
 
-static int Open(vlc_va_t *p_external, int pixfmt, int codec_id,
-                const es_format_t *fmt)
+static int Open(vlc_va_t *p_external, int codec_id, const es_format_t *fmt)
 {
-    /* Only one VLD supported */
-    if (pixfmt !=  PIX_FMT_DXVA2_VLD)
-        return VLC_EGENERIC;
-
     vlc_va_dxva2_t *va = calloc(1, sizeof(*va));
     if (!va)
         return NULL;
@@ -563,6 +558,7 @@ static int Open(vlc_va_t *p_external, int pixfmt, int codec_id,
 
     /* TODO print the hardware name/vendor for debugging purposes */
     external->description = DxDescribe(va);
+    external->pix_fmt = PIX_FMT_DXVA2_VLD;
     external->setup   = Setup;
     external->get     = Get;
     external->put     = NULL;

modules/codec/avcodec/va.h

@@ -34,6 +34,7 @@ struct vlc_va_t {
     module_t *module;
     char *description;
     bool direct_rendering;
+    int pix_fmt;
 
     int  (*setup)(vlc_va_t *, void **hw, vlc_fourcc_t *output,
                   int width, int height);

modules/codec/avcodec/vaapi.c

@@ -68,7 +68,7 @@
     "By default VLC core recycles the video outputs if possible. If you have " \
     "an AMD/ATI graphics card, then this setting should be false." )
 
-static int Create( vlc_va_t *, int, int, const es_format_t * );
+static int Create( vlc_va_t *, int, const es_format_t * );
 static void Delete( vlc_va_t * );
 
 vlc_module_begin ()
@@ -698,13 +698,8 @@ static bool QuerySurfaceReady( vlc_va_t *p_external, picture_t *pic )
 }
 
 /* */
-static int Create( vlc_va_t *p_va, int pixfmt, int i_codec_id,
-                const es_format_t *fmt )
+static int Create( vlc_va_t *p_va, int i_codec_id, const es_format_t *fmt )
 {
-    /* Only VLD supported */
-    if( pixfmt != PIX_FMT_VAAPI_VLD )
-        return VLC_EGENERIC;
-
     if( !vlc_xlib_init( VLC_OBJECT(p_va) ) )
     {
         msg_Warn( p_va, "Ignoring VA API" );
@@ -728,7 +723,6 @@ static int Create( vlc_va_t *p_va, int pixfmt, int i_codec_id,
     p_va->extract = Extract;
     p_va->display = DisplayPicture;
     p_va->query = QuerySurfaceReady;
-
     return VLC_SUCCESS;
 }
 

modules/codec/avcodec/video.c

@@ -1137,15 +1137,14 @@ static void ffmpeg_ReleaseFrameBuf( struct AVCodecContext *p_context,
 static int ffmpeg_va_Start( void *func, va_list ap )
 {
     vlc_va_t *va = va_arg( ap, vlc_va_t * );
-    int pix = va_arg( ap, int );
     int codec = va_arg( ap, int );
     const es_format_t *fmt = va_arg( ap, const es_format_t * );
-    int (*open)( vlc_va_t *, int, int, const es_format_t * ) = func;
+    int (*open)( vlc_va_t *, int, const es_format_t * ) = func;
 
-    return open( va, pix, codec, fmt );
+    return open( va, codec, fmt );
 }
 
-static vlc_va_t *vlc_va_New( vlc_object_t *parent, int pixfmt, int codec_id,
+static vlc_va_t *vlc_va_New( vlc_object_t *parent, int codec_id,
                              const es_format_t *fmt )
 {
     vlc_va_t *p_va = vlc_object_create( parent, sizeof( *p_va ) );
@@ -1153,7 +1152,7 @@ static vlc_va_t *vlc_va_New( vlc_object_t *parent, int pixfmt, int codec_id,
         return NULL;
 
     p_va->module = vlc_module_load( p_va, "hw decoder", "$avcodec-hw",
-                                    true, ffmpeg_va_Start, p_va, pixfmt,
+                                    true, ffmpeg_va_Start, p_va,
                                     codec_id, fmt );
     if( p_va->module == NULL )
     {
@@ -1182,56 +1181,50 @@ static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
 {
     decoder_t *p_dec = p_context->opaque;
     decoder_sys_t *p_sys = p_dec->p_sys;
+    vlc_va_t *p_va = p_sys->p_va;
 
-    if( p_sys->p_va )
-    {
-        vlc_va_Delete( p_sys->p_va );
-        p_sys->p_va = NULL;
-    }
+    if( p_va != NULL )
+        vlc_va_Delete( p_va );
 
+    p_va = vlc_va_New( VLC_OBJECT(p_dec), p_sys->i_codec_id, &p_dec->fmt_in );
+    if( p_va != NULL )
+    {
         /* Try too look for a supported hw acceleration */
-    for( int i = 0; pi_fmt[i] != PIX_FMT_NONE; i++ )
+        for( size_t i = 0; pi_fmt[i] != PIX_FMT_NONE; i++ )
         {
             const char *name = av_get_pix_fmt_name(pi_fmt[i]);
-        msg_Dbg( p_dec, "Available decoder output format %d (%s)", pi_fmt[i],
-                 name ? name : "unknown" );
-
-        vlc_va_t *p_va = vlc_va_New( VLC_OBJECT(p_dec), pi_fmt[i],
-                                     p_sys->i_codec_id, &p_dec->fmt_in );
-        if( p_va == NULL )
-        {
-            msg_Dbg( p_dec, "acceleration not available" );
+            msg_Dbg( p_dec, "Available decoder output format %d (%s)",
+                     pi_fmt[i], name ? name : "unknown" );
+            if( p_va->pix_fmt != pi_fmt[i] )
                 continue;
-        }
 
-        if( p_sys->p_va &&
-            p_context->width > 0 && p_context->height > 0 )
-        {
             /* We try to call vlc_va_Setup when possible to detect errors when
              * possible (later is too late) */
-            if( vlc_va_Setup( p_sys->p_va,
-                              &p_context->hwaccel_context,
+            if( p_context->width > 0 && p_context->height > 0
+             && vlc_va_Setup( p_va, &p_context->hwaccel_context,
                               &p_dec->fmt_out.video.i_chroma,
                               p_context->width, p_context->height ) )
             {
-                msg_Err( p_dec, "vlc_va_Setup failed" );
-                vlc_va_Delete( p_sys->p_va );
-                p_sys->p_va = NULL;
-            }
+                msg_Err( p_dec, "acceleration setup failure" );
+                break;
             }
 
-        if( p_sys->p_va )
-        {
-            if( p_sys->p_va->description )
-                msg_Info( p_dec, "Using %s for hardware decoding.", p_sys->p_va->description );
+            if( p_va->description )
+                msg_Info( p_dec, "Using %s for hardware decoding.",
+                          p_va->description );
 
             p_sys->b_direct_rendering = p_va->direct_rendering;
             msg_Info( p_dec, "VAAPI uses direct rendering: %s",
                       p_sys->b_direct_rendering ? "yes" : "no" );
+            p_sys->p_va = p_va;
             p_context->draw_horiz_band = NULL;
             return pi_fmt[i];
         }
+
+        msg_Err( p_dec, "acceleration not available" );
+        vlc_va_Delete( p_va );
     }
+    p_sys->p_va = NULL;
 
     /* Fallback to default behaviour */
     return avcodec_default_get_format( p_context, pi_fmt );