codec/avcodec/vaapi: Use VLC_CODEC_VAAPI_SURFACE for pictures only carying a VASurface.

Introduce VLC_CODEC_VAAPI_SURFACE to determine what to do with the
passed picture. If VLC_CODEC_VAAPI_SURFACE, then the picture caries
a VASurface which must be used to display. Else it is a normal
picture and should be treated as such.

include/vlc_fourcc.h

@@ -348,6 +348,10 @@
 /* MPEG-I/II layer 3 audio */
 #define VLC_CODEC_MP3       VLC_FOURCC('m','p','3',' ')
 
+/* Non official codecs, used to force direct rendering in a video output */
+/* VAAPI */
+#define VLC_CODEC_VAAPI_SURFACE VLC_FOURCC('V','A','P','I')
+
 /**
  * It returns the codec associated to a fourcc within a ES category.
  *

modules/codec/avcodec/avcodec.c

@@ -129,7 +129,6 @@ vlc_module_begin ()
                  true )
 #if defined(HAVE_AVCODEC_VAAPI) || defined(HAVE_AVCODEC_DXVA2)
     add_bool( "ffmpeg-hw", false, HW_TEXT, HW_LONGTEXT, false )
-    add_bool( "ffmpeg-vaapi-x11", false, VAX11_TEXT, VAX11_LONGTEXT, false )
 #endif
 #if defined(FF_THREAD_FRAME)
     add_integer( "ffmpeg-threads", 0, THREADS_TEXT, THREADS_LONGTEXT, true );

modules/codec/avcodec/avcodec.h

@@ -143,11 +143,6 @@ int ffmpeg_OpenCodec( decoder_t *p_dec );
 #define HW_TEXT N_("Hardware decoding")
 #define HW_LONGTEXT N_("This allows hardware decoding when available.")
 
-#define VAX11_TEXT N_("Display using vaapi-x11")
-#define VAX11_LONGTEXT N_("Pass picture handle directly to vaapi-x11 " \
-    "video output without first copying it back to main memory. Use of " \
-    "-V vaapi-x11 is mandatory when using this option." )
-
 #define THREADS_TEXT N_( "Threads" )
 #define THREADS_LONGTEXT N_( "Number of threads used for decoding, 0 meaning auto" )
 

modules/codec/avcodec/vaapi_x11.c

@@ -720,7 +720,8 @@ static void Render(vout_display_t *vd, picture_t *picture, subpicture_t *subpict
     if (subpicture->b_ephemer)
         return;
 
-    if (!picture->p_sys)
+    if (!picture->p_sys &&
+        (picture->format.i_chroma != VLC_CODEC_VAAPI_SURFACE))
     {
         picture->p_sys = RenderImportPicture(vd, picture);
         if (!picture->p_sys)
@@ -812,7 +813,8 @@ static void DisplayPicture(vout_display_t *vd, picture_t *pic, subpicture_t *sub
 
     unsigned int i_order = (sys->i_display_order > 0) ?
         sys->i_display_order++ : pic->p_sys->surface->i_order;
-    if (i_order != pic->p_sys->surface->i_order)
+    if ((i_order != pic->p_sys->surface->i_order) &&
+        (pic->format.i_chroma == VLC_CODEC_VAAPI_SURFACE))
     {
         /* Reset picture */
         msg_Err(vd, "Reclaimed picture - id=%d, order=%d, refcount=%d (%p)",
@@ -880,6 +882,7 @@ static picture_pool_t *Pool (vout_display_t *vd, unsigned requested_count)
             picture_Release(pic);
             break;
         }
+        pic->format.i_chroma = VLC_CODEC_VAAPI_SURFACE;
         sys->b_own_vout = false;
 
         pic_array[count] = pic;

modules/codec/avcodec/video.c

@@ -95,7 +95,6 @@ struct decoder_sys_t
 
     /* VA API */
     vlc_va_t *p_va;
-    bool b_va_display; /* use vaapi-x11 vout */
 
     vlc_sem_t sem_mt;
 };
@@ -219,7 +218,7 @@ int InitVideoDec( decoder_t *p_dec, AVCodecContext *p_context,
     p_sys->p_ff_pic = avcodec_alloc_frame();
     p_sys->b_delayed_open = true;
     p_sys->p_va = NULL;
-    p_sys->b_va_display = var_InheritBool(p_dec, "ffmpeg-vaapi-x11");
+
     vlc_sem_init( &p_sys->sem_mt, 0 );
 
     /* ***** Fill p_context with init values ***** */
@@ -888,7 +887,7 @@ static void ffmpeg_CopyPicture( decoder_t *p_dec,
 
     if( p_sys->p_va )
     {
-        if (p_sys->b_va_display)
+        if (p_pic->format.i_chroma == VLC_CODEC_VAAPI_SURFACE)
             vlc_va_Display( p_sys->p_va, p_pic, p_ff_pic );
         else
             vlc_va_Extract( p_sys->p_va, p_pic, p_ff_pic );