avcodec: allow selecting hw accelerated decoding with and without accelerated video outputs.

Use p_va->pix_fmt to decide what to do.

modules/codec/avcodec/video.c

@@ -716,16 +716,42 @@ picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
         if( !b_drawpicture || ( !p_sys->p_va && !p_sys->p_ff_pic->linesize[0] ) )
             continue;
 
-        if( p_sys->p_va && p_sys->p_ff_pic->opaque )
+        if( p_sys->p_va != NULL || p_sys->p_ff_pic->opaque == NULL )
+        {
+            if( p_sys->p_va )
+            {
+                switch( p_sys->p_va->pix_fmt )
+                {
+                case PIX_FMT_VAAPI_VLD:
+                    if( p_sys->p_ff_pic->opaque )
                     {
                         p_pic = (picture_t *)p_sys->p_ff_pic->opaque;
-            if( p_pic ) decoder_LinkPicture( p_dec, p_pic );
-
-            /* Fill p_picture_t from AVVideoFrame and do chroma conversion
-             * if needed */
-            ffmpeg_CopyPicture( p_dec, p_pic, p_sys->p_ff_pic );
+                        if( p_pic )
+                        {
+                            decoder_LinkPicture( p_dec, p_pic );
+                            break;
+                        }
                     }
-        else if( p_sys->p_va != NULL || p_sys->p_ff_pic->opaque == NULL )
+                    /* NOTE: intentional fallthrough !!!
+                     * When no VAAPI accelerated video output is loaded then only use
+                     * hardware accelerated decoding. */
+#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT( 54, 92, 100 )
+                case AV_PIX_FMT_VDPAU:
+#endif
+                case PIX_FMT_DXVA2_VLD:
+                default:
+                    assert( p_pic );
+                    /* Get a new picture */
+                    p_pic = ffmpeg_NewPictBuf( p_dec, p_context );
+                    if( !p_pic )
+                    {
+                        block_Release( p_block );
+                        return NULL;
+                    }
+                    break;
+                }
+            }
+            else
             {
                 /* Get a new picture */
                 p_pic = ffmpeg_NewPictBuf( p_dec, p_context );
@@ -734,6 +760,7 @@ picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
                     block_Release( p_block );
                     return NULL;
                 }
+            }
 
             /* Fill p_picture_t from AVVideoFrame and do chroma conversion
              * if needed */
@@ -911,10 +938,26 @@ static void ffmpeg_CopyPicture( decoder_t *p_dec,
 
     if( p_sys->p_va )
     {
+        switch( p_sys->p_va->pix_fmt )
+        {
+            case PIX_FMT_VAAPI_VLD:
                 if (p_pic->format.i_chroma == VLC_CODEC_VAAPI_SURFACE)
+                {
                     vlc_va_Display( p_sys->p_va, p_pic );
-        else
+                    break;
+                }
+
+                /* NOTE: intentional fallthrough !!!
+                 * When no VAAPI accelerated video output is loaded then only use
+                 * hardware accelerated decoding. */
+#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT( 54, 92, 100 )
+            case AV_PIX_FMT_VDPAU:
+#endif
+            case PIX_FMT_DXVA2_VLD:
+            default:
                 vlc_va_Extract( p_sys->p_va, p_pic, p_ff_pic );
+                break;
+        }
     }
     else if( TestFfmpegChroma( p_sys->p_context->pix_fmt, -1 ) == VLC_SUCCESS )
     {
@@ -958,7 +1001,7 @@ static int ffmpeg_GetFrameBuf( struct AVCodecContext *p_context,
 {
     decoder_t *p_dec = (decoder_t *)p_context->opaque;
     decoder_sys_t *p_sys = p_dec->p_sys;
-    picture_t *p_pic;
+    picture_t *p_pic = NULL;
 
     /* */
     p_ff_pic->opaque = NULL;
@@ -985,26 +1028,40 @@ static int ffmpeg_GetFrameBuf( struct AVCodecContext *p_context,
         assert(0);
 #endif
 
+        switch( p_sys->p_va->pix_fmt )
+        {
             /* NOTE: Get first picture from video output instead of from the
              * normal (non-accellerated vouts) mechanism. The drawback is that
              * decoding the first pictures is delayed with the creation of the
-         * video output module.
-         */
+             * video output module. */
+            case PIX_FMT_VAAPI_VLD:
                 p_pic = ffmpeg_NewPictBuf( p_dec, p_sys->p_context );
-        if( p_pic && ( p_pic->format.i_chroma == VLC_CODEC_VAAPI_SURFACE ) )
+                if( p_pic )
                 {
+                    if( p_pic->format.i_chroma == VLC_CODEC_VAAPI_SURFACE )
+                    {
+                        /* keep picture around */
                         p_ff_pic->opaque = (void*)p_pic;
                         vlc_va_Put( p_sys->p_va, p_ff_pic, p_pic );
+                        return 0;
+                    }
+                    picture_Release( p_pic );
                 }
-        else
-        {
-            if( p_pic ) picture_Release( p_pic );
 
+                /* NOTE: intentional fallthrough !!!
+                 * When no VAAPI accelerated video output is loaded then only use
+                 * hardware accelerated decoding. */
+#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT( 54, 92, 100 )
+            case AV_PIX_FMT_VDPAU:
+#endif
+            case PIX_FMT_DXVA2_VLD:
+            default:
                 if( vlc_va_Get( p_sys->p_va, p_ff_pic ) )
                 {
-                msg_Err( p_dec, "vaGrabSurface failed" );
+                    msg_Err( p_dec, "vlc_va_Get() failed" );
                     return -1;
                 }
+                break;
         }
         return 0;
     }
@@ -1113,12 +1170,31 @@ static void ffmpeg_ReleaseFrameBuf( struct AVCodecContext *p_context,
     decoder_sys_t *p_sys = p_dec->p_sys;
 
     if( p_sys->p_va )
+    {
+        switch( p_sys->p_va->pix_fmt )
+        {
+            case PIX_FMT_VAAPI_VLD:
             {
                 picture_t *p_pic = (picture_t*)p_ff_pic->opaque;
                 if( p_pic )
+                {
                     decoder_UnlinkPicture( p_dec, p_pic );
-        else
+                    break;
+                }
+                /* NOTE: intentional fallthrough !!
+                 * When no hw accelerated video output is loaded, then
+                 * p_ff_pic->opaque will be NULL. Instead copy the
+                 * picture from GPU to main memory.
+                 */
+            }
+#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT( 54, 92, 100 )
+            case AV_PIX_FMT_VDPAU:
+#endif
+            case PIX_FMT_DXVA2_VLD:
+            default:
                 vlc_va_Release( p_sys->p_va, p_ff_pic );
+                break;
+        }
     }
     else if( !p_ff_pic->opaque )
     {