avcodec: remove legacy VDA implementation and add stricter sanity checks before starting to decode

modules/codec/Makefile.am

@@ -367,6 +367,7 @@ endif
 
 libvda_plugin_la_SOURCES = \
 	video_chroma/copy.c video_chroma/copy.h \
+	packetizer/h264_nal.c packetizer/h264_nal.h \
 	codec/avcodec/vda.c
 libvda_plugin_la_CFLAGS = $(AM_CFLAGS) $(AVCODEC_CFLAGS)
 libvda_plugin_la_LDFLAGS = -Wl,-framework,CoreFoundation,-framework,VideoDecodeAcceleration,-framework,QuartzCore

modules/codec/avcodec/vda.c

 /*****************************************************************************
  * vda.c: VDA helpers for the libavcodec decoder
  *****************************************************************************
- * Copyright (C) 2012-2014 VLC authors VideoLAN
+ * Copyright (C) 2012-2015 VLC authors VideoLAN
  *
  * Authors: Sebastien Zwickert <dilaroga@free.fr>
  *          Rémi Denis-Courmont <remi # remlab : net>
@@ -37,6 +37,7 @@
 
 #include "avcodec.h"
 #include "va.h"
+#include "../../packetizer/h264_nal.h"
 #include "../../video_chroma/copy.h"
 
 #include <libavcodec/vda.h>
@@ -50,6 +51,7 @@ static void Close( vlc_va_t * , AVCodecContext *);
 static int Setup( vlc_va_t *, AVCodecContext *, vlc_fourcc_t *);
 static int Get( vlc_va_t *, picture_t *, uint8_t ** );
 static int Extract( vlc_va_t *, picture_t *, uint8_t * );
+static void Release( void *opaque, uint8_t *data );
 
 static void vda_Copy422YpCbCr8( picture_t *p_pic,
                                 CVPixelBufferRef buffer )
@@ -78,259 +80,78 @@ static void vda_Copy422YpCbCr8( picture_t *p_pic,
     CVPixelBufferUnlockBaseAddress( buffer, 0 );
 }
 
-#ifndef HAVE_AV_VDA_ALLOC_CONTEXT
-
-static const int  nvda_pix_fmt_list[] = { 0, 1 };
-static const char *const nvda_pix_fmt_list_text[] =
-  { N_("420YpCbCr8Planar"), N_("422YpCbCr8") };
-
 vlc_module_begin ()
     set_description( N_("Video Decode Acceleration Framework (VDA)") )
     set_capability( "hw decoder", 0 )
     set_category( CAT_INPUT )
     set_subcategory( SUBCAT_INPUT_VCODEC )
     set_callbacks( Open, Close )
-    add_integer ( "avcodec-vda-pix-fmt", 0, VDA_PIX_FMT_TEXT,
-                  VDA_PIX_FMT_LONGTEXT, false)
-        change_integer_list( nvda_pix_fmt_list, nvda_pix_fmt_list_text )
+    add_obsolete_integer("avcodec-vda-pix-fmt") /* since 3.0.0 */
 vlc_module_end ()
 
 struct vlc_va_sys_t
 {
     struct vda_context  hw_ctx;
-
-    uint8_t             *p_extradata;
-    int                 i_extradata;
-
-    vlc_fourcc_t        i_chroma;
-
-    copy_cache_t        image_cache;
 };
 
-typedef struct vlc_va_sys_t vlc_va_vda_t;
-
-static vlc_va_vda_t *vlc_va_vda_Get( vlc_va_t *va )
-{
-    return va->sys;
-}
-
-#pragma mark - module handling
-
-static int Open( vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
-                 const es_format_t *fmt, picture_sys_t *p_sys )
+static int Open(vlc_va_t *va,
+                AVCodecContext *avctx,
+                enum PixelFormat pix_fmt,
+                const es_format_t *fmt,
+                picture_sys_t *p_sys)
 {
-    if( pix_fmt != AV_PIX_FMT_VDA_VLD )
+    if( pix_fmt != AV_PIX_FMT_VDA )
         return VLC_EGENERIC;
 
+    (void) fmt;
     (void) p_sys;
+    VLC_UNUSED(avctx);
 
-    msg_Dbg( va, "opening VDA module" );
-    if( ctx->codec_id != AV_CODEC_ID_H264 )
-    {
-        msg_Warn( va, "input codec isn't H264, canceling VDA decoding" );
-        return VLC_EGENERIC;
-    }
+    size_t i_profile = 0xFFFF, i_level = 0xFFFF;
+    bool b_ret = false;
 
-    if( fmt->p_extra == NULL || fmt->i_extra < 7 )
-    {
-        msg_Warn( va, "VDA requires extradata." );
+    switch (fmt->i_codec) {
+        case VLC_CODEC_H264:
+            b_ret = h264_get_profile_level(fmt, &i_profile, &i_level, NULL);
+            if (!b_ret) {
+                msg_Warn( va, "H264 profile and level parsing failed because it didn't arrive yet");
                 return VLC_EGENERIC;
             }
 
-    vlc_va_vda_t *p_vda = calloc( 1, sizeof(*p_vda) );
-    if( unlikely(p_vda == NULL) )
-        return VLC_EGENERIC;
-
-    p_vda->p_extradata = fmt->p_extra;
-    p_vda->i_extradata = fmt->i_extra;
-
-    va->sys = p_vda;
-    va->description = "VDA";
-    va->setup = Setup;
-    va->get = Get;
-    va->release = NULL;
-    va->extract = Extract;
+            msg_Dbg( va, "trying to decode MPEG-4 Part 10: profile %zu, level %zu", i_profile, i_level);
 
-    return VLC_SUCCESS;
-}
-
-static void Close( vlc_va_t *va, AVCodecContext *ctx )
-{
-    vlc_va_vda_t *p_vda = vlc_va_vda_Get( va );
-
-    msg_Dbg(va, "destroying VDA decoder");
-
-    ff_vda_destroy_decoder( &p_vda->hw_ctx ) ;
-
-    if( p_vda->hw_ctx.cv_pix_fmt_type == kCVPixelFormatType_420YpCbCr8Planar )
-        CopyCleanCache( &p_vda->image_cache );
-
-    free( p_vda );
-    (void) ctx;
-}
-
-static int Setup( vlc_va_t *va, AVCodecContext *avctx, vlc_fourcc_t *pi_chroma )
-{
-    vlc_va_vda_t *p_vda = vlc_va_vda_Get( va );
-
-    if( p_vda->hw_ctx.width == avctx->coded_width
-       && p_vda->hw_ctx.height == avctx->coded_height
-       && p_vda->hw_ctx.decoder )
-    {
-        avctx->hwaccel_context = &p_vda->hw_ctx;
-        *pi_chroma = p_vda->i_chroma;
-        return VLC_SUCCESS;
-    }
-
-    if( p_vda->hw_ctx.decoder )
-    {
-        ff_vda_destroy_decoder( &p_vda->hw_ctx );
-        goto ok;
-    }
-
-    memset( &p_vda->hw_ctx, 0, sizeof(p_vda->hw_ctx) );
-    p_vda->hw_ctx.format = 'avc1';
-    p_vda->hw_ctx.use_ref_buffer = 1;
-
-    int i_pix_fmt = var_CreateGetInteger( va, "avcodec-vda-pix-fmt" );
-
-    switch( i_pix_fmt )
-    {
-        case 1 :
-            p_vda->hw_ctx.cv_pix_fmt_type = kCVPixelFormatType_422YpCbCr8;
-            p_vda->i_chroma = VLC_CODEC_UYVY;
-            msg_Dbg(va, "using pixel format 422YpCbCr8");
+            switch (i_profile) {
+                case PROFILE_H264_BASELINE:
+                case PROFILE_H264_MAIN:
+                case PROFILE_H264_HIGH:
                     break;
-        case 0 :
-        default :
-            p_vda->hw_ctx.cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8Planar;
-            p_vda->i_chroma = VLC_CODEC_I420;
-            CopyInitCache( &p_vda->image_cache, avctx->coded_width );
-            msg_Dbg(va, "using pixel format 420YpCbCr8Planar");
-    }
-
-ok:
-    /* Setup the libavcodec hardware context */
-    avctx->hwaccel_context = &p_vda->hw_ctx;
-    *pi_chroma = p_vda->i_chroma;
-
-    p_vda->hw_ctx.width = avctx->coded_width;
-    p_vda->hw_ctx.height = avctx->coded_height;
-
-    /* create the decoder */
-    int status = ff_vda_create_decoder( &p_vda->hw_ctx,
-                                       p_vda->p_extradata,
-                                       p_vda->i_extradata );
-    if( status )
-    {
-        msg_Err( va, "Failed to create decoder: %i", status );
-        return VLC_EGENERIC;
-    }
-    else
-        msg_Dbg( va, "VDA decoder created");
-
-    return VLC_SUCCESS;
-}
-
-#pragma mark - actual data handling
-
-static void vda_Copy420YpCbCr8Planar( picture_t *p_pic,
-                                      CVPixelBufferRef buffer,
-                                      unsigned i_width,
-                                      unsigned i_height,
-                                      copy_cache_t *cache )
-{
-    uint8_t *pp_plane[3];
-    size_t  pi_pitch[3];
-
-    if (!buffer)
-        return;
-
-    CVPixelBufferLockBaseAddress( buffer, 0 );
 
-    for( int i = 0; i < 3; i++ )
+                default:
                 {
-        pp_plane[i] = CVPixelBufferGetBaseAddressOfPlane( buffer, i );
-        pi_pitch[i] = CVPixelBufferGetBytesPerRowOfPlane( buffer, i );
+                    msg_Dbg( va, "unsupported H264 profile %zu", i_profile);
+                    return -1;
                 }
-
-    CopyFromYv12( p_pic, pp_plane, pi_pitch,
-                  i_width, i_height, cache );
-
-    CVPixelBufferUnlockBaseAddress( buffer, 0 );
-}
-
-static int Get( vlc_va_t *va, picture_t *pic, uint8_t **data )
-{
-    VLC_UNUSED( va );
-
-    (void) pic;
-    *data = (uint8_t *)1; // dummy
-    return VLC_SUCCESS;
-}
-
-static int Extract( vlc_va_t *va, picture_t *p_picture, uint8_t *data )
-{
-    vlc_va_vda_t *p_vda = vlc_va_vda_Get( va );
-    CVPixelBufferRef cv_buffer = ( CVPixelBufferRef )data;
-
-    if( !cv_buffer )
-    {
-        msg_Dbg( va, "Frame buffer is empty.");
-        return VLC_EGENERIC;
-    }
-    if (!CVPixelBufferGetDataSize(cv_buffer) > 0)
-    {
-        msg_Dbg( va, "Empty frame buffer");
-        return VLC_EGENERIC;
             }
+            break;
 
-    if( p_vda->hw_ctx.cv_pix_fmt_type == kCVPixelFormatType_420YpCbCr8Planar )
-    {
-        if( !p_vda->image_cache.buffer ) {
-            CVPixelBufferRelease( cv_buffer );
+        default:
+#ifndef NDEBUG
+            msg_Err( va, "'%4.4s' is not supported", (char *)&fmt->i_codec);
+#endif
             return VLC_EGENERIC;
+            break;
     }
 
-        vda_Copy420YpCbCr8Planar( p_picture,
-                                  cv_buffer,
-                                  p_vda->hw_ctx.width,
-                                  p_vda->hw_ctx.height,
-                                  &p_vda->image_cache );
-    }
-    else
-        vda_Copy422YpCbCr8( p_picture, cv_buffer );
-    return VLC_SUCCESS;
-}
-
-#else
-
-vlc_module_begin ()
-    set_description( N_("Video Decode Acceleration Framework (VDA)") )
-    set_capability( "hw decoder", 0 )
-    set_category( CAT_INPUT )
-    set_subcategory( SUBCAT_INPUT_VCODEC )
-    set_callbacks( Open, Close )
-vlc_module_end ()
-
-static int Open( vlc_va_t *va, AVCodecContext *avctx,
-                 enum PixelFormat pix_fmt, const es_format_t *fmt, picture_sys_t *p_sys )
-{
-    if( pix_fmt != AV_PIX_FMT_VDA )
-        return VLC_EGENERIC;
-
-    msg_Dbg( va, "VDA decoder Open");
+    vlc_va_sys_t *sys = calloc(1, sizeof (*sys));
+    if (unlikely(sys == NULL))
+        return VLC_ENOMEM;
 
+    va->sys = sys;
     va->description = (char *)"VDA";
     va->setup = Setup;
     va->get = Get;
-    va->release = NULL;
+    va->release = Release;
     va->extract = Extract;
-    msg_Dbg( va, "VDA decoder Open success!");
-
-    (void) fmt;
-    (void) avctx;
-    (void) p_sys;
 
     return VLC_SUCCESS;
 }
@@ -341,14 +162,18 @@ static void Close( vlc_va_t *va, AVCodecContext *avctx )
     (void) va;
 }
 
-static int Setup( vlc_va_t *va, AVCodecContext *avctx, vlc_fourcc_t *p_chroma )
+static int Setup( vlc_va_t *va, AVCodecContext *avctx, vlc_fourcc_t *pi_chroma )
 {
-    av_vda_default_free(avctx);
+    int i_ret = av_vda_default_init(avctx);
 
-    (void) va;
-    *p_chroma = VLC_CODEC_UYVY;
+    msg_Dbg(va, "Creating VDA decoder %i", i_ret);
+
+    if (i_ret != 0)
+        return VLC_EGENERIC;
+
+    *pi_chroma = VLC_CODEC_UYVY;
 
-    return (av_vda_default_init(avctx) < 0) ? VLC_EGENERIC : VLC_SUCCESS;
+    return VLC_SUCCESS;
 }
 
 // Never called
@@ -360,6 +185,13 @@ static int Get( vlc_va_t *va, picture_t *p_picture, uint8_t **data )
     return VLC_SUCCESS;
 }
 
+// Never called
+static void Release( void *opaque, uint8_t *data )
+{
+    VLC_UNUSED(opaque);
+    (void) data;
+}
+
 static int Extract( vlc_va_t *va, picture_t *p_picture, uint8_t *data )
 {
     CVPixelBufferRef cv_buffer = (CVPixelBufferRef)data;
@@ -379,4 +211,3 @@ static int Extract( vlc_va_t *va, picture_t *p_picture, uint8_t *data )
 
     return VLC_SUCCESS;
 }
-#endif