vdpau_avcodec: fold most of Setup() into Open() and simplify

modules/hw/vdpau/avcodec.c

@@ -46,7 +46,7 @@ struct vlc_va_sys_t
     VdpChromaType type;
     uint32_t width;
     uint32_t height;
-    vlc_vdp_video_field_t **pool;
+    vlc_vdp_video_field_t *pool[6];
 };
 
 #if !LIBAVCODEC_VERSION_CHECK(56, 10, 0, 19, 100)
@@ -149,35 +149,11 @@ static int Setup(vlc_va_t *va, AVCodecContext *avctx, vlc_fourcc_t *chromap)
     if (av_vdpau_get_surface_parameters(avctx, &type, &width, &height))
         return VLC_EGENERIC;
 
-    if (sys->type == type && sys->width == width && sys->height == height
-     && sys->pool != NULL)
-        return VLC_SUCCESS;
-
-#if (LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 2, 0))
-    AVVDPAUContext *hwctx = avctx->hwaccel_context;
-    VdpDecoderProfile profile;
-    VdpStatus err;
-
-    if (hwctx->decoder != VDP_INVALID_HANDLE)
-    {
-        vdp_decoder_destroy(sys->vdp, hwctx->decoder);
-        hwctx->decoder = VDP_INVALID_HANDLE;
-    }
-#endif
-
-    if (sys->pool != NULL)
-    {
-        for (unsigned i = 0; sys->pool[i] != NULL; i++)
-            sys->pool[i]->destroy(sys->pool[i]);
-        free(sys->pool);
-        sys->pool = NULL;
-    }
-
-    sys->type = type;
-    sys->width = width;
-    sys->height =  height;
+    assert(type == sys->type);
+    if (sys->width != width || sys->height != height)
+        return VLC_EGENERIC;
 
-    switch (type)
+    switch (sys->type)
     {
         case VDP_CHROMA_TYPE_420:
             *chromap = VLC_CODEC_VDPAU_VIDEO_420;
@@ -189,54 +165,9 @@ static int Setup(vlc_va_t *va, AVCodecContext *avctx, vlc_fourcc_t *chromap)
             *chromap = VLC_CODEC_VDPAU_VIDEO_444;
             break;
         default:
-            msg_Err(va, "unsupported chroma type %"PRIu32, type);
-            return VLC_EGENERIC;
-    }
-
-    vlc_vdp_video_field_t **pool = malloc(sizeof (*pool) * (avctx->refs + 6));
-    if (unlikely(pool == NULL))
-        return VLC_ENOMEM;
-
-    int i = 0;
-    while (i < avctx->refs + 5)
-    {
-        pool[i] = CreateSurface(va);
-        if (pool[i] == NULL)
-            break;
-        i++;
-    }
-    pool[i] = NULL;
-
-    if (i < avctx->refs + 3)
-    {
-        msg_Err(va, "not enough video RAM");
-        while (i > 0)
-            DestroySurface(pool[--i]);
-        free(pool);
-        return VLC_ENOMEM;
-    }
-    sys->pool = pool;
-
-#if (LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 2, 0))
-    if (av_vdpau_get_profile(avctx, &profile))
-    {
-        msg_Err(va, "no longer supported codec profile");
-        return VLC_EGENERIC;
+            vlc_assert_unreachable();
     }
 
-    err = vdp_decoder_create(sys->vdp, sys->device, profile, width, height,
-                             avctx->refs, &hwctx->decoder);
-    if (err != VDP_STATUS_OK)
-    {
-        msg_Err(va, "%s creation failure: %s", "decoder",
-                vdp_get_error_string(sys->vdp, err));
-        while (i > 0)
-            DestroySurface(pool[--i]);
-        free(pool);
-        hwctx->decoder = VDP_INVALID_HANDLE;
-        return VLC_EGENERIC;
-    }
-#endif
     return VLC_SUCCESS;
 }
 
@@ -277,6 +208,22 @@ static int Open(vlc_va_t *va, AVCodecContext *avctx, enum PixelFormat pix_fmt,
             break;
     }
 #endif
+    VdpChromaType type;
+    uint32_t width, height;
+
+    if (av_vdpau_get_surface_parameters(avctx, &type, &width, &height))
+        return VLC_EGENERIC;
+
+    switch (type)
+    {
+        case VDP_CHROMA_TYPE_420:
+        case VDP_CHROMA_TYPE_422:
+        case VDP_CHROMA_TYPE_444:
+            break;
+        default:
+            msg_Err(va, "unsupported chroma type %"PRIu32, type);
+            return VLC_EGENERIC;
+    }
 
     if (!vlc_xlib_init(VLC_OBJECT(va)))
     {
@@ -284,10 +231,15 @@ static int Open(vlc_va_t *va, AVCodecContext *avctx, enum PixelFormat pix_fmt,
         return VLC_EGENERIC;
     }
 
-    vlc_va_sys_t *sys = malloc(sizeof (*sys));
+    vlc_va_sys_t *sys = malloc(sizeof (*sys)
+                               + avctx->refs * sizeof (sys->pool[0]));
     if (unlikely(sys == NULL))
        return VLC_ENOMEM;
 
+    sys->type = type;
+    sys->width = width;
+    sys->height = height;
+
     err = vdp_get_x11(NULL, -1, &sys->vdp, &sys->device);
     if (err != VDP_STATUS_OK)
     {
@@ -295,11 +247,6 @@ static int Open(vlc_va_t *va, AVCodecContext *avctx, enum PixelFormat pix_fmt,
         return VLC_EGENERIC;
     }
 
-    sys->type = VDP_CHROMA_TYPE_420;
-    sys->width = 0;
-    sys->height = 0;
-    sys->pool = NULL;
-
 #if (LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 2, 0))
     unsigned flags = AV_HWACCEL_FLAG_ALLOW_HIGH_DEPTH;
 
@@ -313,18 +260,18 @@ static int Open(vlc_va_t *va, AVCodecContext *avctx, enum PixelFormat pix_fmt,
 
     (void) fmt;
 #else
-    avctx->hwaccel_context = av_vdpau_alloc_context();
-    if (unlikely(avctx->hwaccel_context == NULL))
+    AVVDPAUContext *hwctx = av_vdpau_alloc_context();
+    if (unlikely(hwctx == NULL))
         goto error;
 
+    hwctx->decoder = VDP_INVALID_HANDLE;
+    avctx->hwaccel_context = hwctx;
+
     err = vdp_get_proc_address(sys->vdp, sys->device,
                                VDP_FUNC_ID_DECODER_RENDER, &func);
     if (err != VDP_STATUS_OK)
         goto error;
 
-    AVVDPAUContext *hwctx = avctx->hwaccel_context;
-
-    hwctx->decoder = VDP_INVALID_HANDLE;
     hwctx->render = func;
 
     /* Check capabilities */
@@ -363,13 +310,40 @@ static int Open(vlc_va_t *va, AVCodecContext *avctx, enum PixelFormat pix_fmt,
                 level, fmt->video.i_width, fmt->video.i_height);
         goto error;
     }
+
+    err = vdp_decoder_create(sys->vdp, sys->device, profile, width, height,
+                             avctx->refs, &hwctx->decoder);
+    if (err != VDP_STATUS_OK)
+    {
+        msg_Err(va, "%s creation failure: %s", "decoder",
+                vdp_get_error_string(sys->vdp, err));
+        goto error;
+    }
 #endif
+    va->sys = sys;
+
+    int i = 0;
+    while (i < avctx->refs + 5)
+    {
+        sys->pool[i] = CreateSurface(va);
+        if (sys->pool[i] == NULL)
+            break;
+        i++;
+    }
+    sys->pool[i] = NULL;
+
+    if (i < avctx->refs + 3)
+    {
+        msg_Err(va, "not enough video RAM");
+        while (i > 0)
+            DestroySurface(sys->pool[--i]);
+        goto error;
+    }
 
     const char *infos;
     if (vdp_get_information_string(sys->vdp, &infos) != VDP_STATUS_OK)
         infos = "VDPAU";
 
-    va->sys = sys;
     va->description = infos;
     va->setup = Setup;
     va->get = Lock;
@@ -378,10 +352,15 @@ static int Open(vlc_va_t *va, AVCodecContext *avctx, enum PixelFormat pix_fmt,
     return VLC_SUCCESS;
 
 error:
-    vdp_release_x11(sys->vdp);
 #if (LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 2, 0))
+    if (hwctx != NULL)
+    {
+        if (hwctx->decoder != VDP_INVALID_HANDLE)
+            vdp_decoder_destroy(sys->vdp, hwctx->decoder);
         av_freep(&avctx->hwaccel_context);
+    }
 #endif
+    vdp_release_x11(sys->vdp);
     free(sys);
     return VLC_EGENERIC;
 }
@@ -392,18 +371,11 @@ static void Close(vlc_va_t *va, AVCodecContext *avctx)
 #if (LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 2, 0))
     AVVDPAUContext *hwctx = avctx->hwaccel_context;
 
-    if (hwctx->decoder != VDP_INVALID_HANDLE)
-    {
     vdp_decoder_destroy(sys->vdp, hwctx->decoder);
-    }
 #endif
 
-    if (sys->pool != NULL)
-    {
     for (unsigned i = 0; sys->pool[i] != NULL; i++)
         DestroySurface(sys->pool[i]);
-        free(sys->pool);
-    }
     vdp_release_x11(sys->vdp);
     av_freep(&avctx->hwaccel_context);
     free(sys);