vdpau: determine correct decoder profile and level (fixes #8400)

This should render vdpau safe to enable with profiles not supported
by the hardware (such as high depth or high chroma). Software fallback
will be used automatically.
(cherry picked from commit 1b83ab71d0f2787be4070b1a43235dc99d3d5ad2)
Signed-off-by: Jean-Paul Saman <jpsaman@videolan.org>

modules/codec/avcodec/vdpau.c

@@ -153,8 +153,14 @@ static int Init (vlc_va_t *va, void **ctxp, vlc_fourcc_t *chromap,
     height = (height + 3) & ~3;
 
     unsigned surfaces = 2;
-    if (sys->profile == VDP_DECODER_PROFILE_H264_HIGH)
+    switch (sys->profile)
+    {
+      case VDP_DECODER_PROFILE_H264_BASELINE:
+      case VDP_DECODER_PROFILE_H264_MAIN:
+      case VDP_DECODER_PROFILE_H264_HIGH:
         surfaces = 16;
+        break;
+    }
 
     err = sys->DecoderCreate (sys->device, sys->profile, width, height,
                               surfaces, &sys->context.decoder);
@@ -262,29 +268,97 @@ static int Open (vlc_va_t *va, int codec, const es_format_t *fmt)
 {
     VdpStatus err;
     VdpDecoderProfile profile;
+    int level;
 
     switch (codec)
     {
-        case AV_CODEC_ID_MPEG1VIDEO:
-            profile = VDP_DECODER_PROFILE_MPEG1;
-            break;
-        case AV_CODEC_ID_MPEG2VIDEO:
+      case AV_CODEC_ID_MPEG1VIDEO:
+        profile = VDP_DECODER_PROFILE_MPEG1;
+        level = VDP_DECODER_LEVEL_MPEG1_NA;
+        break;
+
+      case AV_CODEC_ID_MPEG2VIDEO:
+        switch (fmt->i_profile)
+        {
+          case FF_PROFILE_MPEG2_MAIN:
             profile = VDP_DECODER_PROFILE_MPEG2_MAIN;
             break;
-        case AV_CODEC_ID_H263:
-        case AV_CODEC_ID_MPEG4:
+          case FF_PROFILE_MPEG2_SIMPLE:
+            profile = VDP_DECODER_PROFILE_MPEG2_SIMPLE;
+            break;
+          default:
+            msg_Err (va, "unsupported %s profile %d", "MPEG2", fmt->i_profile);
+            return VLC_EGENERIC;
+        }
+        level = VDP_DECODER_LEVEL_MPEG2_HL;
+        break;
+
+      case AV_CODEC_ID_H263:
+        profile = VDP_DECODER_PROFILE_MPEG4_PART2_ASP;
+        level = VDP_DECODER_LEVEL_MPEG4_PART2_ASP_L5;
+        break;
+      case AV_CODEC_ID_MPEG4:
+        switch (fmt->i_profile)
+        {
+          case FF_PROFILE_MPEG4_SIMPLE:
+            profile = VDP_DECODER_PROFILE_MPEG4_PART2_SP;
+            break;
+          case FF_PROFILE_MPEG4_SIMPLE_STUDIO:
+            msg_Err (va, "unsupported %s profile %d", "MPEG4", fmt->i_profile);
+            return VLC_EGENERIC;
+          default:
             profile = VDP_DECODER_PROFILE_MPEG4_PART2_ASP;
             break;
-        case AV_CODEC_ID_H264:
+        }
+        level = fmt->i_level;
+        break;
+
+      case AV_CODEC_ID_H264:
+        switch (fmt->i_profile
+                        & ~(FF_PROFILE_H264_CONSTRAINED|FF_PROFILE_H264_INTRA))
+        {
+          case FF_PROFILE_H264_BASELINE:
+            profile = VDP_DECODER_PROFILE_H264_BASELINE;
+            break;
+          case FF_PROFILE_H264_MAIN:
+            profile = VDP_DECODER_PROFILE_H264_MAIN;
+            break;
+          case FF_PROFILE_H264_HIGH:
             profile = VDP_DECODER_PROFILE_H264_HIGH;
             break;
-        case AV_CODEC_ID_WMV3:
-        case AV_CODEC_ID_VC1:
+          case FF_PROFILE_H264_EXTENDED:
+          default:
+            msg_Err (va, "unsupported %s profile %d", "H.264", fmt->i_profile);
+            return VLC_EGENERIC;
+        }
+        level = fmt->i_level;
+        if ((fmt->i_profile & FF_PROFILE_H264_INTRA) && (fmt->i_level == 11))
+            level = VDP_DECODER_LEVEL_H264_1b;
+        break;
+
+      case AV_CODEC_ID_WMV3:
+      case AV_CODEC_ID_VC1:
+        switch (fmt->i_profile)
+        {
+          case FF_PROFILE_VC1_SIMPLE:
+            profile = VDP_DECODER_PROFILE_VC1_SIMPLE;
+            break;
+          case FF_PROFILE_VC1_MAIN:
+            profile = VDP_DECODER_PROFILE_VC1_MAIN;
+            break;
+          case FF_PROFILE_VC1_ADVANCED:
             profile = VDP_DECODER_PROFILE_VC1_ADVANCED;
             break;
-        default:
-            msg_Err (va, "unknown codec (%d)", codec);
+          default:
+            msg_Err (va, "unsupported %s profile %d", "VC-1", fmt->i_profile);
             return VLC_EGENERIC;
+        }
+        level = fmt->i_level;
+        break;
+
+      default:
+        msg_Err (va, "unknown codec %d", codec);
+        return VLC_EGENERIC;
     }
 
     vlc_va_sys_t *sys = malloc (sizeof (*sys));
@@ -336,7 +410,7 @@ static int Open (vlc_va_t *va, int codec, const es_format_t *fmt)
 
     /* Check capabilities */
     VdpBool support;
-    uint32_t level, mb, width, height;
+    uint32_t lvl, mb, width, height;
 
     if (sys->VideoSurfaceQueryCapabilities (device, VDP_CHROMA_TYPE_420,
                                    &support, &width, &height) != VDP_STATUS_OK)
@@ -360,17 +434,18 @@ static int Open (vlc_va_t *va, int codec, const es_format_t *fmt)
         goto error;
     }
 
-    if (sys->DecoderQueryCapabilities (device, profile, &support, &level,
+    if (sys->DecoderQueryCapabilities (device, profile, &support, &lvl,
                                        &mb, &width, &height) != VDP_STATUS_OK)
         support = VDP_FALSE;
-    if (!support || width < fmt->video.i_width || height < fmt->video.i_height)
+    if (!support || (int)lvl < level
+     || width < fmt->video.i_width || height < fmt->video.i_height)
     {
-        msg_Err (va, "decoding profile not supported: %"PRIu32" %ux%u",
-                 profile, fmt->video.i_width, fmt->video.i_height);
+        msg_Err (va, "decoding profile not supported: %"PRIu32".%d %ux%u",
+                 profile, lvl, fmt->video.i_width, fmt->video.i_height);
         goto error;
     }
     msg_Dbg (va, "decoding profile supported maximum: %"PRIu32".%"PRIu32" mb %"
-             PRIu32", %"PRIu32"x%"PRIu32, profile, level, mb, width, height);
+             PRIu32", %"PRIu32"x%"PRIu32, profile, lvl, mb, width, height);
 
     const char *infos;
     if (sys->GetInformationString (&infos) != VDP_STATUS_OK)