VAAPI: refactor VAProfile selection.

modules/codec/avcodec/vaapi.c

@@ -80,10 +80,88 @@ static vlc_va_vaapi_t *vlc_va_vaapi_Get( void *p_va )
 }
 
 /* */
-static int Open( vlc_va_vaapi_t *p_va, int i_codec_id, int i_count )
+static bool HasProfile( VAProfile profile, VAProfile *list, const int list_size )
 {
-    VAProfile i_profile;
-    int i_surface_count;
+    for (int i = 0; i < list_size; i++)
+    {
+        if (list[i] == profile)
+            return true;
+    }
+    return false;
+}
+
+static VAProfile *GetHwProfiles( vlc_va_conn_t *p_conn, int *count )
+{
+    int num = vaMaxNumProfiles( p_conn->p_display );
+    VAProfile *p_list = calloc( num, sizeof( VAProfile ) );
+    if ( !p_list )
+        return NULL;
+
+    VAStatus status = vaQueryConfigProfiles( p_conn->p_display, p_list, &num );
+    if ( status != VA_STATUS_SUCCESS )
+    {
+        free( p_list );
+        return NULL;
+    }
+    *count = num;
+    return p_list;
+}
+
+static VAProfile GetSupportedProfile( vlc_va_conn_t *p_conn, const int i_codec_id )
+{
+    static const int mpeg2_profiles[] =
+        { VAProfileMPEG2Main, VAProfileMPEG2Simple, -1 };
+    static const int mpeg4_profiles[] =
+        { VAProfileMPEG4Main, VAProfileMPEG4AdvancedSimple, VAProfileMPEG4Simple, -1 };
+    static const int h264_profiles[] =
+        { VAProfileH264High, VAProfileH264Main, VAProfileH264Baseline, -1 };
+    static const int wmv3_profiles[] =
+        { VAProfileVC1Main, VAProfileVC1Simple, -1 };
+    static const int vc1_profiles[] =
+        { VAProfileVC1Advanced, -1 };
+
+    const int *profiles = NULL;
+    switch( i_codec_id )
+    {
+    case CODEC_ID_MPEG1VIDEO:
+    case CODEC_ID_MPEG2VIDEO: profiles = mpeg2_profiles; break;
+    case CODEC_ID_MPEG4:      profiles = mpeg4_profiles; break;
+    case CODEC_ID_WMV3:       profiles = wmv3_profiles; break;
+    case CODEC_ID_VC1:        profiles = vc1_profiles; break;
+    case CODEC_ID_H264:       profiles = h264_profiles; break;
+    default:
+        return -1;
+    }
+
+    /* Get supported profiles from HW */
+    int count = 0;
+    VAProfile *p_list = GetHwProfiles( p_conn, &count );
+    if( !p_list )
+        return -1;
+
+    /* Select best match of highest possible quality */
+    int i_profile = -1;
+    bool b_supported = false;
+    for (int i = 0; profiles[i] > -1; i++)
+    {
+        b_supported = HasProfile( profiles[i], p_list, count );
+        if (b_supported)
+        {
+            i_profile = profiles[i];
+            break;
+        }
+    }
+    free(p_list);
+
+    if (!b_supported)
+        return -1;
+
+    return i_profile;
+}
+
+static int CalculateSurfaceCount( const int i_codec_id, const int i_requested )
+{
+    int i_count = -1;
 
     /* NOTE: The number of surfaces requested is calculated
        based on the amount of pictures vout core needs for
@@ -95,39 +173,36 @@ static int Open( vlc_va_vaapi_t *p_va, int i_codec_id, int i_count )
     switch( i_codec_id )
     {
     case CODEC_ID_MPEG1VIDEO:
-    case CODEC_ID_MPEG2VIDEO:
-        i_profile = VAProfileMPEG2Main;
-        i_surface_count = 2+1;
-        break;
-    case CODEC_ID_MPEG4:
-        i_profile = VAProfileMPEG4AdvancedSimple;
-        i_surface_count = 2+1;
-        break;
-    case CODEC_ID_WMV3:
-        i_profile = VAProfileVC1Main;
-        i_surface_count = 2+1;
-        break;
-    case CODEC_ID_VC1:
-        i_profile = VAProfileVC1Advanced;
-        i_surface_count = 2+1;
-        break;
-    case CODEC_ID_H264:
-        i_profile = VAProfileH264High;
-        i_surface_count = 20+1;
-        break;
+    case CODEC_ID_MPEG2VIDEO: i_count = 2+1; break;
+    case CODEC_ID_MPEG4:      i_count = 2+1; break;
+    case CODEC_ID_WMV3:       i_count = 2+1; break;
+    case CODEC_ID_VC1:        i_count = 2+1; break;
+    case CODEC_ID_H264:       i_count = 20+1;break;
     default:
-        return VLC_EGENERIC;
+        return -1;
     }
 
-    int i_needed = __MAX(i_surface_count, i_count);
-    i_surface_count += i_needed;
-    assert( i_surface_count >= 23 );
+    int i_needed = __MAX(i_requested, i_count);
+    i_count += i_needed;
+    assert( i_count >= 23 );
+
+    return i_count;
+}
+
+/* */
+static int Open( vlc_va_vaapi_t *p_va, int i_codec_id, int i_requested )
+{
+    int i_profile;
 
     /* */
     memset( p_va, 0, sizeof(*p_va) );
     p_va->i_config_id  = VA_INVALID_ID;
     p_va->image.image_id = VA_INVALID_ID;
 
+    p_va->i_surface_count = CalculateSurfaceCount(i_codec_id, i_requested);
+    if (p_va->i_surface_count < 0 )
+        return VLC_EGENERIC;
+
     /* Create a VA display */
     p_va->conn = vlc_va_Initialize(NULL);
     if (!p_va->conn)
@@ -136,25 +211,8 @@ static int Open( vlc_va_vaapi_t *p_va, int i_codec_id, int i_count )
     p_va->conn->lock();
 
     /* Check if the selected profile is supported */
-    bool b_supported_profile = false;
-    int i_profiles_nb = vaMaxNumProfiles( p_va->conn->p_display );
-    VAProfile *p_profiles_list = calloc( i_profiles_nb, sizeof( VAProfile ) );
-    if ( !p_profiles_list )
-        goto unlock;
-    VAStatus status = vaQueryConfigProfiles( p_va->conn->p_display, p_profiles_list, &i_profiles_nb );
-    if ( status == VA_STATUS_SUCCESS )
-    {
-        for ( int i = 0; i < i_profiles_nb; i++ )
-        {
-            if ( p_profiles_list[i] == i_profile )
-            {
-                b_supported_profile = true;
-                break;
-            }
-        }
-    }
-    free( p_profiles_list );
-    if ( !b_supported_profile )
+    i_profile = GetSupportedProfile( p_va->conn, i_codec_id );
+    if (i_profile < 0)
         goto unlock;
 
     /* Create a VA configuration */
@@ -162,21 +220,20 @@ static int Open( vlc_va_vaapi_t *p_va, int i_codec_id, int i_count )
     memset( &attrib, 0, sizeof(attrib) );
     attrib.type = VAConfigAttribRTFormat;
     if( vaGetConfigAttributes( p_va->conn->p_display,
-                               i_profile, VAEntrypointVLD, &attrib, 1 ) )
+                               (VAProfile) i_profile, VAEntrypointVLD, &attrib, 1 ) )
         goto unlock;
 
     /* Not sure what to do if not, I don't have a way to test */
     if( (attrib.value & VA_RT_FORMAT_YUV420) == 0 )
         goto unlock;
-    if( vaCreateConfig( p_va->conn->p_display,
-                        i_profile, VAEntrypointVLD, &attrib, 1, &p_va->i_config_id ) )
+
+    if( vaCreateConfig( p_va->conn->p_display, (VAProfile) i_profile,
+                        VAEntrypointVLD, &attrib, 1, &p_va->i_config_id ) )
     {
         p_va->i_config_id = VA_INVALID_ID;
         goto unlock;
     }
 
-    p_va->i_surface_count = i_surface_count;
-
     p_va->b_supports_derive = false;
 
     if( asprintf( &p_va->va.description, "VA API version %d.%d",