codec/avcodec/vaapi_x11.c: Use VAImageFormat information from VAAPI to fill vd->fmt.

Use VAImageFormat information from VAAPI to fill vd->fmt.

modules/codec/avcodec/vaapi_x11.c

@@ -82,6 +82,48 @@ static void PictureDisplay (vout_display_t *, picture_t *, subpicture_t *);
 static int Control  (vout_display_t *, int, va_list);
 static void Manage  (vout_display_t *);
 
+
+static int FindVAFourCC(vout_display_t *vd)
+{
+    vout_display_sys_t *sys = (vout_display_sys_t *) vd->sys;
+
+    /* Find and create a supported image chroma */
+    int i_fmt_count = vaMaxNumImageFormats(sys->conn->p_display);
+    VAImageFormat *p_fmt = calloc(i_fmt_count, sizeof(*p_fmt));
+    if( !p_fmt )
+        goto error;
+
+    if( vaQueryImageFormats( sys->conn->p_display, p_fmt, &i_fmt_count ) )
+    {
+        free( p_fmt );
+        goto error;
+    }
+
+    vlc_fourcc_t  i_chroma = 0;
+    int32_t i_bits_per_pixel = 0;
+    for( int i = 0; i < i_fmt_count; i++ )
+    {
+        if( p_fmt[i].fourcc == VA_FOURCC( 'Y', 'V', '1', '2' ) ||
+            p_fmt[i].fourcc == VA_FOURCC( 'I', '4', '2', '0' ) ||
+            p_fmt[i].fourcc == VA_FOURCC( 'N', 'V', '1', '2' ) )
+        {
+            i_chroma = VLC_CODEC_YV12;
+            i_bits_per_pixel = p_fmt[i].bits_per_pixel;
+            break;
+        }
+    }
+    free( p_fmt );
+    if( !i_chroma )
+        goto error;
+
+    vd->fmt.i_chroma = i_chroma;
+    vd->fmt.i_bits_per_pixel = i_bits_per_pixel;
+    return VLC_SUCCESS;
+
+error:
+    return VLC_EGENERIC;
+}
+
 /* X11 Window */
 static int CreateWindow(vout_display_t *vd)
 {
@@ -93,44 +135,16 @@ static int CreateWindow(vout_display_t *vd)
 
     XWindowAttributes attr;
     XGetWindowAttributes(sys->conn->p_display_x11, sys->x11.root, &attr);
-    int32_t depth = attr.depth;
+    uint32_t depth = attr.depth;
     switch(attr.depth)
     {
         case 8:
-            if (vd->fmt.i_bits_per_pixel != 8)
-                vd->fmt.i_bits_per_pixel = 8;
-            vd->fmt.i_chroma = VLC_CODEC_RGB8;
-            break;
         case 15:
-            if (vd->fmt.i_bits_per_pixel != 15)
-                vd->fmt.i_bits_per_pixel = 15;
-            vd->fmt.i_chroma = VLC_CODEC_RGB15;
-            break;
         case 16:
-            if (vd->fmt.i_bits_per_pixel != 16)
-                vd->fmt.i_bits_per_pixel = 16;
-            vd->fmt.i_chroma = VLC_CODEC_RGB16;
-            break;
         case 24:
-            if (vd->fmt.i_bits_per_pixel == 32)
-                vd->fmt.i_chroma = VLC_CODEC_RGB32;
-            else if (vd->fmt.i_bits_per_pixel == 24)
-                vd->fmt.i_chroma = VLC_CODEC_RGB24;
-            else
-            {
-                vd->fmt.i_bits_per_pixel = 24;
-                vd->fmt.i_chroma = VLC_CODEC_RGB24;
-            }
-            break;
-        case 32:
-            if (vd->fmt.i_bits_per_pixel != 32)
-                vd->fmt.i_bits_per_pixel = 32;
-            vd->fmt.i_chroma = VLC_CODEC_RGB32;
+        case 32: /* de nothing */
             break;
         default:
-            if (vd->fmt.i_bits_per_pixel != 24)
-                vd->fmt.i_bits_per_pixel = 24;
-            vd->fmt.i_chroma = VLC_CODEC_RGB24;
             depth = 24;
             break;
     }
@@ -216,6 +230,9 @@ int OpenVaapiX11 (vlc_object_t *obj)
     if (!sys->conn)
         goto error;
 
+    if (FindVAFourCC(vd) != VLC_SUCCESS)
+        goto error;
+
     if (CreateWindow(vd) != VLC_SUCCESS)
         goto error;