VAAPI: rework GPU image format detection logic

modules/codec/avcodec/va.c

 /*****************************************************************************
  * va.c: Video Acceleration API for avcodec
  *****************************************************************************
- * Copyright (C) 2011, M2X BV
+ * Copyright (C) 2011 - 2013, M2X BV
  * $Id$
  *
  * Authors: Jean-Paul Saman
@@ -27,6 +27,7 @@
 
 #include <vlc_common.h>
 #include <vlc_fourcc.h>
+#include <vlc_es.h>
 #include <assert.h>
 
 #ifdef HAVE_LIBAVCODEC_AVCODEC_H
@@ -48,6 +49,7 @@
 #ifdef HAVE_AVCODEC_VAAPI
 
 static vlc_mutex_t vlc_va_conn_lock = VLC_STATIC_MUTEX;
+static vlc_va_conn_t vlc_va_conn;
 
 static void vlc_va_lock(void)
 {
@@ -59,6 +61,86 @@ static void vlc_va_unlock(void)
     vlc_mutex_unlock(&vlc_va_conn_lock);
 }
 
+/* */
+static const struct
+{
+    unsigned int chroma;
+    vlc_fourcc_t fcc;
+} chroma_table[] =
+{
+   { VA_FOURCC( 'Y', 'V', '1', '2' ), VLC_CODEC_YV12 },
+   { VA_FOURCC( 'I', '4', '2', '0' ), VLC_CODEC_I420 },
+   { VA_FOURCC( 'N', 'V', '1', '2' ), VLC_CODEC_NV12 },
+   { VA_FOURCC( 'U', 'Y', 'V', 'Y' ), VLC_CODEC_UYVY }
+};
+
+static bool is_derive_supported( vlc_va_conn_t *conn, VASurfaceID surface )
+{
+    assert( surface != VA_INVALID_ID );
+
+    VAImage image;
+    image.image_id = VA_INVALID_ID;
+    if( vaDeriveImage( conn->p_display,
+                       surface,
+                       &image ) == VA_STATUS_SUCCESS )
+    {
+        memcpy((void*)&conn->derive_format, (void*)&image.format, sizeof(VAImageFormat));
+        vaDestroyImage( conn->p_display, image.image_id );
+        return true;
+    }
+    return false;
+}
+
+int vlc_va_get_video_format( const VAImageFormat *format, video_format_t *fmt )
+{
+    for( unsigned int i = 0; i <= ARRAY_SIZE(chroma_table); i++ )
+    {
+        if( format->fourcc == chroma_table[i].chroma )
+        {
+            fmt->i_chroma = chroma_table[i].fcc;
+            fmt->i_bits_per_pixel = format->bits_per_pixel;
+
+            fmt->i_rmask = format->red_mask;
+            fmt->i_gmask = format->green_mask;
+            fmt->i_bmask = format->blue_mask;
+
+            return VLC_SUCCESS;
+        }
+    }
+    return VLC_EGENERIC;
+}
+
+int vlc_va_get_image_format( vlc_va_conn_t *conn, const unsigned int fourcc,
+                             VAImageFormat *format, video_format_t *fmt )
+{
+    /* Find and create a supported image chroma */
+    int count = vaMaxNumImageFormats(conn->p_display);
+    VAImageFormat *p_format = calloc(count, sizeof(*p_format));
+    if( !p_format )
+        return VLC_EGENERIC;
+
+    if( vaQueryImageFormats(conn->p_display, p_format, &count) )
+    {
+        free( p_format );
+        return VLC_EGENERIC;
+    }
+
+    for( int i = 0; i < count; i++ )
+    {
+        if( p_format[i].fourcc == fourcc )
+        {
+            memcpy((void*)format, (void*)&p_format[i], sizeof(VAImageFormat));
+            if( vlc_va_get_video_format(&p_format[i], fmt) == VLC_EGENERIC)
+                abort();
+            free( p_format );
+            return VLC_SUCCESS;
+        }
+    }
+
+    free( p_format );
+    return VLC_EGENERIC;
+}
+
 /* */
 static vlc_va_surface_t *alloc_surfaces( vlc_va_conn_t *conn,
                                          const int width, const int height,
@@ -130,8 +212,8 @@ static vlc_va_surface_t *realloc_surfaces( vlc_va_conn_t *conn,
 }
 
 static bool create_surfaces( vlc_va_conn_t *conn,
-                                    const int width, const int height,
-                                    const int fourcc, const unsigned int num )
+                             const int width, const int height,
+                             const int fourcc, const unsigned int num )
 {
     assert(conn);
 
@@ -164,6 +246,8 @@ static bool create_surfaces( vlc_va_conn_t *conn,
     conn->pool.current = 0;
     if( conn->pool.p_surfaces )
     {
+        conn->b_supports_derive = is_derive_supported( conn,
+                                conn->pool.p_surfaces[0].i_id );
         conn->pool.count = requested;
         return true;
     }
@@ -297,31 +381,38 @@ static void release_surface_by_id(vlc_va_conn_t *conn, VASurfaceID id)
 }
 
 /* Global VAAPI connection state */
-static vlc_va_conn_t vlc_va_conn = {
-    .x11 = NULL,
-    .p_display = 0,
-    .b_direct_rendering = false,
+static void vlc_va_conn_init( vlc_va_conn_t *conn )
+{
+    assert(conn);
+
+    /* libva connection */
+    conn->x11 = NULL;
+    conn->p_display = 0;
     /* libva version */
-    .i_version_major = 0,
-    .i_version_minor = 0,
+    conn->i_version_major = 0;
+    conn->i_version_minor = 0;
     /* ref counting */
-    .i_ref_count = 0,
+    conn->i_ref_count = 0;
+    /* decoding */
+    conn->i_context_id = VA_INVALID_ID;
+    /* features */
+    memset(&conn->derive_format, 0, sizeof(VAImageFormat));
+    conn->b_direct_rendering = false;
+    conn->b_supports_derive = false;
+    /* surfaces pool - NOTE: must be called with lock held */
+    conn->pool.count = 0;
+    conn->pool.current = 0;
+    conn->pool.p_surfaces = NULL;
     /* locking functions */
-    .lock = vlc_va_lock,
-    .unlock = vlc_va_unlock,
-    /* surfaces - NOTE: must be called with lock held */
-    .pool = {
-        .count = 0,
-        .current = 0,
-        .p_surfaces = NULL,
-    },
-    .i_context_id = VA_INVALID_ID,
-    .create_surfaces = create_surfaces,
-    .destroy_surfaces = destroy_surfaces,
-    .get_surface = get_surface,
-    .release_surface = release_surface,
-    .release_surface_by_id = release_surface_by_id,
-};
+    conn->lock = vlc_va_lock;
+    conn->unlock = vlc_va_unlock;
+    /* functions */
+    conn->create_surfaces = create_surfaces;
+    conn->destroy_surfaces = destroy_surfaces;
+    conn->get_surface = get_surface;
+    conn->release_surface = release_surface;
+    conn->release_surface_by_id = release_surface_by_id;
+}
 
 /* */
 static vlc_va_conn_t *vlc_va_get_conn( void )
@@ -369,6 +460,8 @@ vlc_va_conn_t *vlc_va_Initialize( const char *display )
         return conn;
     }
 
+    vlc_va_conn_init( conn );
+
     /* X11 display */
     conn->x11 = XOpenDisplay(display);
     if( !conn->x11 )
@@ -411,10 +504,7 @@ void vlc_va_Terminate( vlc_va_conn_t *conn )
         XCloseDisplay( conn->x11 );
 
         /* Reset values */
-        conn->x11 = NULL;
-        conn->p_display = 0;
-        conn->i_version_major = conn->i_version_minor = 0;
-        conn->i_ref_count = 0;
+        vlc_va_conn_init( conn );
         conn = NULL;
     }
 

modules/codec/avcodec/vaapi.c

@@ -28,6 +28,7 @@
 #include <vlc_common.h>
 #include <vlc_fourcc.h>
 #include <vlc_picture.h>
+#include <vlc_arrays.h>
 #include <assert.h>
 
 #ifdef HAVE_LIBAVCODEC_AVCODEC_H
@@ -69,11 +70,18 @@ typedef struct
     VAImage      image;
     copy_cache_t image_cache;
 
-    bool b_supports_derive;
 } vlc_va_vaapi_t;
 
 static void Close( vlc_va_vaapi_t *p_va );
 
+static unsigned int vaFallbackFourCC[] =
+{
+    VA_FOURCC( 'Y', 'V', '1', '2' ),
+    VA_FOURCC( 'I', '4', '2', '0' ),
+    VA_FOURCC( 'N', 'V', '1', '2' )
+};
+
+/* */
 static vlc_va_vaapi_t *vlc_va_vaapi_Get( void *p_va )
 {
     return p_va;
@@ -234,8 +242,6 @@ static int Open( vlc_va_vaapi_t *p_va, int i_codec_id, int i_requested )
         goto unlock;
     }
 
-    p_va->b_supports_derive = false;
-
     if( asprintf( &p_va->va.description, "VA API version %d.%d",
                   p_va->conn->i_version_major, p_va->conn->i_version_minor ) < 0 )
         p_va->va.description = NULL;
@@ -258,7 +264,7 @@ static void DestroyDecodingContext( vlc_va_vaapi_t *p_va )
         CopyCleanCache( &p_va->image_cache );
         vaDestroyImage( p_va->conn->p_display, p_va->image.image_id );
     }
-    else if(p_va->b_supports_derive)
+    else if( p_va->conn->b_supports_derive )
     {
         CopyCleanCache( &p_va->image_cache );
     }
@@ -316,33 +322,28 @@ static int CreateDecodingContext( vlc_va_vaapi_t *p_va, void **pp_hw_ctx, vlc_fo
     }
     free(p_ids);
 
-    /* Find and create a supported image chroma */
-    int i_fmt_count = vaMaxNumImageFormats( p_va->conn->p_display );
-    VAImageFormat *p_fmt = calloc( i_fmt_count, sizeof(*p_fmt) );
-    if( !p_fmt )
-        goto error;
-
-    if( vaQueryImageFormats( p_va->conn->p_display, p_fmt, &i_fmt_count ) )
+    if( p_va->conn->b_supports_derive )
     {
-        free( p_fmt );
-        goto error;
-    }
+        video_format_t fmt;
+        if( vlc_va_get_video_format( &p_va->conn->derive_format, &fmt ) != VLC_SUCCESS )
+            goto error;
 
-    VAImage testImage;
-    if(vaDeriveImage(p_va->conn->p_display, p_va->conn->pool.p_surfaces[0].i_id, &testImage) == VA_STATUS_SUCCESS)
-    {
-        p_va->b_supports_derive = true;
-        vaDestroyImage(p_va->conn->p_display, testImage.image_id);
+        *pi_chroma = fmt.i_chroma;
     }
-
-    vlc_fourcc_t  i_chroma = 0;
-    for( int i = 0; i < i_fmt_count; i++ )
+    else
     {
-        if( p_fmt[i].fourcc == VA_FOURCC( 'Y', 'V', '1', '2' ) ||
-            p_fmt[i].fourcc == VA_FOURCC( 'I', '4', '2', '0' ) ||
-            p_fmt[i].fourcc == VA_FOURCC( 'N', 'V', '1', '2' ) )
+        /* Find and create a supported image chroma */
+        bool b_supported = false;
+        for( unsigned int i = 0; i < ARRAY_SIZE(vaFallbackFourCC); i++ )
         {
-            if( vaCreateImage( p_va->conn->p_display, &p_fmt[i], i_width, i_height, &p_va->image ) )
+            video_format_t fmt;
+            VAImageFormat  format;
+            if( vlc_va_get_image_format(p_va->conn, vaFallbackFourCC[i],
+                                        &format, &fmt) != VLC_SUCCESS )
+                continue;
+
+            if( vaCreateImage( p_va->conn->p_display, &format,
+                               i_width, i_height, &p_va->image ) )
             {
                 p_va->image.image_id = VA_INVALID_ID;
                 continue;
@@ -351,26 +352,19 @@ static int CreateDecodingContext( vlc_va_vaapi_t *p_va, void **pp_hw_ctx, vlc_fo
             if( vaGetImage( p_va->conn->p_display,
                             p_va->conn->pool.p_surfaces[0].i_id,
                             0, 0, i_width, i_height,
-                            p_va->image.image_id) )
+                            p_va->image.image_id ) )
             {
                 vaDestroyImage( p_va->conn->p_display, p_va->image.image_id );
                 p_va->image.image_id = VA_INVALID_ID;
                 continue;
             }
 
-            i_chroma = VLC_CODEC_YV12;
+            *pi_chroma = fmt.i_chroma;
+            b_supported = true;
             break;
         }
-    }
-    free( p_fmt );
-    if( !i_chroma )
-        goto error;
-    *pi_chroma = i_chroma;
-
-    if(p_va->b_supports_derive)
-    {
-        vaDestroyImage( p_va->conn->p_display, p_va->image.image_id );
-        p_va->image.image_id = VA_INVALID_ID;
+        if( !b_supported )
+            goto error;
     }
 
     if( unlikely(CopyInitCache( &p_va->image_cache, i_width )) )
@@ -385,7 +379,7 @@ static int CreateDecodingContext( vlc_va_vaapi_t *p_va, void **pp_hw_ctx, vlc_fo
     p_va->hw_ctx.context_id = p_va->conn->i_context_id;
 
     /* */
-    p_va->i_surface_chroma = i_chroma;
+    p_va->i_surface_chroma = *pi_chroma;
     p_va->i_surface_width = i_width;
     p_va->i_surface_height = i_height;
 
@@ -439,16 +433,17 @@ static int Extract( vlc_va_t *p_external, picture_t *p_picture, AVFrame *p_ff )
 #endif
         goto error;
 
-    if(p_va->b_supports_derive)
+    if( p_va->conn->b_supports_derive )
     {
-        if(vaDeriveImage(p_va->conn->p_display, i_surface_id, &(p_va->image)) != VA_STATUS_SUCCESS)
+        if( vaDeriveImage( p_va->conn->p_display, i_surface_id,
+                           &(p_va->image) ) != VA_STATUS_SUCCESS )
             goto error;
     }
     else
     {
         if( vaGetImage( p_va->conn->p_display, i_surface_id,
                         0, 0, p_va->i_surface_width, p_va->i_surface_height,
-                        p_va->image.image_id) )
+                        p_va->image.image_id ) )
             goto error;
     }
 
@@ -495,7 +490,7 @@ static int Extract( vlc_va_t *p_external, picture_t *p_picture, AVFrame *p_ff )
     if( vaUnmapBuffer( p_va->conn->p_display, p_va->image.buf ) )
         goto error;
 
-    if(p_va->b_supports_derive)
+    if( p_va->conn->b_supports_derive )
     {
         vaDestroyImage( p_va->conn->p_display, p_va->image.image_id );
         p_va->image.image_id = VA_INVALID_ID;

modules/codec/avcodec/vaapi.h

 /*****************************************************************************
- * Copyright © 2011, M2X BV
+ * Copyright © 2011 - 2013, M2X BV
  * Author: Jean-Paul Saman
  *
  * This library is free software; you can redistribute it and/or
@@ -35,14 +35,16 @@ struct vlc_va_conn_t
 {
     Display      *x11; /* x11 display connection */
     VADisplay     p_display;
-    bool          b_direct_rendering; /* VAAPI uses direct rendering */
+
     int           i_version_major;
     int           i_version_minor;
     int           i_ref_count;
 
-    /* locking functions */
-    void (*lock)(void);
-    void (*unlock)(void);
+    VAContextID   i_context_id;
+    VAImageFormat derive_format;
+
+    bool          b_direct_rendering; /* VAAPI uses direct rendering */
+    bool          b_supports_derive;  /* supports vaDerive */
 
     /* NOTE: must be called with conn->lock held */
     struct {
@@ -51,8 +53,11 @@ struct vlc_va_conn_t
         vlc_va_surface_t *p_surfaces;
     } pool; /* vlc_va_surface_t pool */
 
-    VAContextID   i_context_id;
+    /* locking functions */
+    void (*lock)(void);
+    void (*unlock)(void);
 
+    /* surface functions */
     bool (*create_surfaces)(vlc_va_conn_t *conn, const int width, const int height,
                                    const int fourcc, const unsigned num);
     void (*destroy_surfaces)(vlc_va_conn_t *conn);
@@ -99,6 +104,11 @@ static inline unsigned int vlc_va_conn_SurfacesCount(vlc_va_conn_t *conn)
     return conn->pool.count;
 }
 
+/* Lock libva connection before use */
+int vlc_va_get_video_format( const VAImageFormat *format, video_format_t *fmt );
+int vlc_va_get_image_format(vlc_va_conn_t *conn, const unsigned int fourcc,
+                            VAImageFormat *format, video_format_t *fmt);
+
 /* picture_sys_t */
 struct picture_sys_t
 {

modules/codec/avcodec/vaapi_x11.c

@@ -3,7 +3,7 @@
  * @brief VA API using X11 video output module for VLC media player
  */
 /*****************************************************************************
- * Copyright © 2011 - 2012, M2X BV
+ * Copyright © 2011 - 2013, M2X BV
  * Author: Jean-Paul Saman
  *
  * This library is free software; you can redistribute it and/or
@@ -96,7 +96,8 @@ struct vout_display_sys_t
     xcb_cursor_t cursor; /* blank cursor */
     xcb_window_t window; /* drawable X window */
 
-    bool visible;  /* whether to draw */
+    /* whether to draw */
+    bool visible;
 };
 
 static picture_pool_t *Pool(vout_display_t *, unsigned);
@@ -247,14 +248,6 @@ int OpenVaapiX11(vlc_object_t *obj)
 
     sys->conn = conn;
 
-    vlc_fourcc_t i_chroma;
-    int32_t i_bits_per_pixel;
-    if (FindVAFourCC(sys->vaconn, &sys->img_fmt, &i_chroma, &i_bits_per_pixel) != VLC_SUCCESS)
-        goto error;
-
-    vd->fmt.i_chroma = i_chroma;
-    vd->fmt.i_bits_per_pixel = i_bits_per_pixel;
-
     XSetEventQueueOwner(sys->vaconn->x11, XCBOwnsEventQueue);
 
     RegisterMouseEvents(obj, conn, sys->embed->handle.xid);
@@ -276,9 +269,26 @@ int OpenVaapiX11(vlc_object_t *obj)
         goto error;
     }
 
+    sys->vaconn->lock();
+
+    video_format_t fmt;
+    if (vlc_va_get_image_format(sys->vaconn,
+                                VA_FOURCC( 'Y', 'V', '1', '2' ),
+                                &sys->img_fmt, &fmt) != VLC_SUCCESS)
+    {
+        msg_Err(vd, "GPU does not support requested FOURCC");
+        sys->vaconn->unlock();
+        goto error;
+    }
+
+    sys->vaconn->unlock();
+
     sys->visible = false;
     sys->cursor = CreateBlankCursor(conn, scr);
 
+    vd->fmt.i_chroma = fmt.i_chroma;
+    vd->fmt.i_bits_per_pixel = fmt.i_bits_per_pixel;
+
     msg_Info(vd, "using VAAPI XCB video output (libva version %d.%d)",
              sys->vaconn->i_version_major, sys->vaconn->i_version_minor);
 
@@ -586,7 +596,9 @@ static void DisplayVASurface(vout_display_t *vd, picture_t *picture)
                  (picture->b_top_field_first ? VA_TOP_FIELD : VA_BOTTOM_FIELD);
 
     VAStatus status;
-    status = vaPutSurface(sys->vaconn->p_display, surface->i_id, sys->window,
+    status = vaPutSurface(sys->vaconn->p_display,
+                          surface->i_id,
+                          sys->window,
                           vd->source.i_x_offset, vd->source.i_y_offset,
                           vd->source.i_visible_width, vd->source.i_visible_height,
                           place.x, place.y, place.width, place.height,

modules/codec/avcodec/vasub.c

 /*****************************************************************************
- * Copyright © 2012, M2X BV
+ * Copyright © 2012 - 2013, M2X BV
  * Author: Jean-Paul Saman
  *
  * This library is free software; you can redistribute it and/or
@@ -263,50 +263,6 @@ out_warning:
     return VLC_EGENERIC;
 }
 
-/* VA API support functions */
-int FindVAFourCC(vlc_va_conn_t *vaconn, VAImageFormat *fmt,
-                vlc_fourcc_t *i_chroma, int32_t *i_bits_per_pixel)
-{
-    vaconn->lock();
-
-    /* Find and create a supported image chroma */
-    int i_fmt_count = vaMaxNumImageFormats(vaconn->p_display);
-    VAImageFormat *p_fmt = calloc(i_fmt_count, sizeof(*p_fmt));
-    if( !p_fmt )
-        goto error;
-
-    if( vaQueryImageFormats(vaconn->p_display, p_fmt, &i_fmt_count) )
-    {
-        free( p_fmt );
-        goto error;
-    }
-
-    *i_chroma = 0;
-    *i_bits_per_pixel = 0;
-    for( int i = 0; i < i_fmt_count; i++ )
-    {
-        if( p_fmt[i].fourcc == VA_FOURCC( 'Y', 'V', '1', '2' ) ||
-            p_fmt[i].fourcc == VA_FOURCC( 'I', '4', '2', '0' ) ||
-            p_fmt[i].fourcc == VA_FOURCC( 'N', 'V', '1', '2' ) )
-        {
-            memcpy((void*)fmt, (void*)&p_fmt[i], sizeof(VAImageFormat));
-            *i_chroma = VLC_CODEC_YV12;
-            *i_bits_per_pixel = p_fmt[i].bits_per_pixel;
-            break;
-        }
-    }
-    free( p_fmt );
-    if( *i_chroma == 0 )
-        goto error;
-
-    vaconn->unlock();
-    return VLC_SUCCESS;
-
-error:
-    vaconn->unlock();
-    return VLC_EGENERIC;
-}
-
 static inline void vaapi_fixup_alpha(uint8_t *p_pixel, size_t i_size)
 {
     for (size_t p = 0; p < i_size; p += 4)

modules/codec/avcodec/vasub.h

 /*****************************************************************************
- * Copyright © 2012, M2X BV
+ * Copyright © 2012 - 2013, M2X BV
  * Author: Jean-Paul Saman
  *
  * This library is free software; you can redistribute it and/or
@@ -23,7 +23,6 @@
 #ifdef HAVE_AVCODEC_VAAPI
 /* Check if subtitle format is supported by VA API */
 int VASubtitleFourCC(vlc_va_conn_t *vaconn, const unsigned int vafourcc, VAImageFormat *sub_fmt, int *flags);
-int FindVAFourCC(vlc_va_conn_t *vaconn, VAImageFormat *fmt, vlc_fourcc_t *i_chroma, int32_t *i_bits_per_pixel);
 
 /* */
 typedef struct vasubpicture_cache_t vasubpicture_cache_t;