codec/avcodec/vaapi: Pass VASurface' s to vaapi-x11 vout

When -V vaapi-x11 is used (and ffmpeg-vaapi-x11 option), then pass
VASurface information in picture->p_sys. This only works with direct
rendering and video filters that directly operate on the picture are
a no go.

* vaapi.c: protect surface->i_refcount
* vaapi_x11.c: use direct rendering

modules/codec/avcodec/vaapi.c

 /*****************************************************************************
  * vaapi.c: VAAPI helpers for the ffmpeg decoder
  *****************************************************************************
- * Copyright (C) 2009 Laurent Aimar
+ * Copyright (C) 2009-2011 Laurent Aimar
  * $Id$
  *
  * Authors: Laurent Aimar <fenrir_AT_ videolan _DOT_ org>
@@ -48,14 +48,6 @@
 
 #ifdef HAVE_AVCODEC_VAAPI
 
-typedef struct
-{
-    VASurfaceID  i_id;
-    int          i_refcount;
-    unsigned int i_order;
-
-} vlc_va_surface_t;
-
 typedef struct
 {
     vlc_va_t     va;
@@ -176,7 +168,7 @@ static void DestroySurfaces( vlc_va_vaapi_t *p_va )
     for( int i = 0; i < p_va->i_surface_count && p_va->p_surface; i++ )
     {
         vlc_va_surface_t *p_surface = &p_va->p_surface[i];
-
+        vlc_mutex_destroy(&p_surface->lock);
         if( p_surface->i_id != VA_INVALID_SURFACE )
             vaDestroySurfaces( p_va->conn->p_display, &p_surface->i_id, 1 );
     }
@@ -218,6 +210,7 @@ static int CreateSurfaces( vlc_va_vaapi_t *p_va, void **pp_hw_ctx, vlc_fourcc_t
         p_surface->i_id = pi_surface_id[i];
         p_surface->i_refcount = 0;
         p_surface->i_order = 0;
+        vlc_mutex_init(&p_surface->lock);
     }
 
     /* Create a context */
@@ -388,12 +381,33 @@ static int Extract( vlc_va_t *p_external, picture_t *p_picture, AVFrame *p_ff )
 
     return VLC_SUCCESS;
 }
-static int DisplayPicture( vlc_va_t *p_external, picture_t *p_picture, AVFrame *p_ff )
+
+static vlc_va_surface_t *FindSurface( vlc_va_t *p_external, const VASurfaceID i_surface_id )
 {
     vlc_va_vaapi_t *p_va = vlc_va_vaapi_Get(p_external);
+    vlc_va_surface_t *p_surface = NULL;
 
-    if( !p_va->image_cache.buffer )
-        return VLC_EGENERIC;
+    for( int i = 0; i < p_va->i_surface_count; i++ )
+    {
+        vlc_va_surface_t *p_tmp = &p_va->p_surface[i];
+        if( p_tmp->i_id == i_surface_id )
+        {
+            vlc_mutex_lock(&p_tmp->lock);
+            /* NOTE: p_tmp->i_refcount can be greater then 1, when surfaces are being reclaimed
+             * this usually only happens when the vout vaapi-x11 is not instantiated yet.
+             */
+            p_tmp->i_refcount++;
+            p_surface = p_tmp;
+            vlc_mutex_unlock(&p_surface->lock);
+            break;
+        }
+    }
+    return p_surface;
+}
+
+static int DisplayPicture( vlc_va_t *p_external, picture_t *p_picture, AVFrame *p_ff )
+{
+    vlc_va_vaapi_t *p_va = vlc_va_vaapi_Get(p_external);
 
     VASurfaceID i_surface_id = (VASurfaceID)(uintptr_t)p_ff->data[3];
 
@@ -404,12 +418,19 @@ static int DisplayPicture( vlc_va_t *p_external, picture_t *p_picture, AVFrame *
 #endif
         return VLC_EGENERIC;
 
+    assert(p_picture->p_sys);
     if (!p_picture->p_sys)
-        return VLC_ENOMEM;
+        return VLC_EGENERIC;
 
     assert( sizeof(p_picture->p_sys) == sizeof(picture_sys_t) );
 
-    p_picture->p_sys->i_surface_id = i_surface_id;
+    /* FindSurface */
+    p_picture->p_sys->surface = FindSurface( p_external, i_surface_id );
+    if( !p_picture->p_sys->surface )
+        return VLC_EGENERIC;
+
+    p_picture->format.i_width = p_picture->format.i_visible_width = p_va->i_surface_width;
+    p_picture->format.i_height = p_picture->format.i_visible_height = p_va->i_surface_height;
     return VLC_SUCCESS;
 }
 
@@ -425,7 +446,7 @@ static int Get( vlc_va_t *p_external, AVFrame *p_ff )
     {
         vlc_va_surface_t *p_surface = &p_va->p_surface[i];
 
-        if( !p_surface->i_refcount )
+        if( p_surface->i_refcount == 0 )
             break;
 
         if( p_surface->i_order < p_va->p_surface[i_old].i_order )
@@ -435,9 +456,13 @@ static int Get( vlc_va_t *p_external, AVFrame *p_ff )
         i = i_old;
 
     vlc_va_surface_t *p_surface = &p_va->p_surface[i];
-
-    p_surface->i_refcount = 1;
+    vlc_mutex_lock(&p_surface->lock);
+    /* NOTE: when the surface is in use and not consumed by vout vaapi-x11,
+     * then p_surface->i_refcount can be greater then 0. Thus always increment.
+     */
+    p_surface->i_refcount++;
     p_surface->i_order = p_va->i_surface_order++;
+    vlc_mutex_unlock(&p_surface->lock);
 
     /* */
     for( int i = 0; i < 4; i++ )
@@ -461,7 +486,11 @@ static void Release( vlc_va_t *p_external, AVFrame *p_ff )
         vlc_va_surface_t *p_surface = &p_va->p_surface[i];
 
         if( p_surface->i_id == i_surface_id )
+        {
+            vlc_mutex_lock(&p_surface->lock);
             p_surface->i_refcount--;
+            vlc_mutex_unlock(&p_surface->lock);
+        }
     }
 }