codec/avcodec/vaapi: subtitle support for RGBA only

Each subpicture region is merged into one for libva 0.31.0 and applied
on top of the display surface by the GPU.
.
Higher versions of the libva library can handle multiple subpicture
region associations on a surface with vaAssociateSubpicture2() function.

- subtitles are readable
- some are clipped, this needs fixing
- crash in vlc_memcpy() (line: 545 of vaapi_x11.c file)
- fix positioning of subtitles (region->i_align)
- alpha is applied when available
- chroma is applied when available

modules/codec/avcodec/vaapi.c

@@ -213,6 +213,8 @@ static int CreateSurfaces( vlc_va_vaapi_t *p_va, void **pp_hw_ctx, vlc_fourcc_t
         p_surface->i_id = pi_surface_id[i];
         p_surface->i_refcount = 0;
         p_surface->i_order = 0;
+        p_surface->i_sub = 0;
+        p_surface->subpicture = NULL;
         vlc_mutex_init(&p_surface->lock);
     }
 
@@ -245,7 +247,7 @@ static int CreateSurfaces( vlc_va_vaapi_t *p_va, void **pp_hw_ctx, vlc_fourcc_t
             p_fmt[i].fourcc == VA_FOURCC( 'I', '4', '2', '0' ) ||
             p_fmt[i].fourcc == VA_FOURCC( 'N', 'V', '1', '2' ) )
         {
-            if( vaCreateImage(  p_va->conn->p_display, &p_fmt[i], i_width, i_height, &p_va->image ) )
+            if( vaCreateImage( p_va->conn->p_display, &p_fmt[i], i_width, i_height, &p_va->image ) )
             {
                 p_va->image.image_id = VA_INVALID_ID;
                 continue;
@@ -393,17 +395,18 @@ static vlc_va_surface_t *FindSurface( vlc_va_t *p_external, const VASurfaceID i_
     for( int i = 0; i < p_va->i_surface_count; i++ )
     {
         vlc_va_surface_t *p_tmp = &p_va->p_surface[i];
+        vlc_mutex_lock(&p_tmp->lock);
         if( p_tmp->i_id == i_surface_id )
         {
-            vlc_mutex_lock(&p_tmp->lock);
             /* NOTE: p_tmp->i_refcount can be greater then 1, when surfaces are being reclaimed
              * this usually only happens when the vout vaapi-x11 is not instantiated yet.
              */
             p_tmp->i_refcount++;
             p_surface = p_tmp;
-            vlc_mutex_unlock(&p_surface->lock);
+            vlc_mutex_unlock(&p_tmp->lock);
             break;
         }
+        vlc_mutex_unlock(&p_tmp->lock);
     }
     return p_surface;
 }
@@ -432,8 +435,10 @@ static int DisplayPicture( vlc_va_t *p_external, picture_t *p_picture, AVFrame *
     if( !p_picture->p_sys->surface )
         return VLC_EGENERIC;
 
+    vlc_mutex_lock(&p_picture->p_sys->surface->lock);
     p_picture->format.i_width = p_picture->format.i_visible_width = p_va->i_surface_width;
     p_picture->format.i_height = p_picture->format.i_visible_height = p_va->i_surface_height;
+    vlc_mutex_unlock(&p_picture->p_sys->surface->lock);
     return VLC_SUCCESS;
 }
 
@@ -465,7 +470,6 @@ static int Get( vlc_va_t *p_external, AVFrame *p_ff )
      */
     p_surface->i_refcount++;
     p_surface->i_order = p_va->i_surface_order++;
-    vlc_mutex_unlock(&p_surface->lock);
 
     /* */
     for( int i = 0; i < 4; i++ )
@@ -476,6 +480,7 @@ static int Get( vlc_va_t *p_external, AVFrame *p_ff )
         if( i == 0 || i == 3 )
             p_ff->data[i] = (void*)(uintptr_t)p_surface->i_id;/* Yummie */
     }
+    vlc_mutex_unlock(&p_surface->lock);
     return VLC_SUCCESS;
 }
 static void Release( vlc_va_t *p_external, AVFrame *p_ff )
@@ -488,12 +493,10 @@ static void Release( vlc_va_t *p_external, AVFrame *p_ff )
     {
         vlc_va_surface_t *p_surface = &p_va->p_surface[i];
 
+        vlc_mutex_lock(&p_surface->lock);
         if( p_surface->i_id == i_surface_id )
-        {
-            vlc_mutex_lock(&p_surface->lock);
             p_surface->i_refcount--;
-            vlc_mutex_unlock(&p_surface->lock);
-        }
+        vlc_mutex_unlock(&p_surface->lock);
     }
 }
 

modules/codec/avcodec/vaapi.h

@@ -31,12 +31,37 @@ struct vlc_va_conn_t
 vlc_va_conn_t *vlc_va_Initialize(const char *display_name);
 void vlc_va_Terminate(vlc_va_conn_t *conn);
 
+typedef struct
+{
+    VASubpictureID  i_id;  /* VASubpictureIDs */
+    VAImage         image;
+
+    /* positioning */
+    unsigned int    i_x;
+    unsigned int    i_y;
+    unsigned int    i_width;
+    unsigned int    i_height;
+
+    /* */
+    unsigned int    i_alpha;
+    struct {
+        unsigned int min;
+        unsigned int max;
+        unsigned int mask;
+    } chromakey;
+
+} vlc_va_subpicture_t;
+
 typedef struct
 {
     VASurfaceID  i_id;
     int          i_refcount;
     unsigned int i_order;
 
+    /* subpicture */
+    vlc_va_subpicture_t *subpicture;
+    unsigned int         i_sub; /* number of vlc_va_subpicures_t */
+
     vlc_mutex_t  lock;
 } vlc_va_surface_t;