VAAPI-X11: Workaround for alpha/chroma keying with Subtitles.

The VDPAU wrapper for VAAPI does not support alpha or chroma key setting
for subpictures. However the VDPAU image manipulation functions do handle
transparency. This results in every subpicture to have a green transparent
background. Overriding this with freetype variables does not seem to have
any effect whatsoever in this combination of VAAPI and VDPAU wrapper.

The workaround is to remove the green value of every pixel whenever this
pixel is not completly opaque. This seems to work (tm), expect side-effects
however.

It is best to remove this code when alpha and chroma keying for subtitles
work properly with VAAPI and VDPAU or XvBA. Untill then this workaround is
sadly needed.

modules/codec/avcodec/vaapi_x11.c

@@ -837,7 +837,8 @@ static void CopyPalette(vout_display_t *vd, VAImage *image, video_palette_t *p_p
 #endif
 
 static int CopyPictureToVAImage(vout_display_t *vd, picture_t *pic,
-                                VAImage *image, VAImageFormat *fmt)
+                                VAImage *image, VAImageFormat *fmt,
+                                const bool fixup_alpha)
 {
     vout_display_sys_t *sys = vd->sys;
 
@@ -906,6 +907,29 @@ static int CopyPictureToVAImage(vout_display_t *vd, picture_t *pic,
                 size_t i_size = __MIN(image->pitches[i_plane],
                                       (unsigned int)pic->p[i_plane].i_pitch);
                 vlc_memcpy(&p_out[i_line * image->pitches[i_plane]], &p_in[i_line * pic->p[i_plane].i_pitch], i_size);
+
+                /* FIXME: Remove this code when VDPAU and XvBA backends for
+                * VAAPI properly support alpha or chroma keying in subpictures. */
+                if (fixup_alpha)
+                {
+                    uint8_t *p_pixel = p_out + (i_line * image->pitches[i_plane]);
+                    for (size_t p = 0; p < i_size; p += 4)
+                    {
+                        /* RGBA */
+                        //uint8_t *R = &p_pixel[p];
+                        uint8_t *G = &p_pixel[p+1];
+                        //uint8_t *B = &p_pixel[p+2];
+                        uint8_t *A = &p_pixel[p+3];
+
+                        if (*A != 255) /* alpha transparancy */
+                        {
+                            *A = 0; /* 0 = transparent, 255 = opaque */
+                            *G = 0;
+                        }
+                        //*R = 0;
+                        //*B = 0;
+                    }
+                }
             }
         }
     }
@@ -959,7 +983,8 @@ static int RenderDirectSubpicture(vout_display_t *vd, picture_t *picture, subpic
         image->image_id = VA_INVALID_ID;
         image->buf = VA_INVALID_ID;
 
-        int err = CopyPictureToVAImage(vd, region->p_picture, image, &sys->sub_fmt);
+        int err = CopyPictureToVAImage(vd, region->p_picture, image, &sys->sub_fmt,
+                                       (sys->sflags == 0x0) ? true : false);
         if (err != VLC_SUCCESS)
         {
             free(vasub_cache);