codec/avcodec/vaapi: Display picture_t using VAAPI surface.

The video output could also get picture_t * that were not decoded
on the GPU. Still use the GPU to display them.

modules/codec/avcodec/vaapi.c

@@ -403,6 +403,7 @@ static vlc_va_surface_t *FindSurface( vlc_va_t *p_external, const VASurfaceID i_
              */
             p_tmp->i_refcount++;
             p_surface = p_tmp;
+            assert(p_surface->subpicture == NULL);
             free(p_surface->subpicture);
             p_surface->subpicture = NULL;
             p_surface->i_sub = 0;
@@ -431,6 +432,9 @@ static int DisplayPicture( vlc_va_t *p_external, picture_t *p_picture, AVFrame *
     if (!p_picture->p_sys)
         return VLC_EGENERIC;
 
+    if (p_picture->p_sys->b_own_vout)
+        return VLC_SUCCESS;
+
     assert( sizeof(p_picture->p_sys) == sizeof(picture_sys_t) );
 
     /* FindSurface */

modules/codec/avcodec/vaapi.h

@@ -53,5 +53,6 @@ typedef struct
 struct picture_sys_t
 {
     vlc_va_surface_t *surface;
+    bool b_own_vout; /* surface owned by vout? */
 };
 #endif

modules/codec/avcodec/vaapi_x11.c

@@ -68,6 +68,7 @@ struct vout_display_sys_t
 
     picture_pool_t *pool; /* picture pool */
     VAImageFormat sub_fmt; /* Subpicture format VA_FOURCC_RGBA */
+    VAImageFormat img_fmt;
 
     unsigned int i_display_order;
     bool b_alpha;  /* whether subpictures support alpha */
@@ -147,6 +148,7 @@ static int FindVAFourCC(vout_display_t *vd)
             p_fmt[i].fourcc == VA_FOURCC( 'I', '4', '2', '0' ) ||
             p_fmt[i].fourcc == VA_FOURCC( 'N', 'V', '1', '2' ) )
         {
+            memcpy((void*)&sys->img_fmt, (void*)&p_fmt[i], sizeof(VAImageFormat));
             i_chroma = VLC_CODEC_YV12;
             i_bits_per_pixel = p_fmt[i].bits_per_pixel;
             break;
@@ -557,6 +559,80 @@ out_va_free:
     return VLC_EGENERIC;
 }
 
+static VASurfaceID CopyPictureToVASurface(vout_display_t *vd, picture_t *picture, VAImage *image)
+{
+    vout_display_sys_t *sys = vd->sys;
+    VASurfaceID surface = VA_INVALID_SURFACE;
+    VAStatus status;
+
+    status = vaCreateSurfaces(sys->conn->p_display,
+                             picture->format.i_visible_width,
+                             picture->format.i_visible_height,
+                             VA_RT_FORMAT_YUV420,
+                             1, &surface);
+    if (status != VA_STATUS_SUCCESS)
+    {
+        msg_Err(vd, "Failed to create display surface");
+        return VA_INVALID_ID;
+    }
+    image->image_id = VA_INVALID_ID;
+    int err = CopyPictureToVAImage(vd, picture, image, &sys->img_fmt);
+    if (err != VLC_SUCCESS)
+        return VA_INVALID_ID;
+
+    vout_display_place_t place;
+    vout_display_PlacePicture (&place, &vd->source, vd->cfg, false);
+
+    status = vaPutImage(sys->conn->p_display,
+                         surface,
+                         image->image_id,
+                         picture->format.i_x_offset, picture->format.i_y_offset,
+                         picture->format.i_width, picture->format.i_height,
+                         place.x, place.y,
+                         place.width, place.height);
+    if (status != VA_STATUS_SUCCESS)
+        return VA_INVALID_ID;
+
+    return surface;
+}
+
+static picture_sys_t *RenderImportPicture(vout_display_t *vd, picture_t *picture)
+{
+    vout_display_sys_t *sys = vd->sys;
+
+    picture_sys_t *pic_sys = calloc(1, sizeof(picture_sys_t));
+    if (!pic_sys)
+        return NULL;
+
+    pic_sys->b_own_vout = true;
+    pic_sys->surface = calloc(1, sizeof(vlc_va_surface_t));
+    if (!pic_sys->surface)
+    {
+        free(pic_sys);
+        return NULL;
+    }
+
+    VAImage image;
+    image.buf = VA_INVALID_ID;
+    image.image_id = VA_INVALID_ID;
+    pic_sys->surface->i_id = CopyPictureToVASurface(vd, picture, &image);
+    if (pic_sys->surface->i_id == VA_INVALID_SURFACE)
+    {
+        if (image.image_id != VA_INVALID_ID)
+            vaDestroyImage(sys->conn->p_display, image.image_id);
+        free(pic_sys->surface);
+        free(pic_sys);
+        return NULL;
+    }
+
+    if (image.image_id != VA_INVALID_ID)
+        vaDestroyImage(sys->conn->p_display, image.image_id);
+
+    pic_sys->surface->i_refcount = 1;
+    vlc_mutex_init(&pic_sys->surface->lock);
+    return pic_sys;
+}
+
 static int RenderDirectSubpicture(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture, unsigned int i_region)
 {
     vout_display_sys_t *sys = vd->sys;
@@ -613,6 +689,29 @@ static int RenderDirectSubpicture(vout_display_t *vd, picture_t *picture, subpic
     return VLC_SUCCESS;
 }
 
+static void PictureRelease(picture_t *picture)
+{
+    free(picture->p_sys);
+    picture->p_sys = NULL;
+}
+
+static void PictureReleaseVoutOwned(picture_t *picture)
+{
+    if (!picture->p_sys->b_own_vout)
+        abort();
+
+    vlc_va_surface_t *surface = picture->p_sys->surface;
+#if 0
+    /* FIXME: pictures discarded by core will leak VASurfaces */
+    if (surface->i_id != VA_INVALID_SURFACE)
+        vaDestroySurfaces(sys->conn->p_display, &psurface->i_id, 1);
+#endif
+    vlc_mutex_destroy(&surface->lock);
+    free(surface);
+    free(picture->p_sys);
+    picture->p_sys = NULL;
+}
+
 static void Render(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture)
 {
     if (!subpicture)
@@ -621,8 +720,17 @@ static void Render(vout_display_t *vd, picture_t *picture, subpicture_t *subpict
     if (subpicture->b_ephemer)
         return;
 
-    if (!picture->p_sys && !picture->p_sys->surface)
-        return;
+    if (!picture->p_sys)
+    {
+        picture->p_sys = RenderImportPicture(vd, picture);
+        if (!picture->p_sys)
+            return;
+        assert(picture->pf_release);
+        picture->pf_release = PictureReleaseVoutOwned;
+    }
+
+    if (picture->p_sys->surface == NULL)
+        abort();
 
     vlc_mutex_lock(&picture->p_sys->surface->lock);
 
@@ -651,40 +759,13 @@ static void PictureReleaseLocked(picture_t *pic)
     assert(pic);
     assert(sizeof(pic->p_sys) == sizeof(picture_sys_t));
 
-    if (pic->p_sys->surface)
+    if (pic->p_sys && pic->p_sys->surface)
     {
         pic->p_sys->surface->i_refcount--;
         assert(pic->p_sys->surface->i_refcount >= 0);
     }
 }
 
-#if 0
-static VASurfaceID CopyPictureToVASurface(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture)
-{
-    vout_display_sys_t *sys = vd->sys;
-
-    int err = CopyPictureToVAImage(vd, picture, sys->image, sys->image.format);
-    if (err != VLC_SUCCES)
-        return;
-
-    vout_display_place_t place;
-    vout_display_PlacePicture (&place, &vd->source, vd->cfg, false);
-
-    VAStatus status;
-    status = vaPutImages(sys->conn->p_display,
-                         sys->surface,
-                         sys->image.image_id,
-                         picture->format.i_x_offset, picture->format.i_y_offset,
-                         picture->format.i_width, picture->format.i_height,
-                         place.x, place.y
-                         place.width, place.height);
-    if (status != VA_STATUS_SUCCESS)
-        return VA_INVALID_ID;
-
-    return sys->surface;
-}
-#endif
-
 static void DisplayVASurface(vout_display_t *vd, VASurfaceID surface, picture_t *picture)
 {
     vout_display_sys_t *sys = vd->sys;
@@ -710,13 +791,18 @@ static void DisplayPicture(vout_display_t *vd, picture_t *pic, subpicture_t *sub
 {
     vout_display_sys_t *sys = vd->sys;
 
-    assert(pic->p_sys);
+    if (pic->p_sys == NULL)
+    {
+        msg_Err(vd, "Discarding picture without picture_sys_t information");
+        if (subpicture)
+            subpicture_Delete(subpicture);
+        picture_Release(pic);
+        return;
+    }
+
     if (pic->p_sys->surface == NULL)
     {
-        /* FIXME: display picture with surface */
-//        VASurfaceID surface = CopyPictureToVASurface(vd, picture, subpicture);
-//        DisplayVASurface(vd, surface, picture, subpicture);
-        msg_Err(vd, "Discarding picture - no surface information");
+        msg_Err(vd, "Discarding picture without surface information");
         if (subpicture)
             subpicture_Delete(subpicture);
         picture_Release(pic);
@@ -758,7 +844,12 @@ static void DisplayPicture(vout_display_t *vd, picture_t *pic, subpicture_t *sub
     PictureReleaseLocked(pic);
     vlc_mutex_unlock(&pic->p_sys->surface->lock);
 
-    pic->p_sys->surface = NULL;
+    if (pic->p_sys->b_own_vout)
+    {
+        if (pic->p_sys->surface->i_id != VA_INVALID_SURFACE)
+            vaDestroySurfaces(sys->conn->p_display, &pic->p_sys->surface->i_id, 1);
+    }
+    else pic->p_sys->surface = NULL;
     picture_Release(pic);
 }
 
@@ -790,9 +881,12 @@ static picture_pool_t *Pool (vout_display_t *vd, unsigned requested_count)
             picture_Release(pic);
             break;
         }
+        sys->b_own_vout = false;
+
         pic_array[count] = pic;
         pic_array[count]->b_progressive = true;
         pic_array[count]->p_sys = sys;
+        pic_array[count]->pf_release = PictureRelease;
     }
 
     if (count == 0)