codec/avcoded/vaapi*: Release VASubpictures from picture->pf_release()

Release subpicture association when picture is released. Note that picture_pool_New() overrides this
pictures pf_release() function and restores it before cleaning up the picture pool.

modules/codec/avcodec/va.c

@@ -51,7 +51,7 @@
 static vlc_va_conn_t vlc_va_conn = { NULL, 0, 0, 0, 0 };
 
 /* */
-static vlc_va_conn_t *vlc_va_get_conn( void )
+vlc_va_conn_t *vlc_va_get_conn( void )
 {
     return (vlc_va_conn_t *) &vlc_va_conn;
 }

modules/codec/avcodec/va.h

@@ -66,5 +66,12 @@ static inline void vlc_va_Delete(vlc_va_t *va)
 vlc_va_t *vlc_va_NewVaapi(vlc_object_t *obj, int codec_id);
 vlc_va_t *vlc_va_NewDxva2(vlc_object_t *log, int codec_id);
 
+/* */
+typedef struct vlc_va_conn_t vlc_va_conn_t;
+vlc_va_conn_t *vlc_va_get_conn( void );
+
+vlc_va_conn_t *vlc_va_Initialize(const char *display_name);
+void vlc_va_Terminate(vlc_va_conn_t *conn);
+
 #endif
 

modules/codec/avcodec/vaapi.c

@@ -433,6 +433,8 @@ static int DisplayPicture( vlc_va_t *p_external, picture_t *p_picture, AVFrame *
     if( !p_picture->p_sys->surface )
         return VLC_EGENERIC;
 
+    assert(p_picture->p_sys->subpicture == NULL);
+
     return VLC_SUCCESS;
 }
 

modules/codec/avcodec/vaapi.h

@@ -18,7 +18,6 @@
  ****************************************************************************/
 
 #ifdef HAVE_AVCODEC_VAAPI
-typedef struct vlc_va_conn_t vlc_va_conn_t;
 struct vlc_va_conn_t
 {
     Display      *p_display_x11;
@@ -28,9 +27,6 @@ struct vlc_va_conn_t
     int           i_ref_count; /* for internal use only */
 };
 
-vlc_va_conn_t *vlc_va_Initialize(const char *display_name);
-void vlc_va_Terminate(vlc_va_conn_t *conn);
-
 typedef struct
 {
     VASubpictureID  i_id;  /* VASubpictureIDs */

modules/codec/avcodec/vaapi_x11.c

@@ -693,6 +693,43 @@ static void PictureRelease(picture_t *picture)
 {
     if (picture->i_refcount > 0)
         picture->i_refcount--;
+    if (picture->p_sys)
+    {
+        vlc_va_conn_t *conn = vlc_va_get_conn();
+        vlc_va_subpicture_t *subpicture = picture->p_sys->subpicture;
+
+        VAStatus status;
+        const int i_sub = picture->p_sys->i_sub;
+        for (int i = 0; i < i_sub; i++)
+        {
+            if ((subpicture[i].i_surface != VA_INVALID_SURFACE) &&
+                (subpicture[i].i_id != VA_INVALID_ID))
+            {
+                status = vaDeassociateSubpicture(conn->p_display,
+                            subpicture[i].i_id, &subpicture[i].i_surface, 1);
+                assert(status == VA_STATUS_SUCCESS);
+            }
+
+            if (subpicture[i].i_id != VA_INVALID_ID)
+            {
+                status = vaDestroySubpicture(conn->p_display, subpicture[i].i_id);
+                assert(status == VA_STATUS_SUCCESS);
+            }
+
+            if (subpicture[i].image.image_id != VA_INVALID_ID)
+            {
+                status = vaDestroyImage(conn->p_display,
+                                        subpicture[i].image.image_id);
+                assert(status == VA_STATUS_SUCCESS);
+            }
+        }
+        free(picture->p_sys->subpicture);
+
+        /* */
+        picture->p_sys->i_sub = 0;
+        picture->p_sys->surface = NULL;
+        picture->p_sys->subpicture = NULL;
+    }
     free(picture->p_sys);
     picture->p_sys = NULL;
 }