VAAPI-X11: use picture_pool_NewExtended() iso of picture_pool_New()

The struct picture_pool_configuration_t has 2 functions called Lock and
Unlock, which are called when a picture is taken from the pool and when
a picture is given back to the pool.

modules/codec/avcodec/vaapi.c

@@ -436,10 +436,7 @@ static vlc_va_surface_t *FindSurface( vlc_va_t *p_external, const VASurfaceID i_
 
             assert( p_tmp->i_state == DECODE );
             assert( p_tmp->i_refcount == 1 );
-            /* NOTE: The surface is still in use by our decoder. Increase the refcount
-             * as we want to keep it until it has been displayed.
-             */
-            p_tmp->i_refcount++;
+
             p_tmp->i_state = DISPLAY;
             p_surface = p_tmp;
             break;

modules/codec/avcodec/vaapi_x11.c

@@ -1216,55 +1216,7 @@ again:
     }
 }
 
-static void ReleaseSurface(vlc_va_surface_t *surface)
-{
-    if (surface->i_refcount > 0)
-        surface->i_refcount--;
-    //assert(surface->i_refcount == 0);
-    surface->i_refcount = 0;
-    surface->i_cache = VA_INVALID_ID;
-    surface->i_state = FREE;
-}
-
-static void PicturePrivateRelease(picture_t *picture)
-{
-    /* FIXME: Do we need this picture longer ? */
-    if (picture->p_sys)
-    {
-        vlc_va_surface_t *surface = (vlc_va_surface_t *) picture->p_sys->surface;
-        vout_display_t *vd = (vout_display_t*) picture->p_sys->obj;
-        vout_display_sys_t *sys = (vout_display_sys_t *) vd->sys;
-
-        if (surface)
-        {
-#if 0 /* FIXME: leaking subpicture associations */
-            if (surface->i_cache != VA_INVALID_ID))
-            {
-                vlc_va_conn_t *conn = vlc_va_get_conn();
-
-                vlc_mutex_lock(&sys->cache_lock);
-                subpicture_cache_t *cache = cache_find(&sys->cache, surface->i_cache);
-                assert(cache);
-                const int count = vlc_array_count(&cache->subpictures);
-                for (int i = 0; i < count; i++)
-                {
-                    vasubpicture_cache_t *vasubpic;
-                    vasubpic = (vasubpicture_cache_t *) vlc_array_item_at_index(&cache->subpictures, i_sub);
-                    assert(vasubpic);
-                    VAStatus status = SubpictureUnlink(conn, surface->i_id, vasubpic->i_id);
-                    if (status != VA_STATUS_SUCCESS)
-                        msg_Err(vd, "- %d unlink failed (status = %d)", i, status);
-                }
-                vlc_mutex_unlock(&sys->cache_lock);
-            }
-#endif
-            ReleaseSurface(surface);
-        }
-        /* */
-        picture->p_sys->surface = NULL;
-    }
-}
-
+/* PictureRelease is called when the picture pool is destroyed. */
 static void PictureRelease(picture_t *picture)
 {
     if (picture->i_refcount > 0)
@@ -1331,29 +1283,7 @@ static void DisplayPicture(vout_display_t *vd, picture_t *pic, subpicture_t *sub
     if (surface->i_state == DISPLAY)
         DisplayVASurface(vd, surface->i_id, pic);
 
-    if (surface->i_cache != VA_INVALID_ID)
-    {
-        vlc_mutex_lock(&sys->cache_lock);
-#ifdef VAAPI_DEBUG
-        msg_Dbg(vd, "unlink surface %d cache %d", surface->i_id, surface->i_cache);
-#endif
-        subpicture_cache_t *cache = cache_find(&sys->cache, surface->i_cache);
-        assert(cache);
-        const int count = vlc_array_count(&cache->subpictures);
-        for (int i = 0; i < count; i++)
-        {
-            vasubpicture_cache_t *sub = vlc_array_item_at_index(&cache->subpictures, i);
-            assert(sub);
-#ifdef VAAPI_DEBUG
-            msg_Dbg(vd, "- %u: subid %d", i, sub->i_id);
-#endif
-            VAStatus status = SubpictureUnlink(sys->conn, surface->i_id, sub->i_id);
-            if (status != VA_STATUS_SUCCESS)
-                msg_Err(vd, "failed unlinking (error %d)", status);
-        }
-        vlc_mutex_unlock(&sys->cache_lock);
-    }
-
+    /* Check if we still need this subpicture cache */
     if (subpicture &&
         (mdate() > subpicture->i_stop) &&
         (surface->i_cache != VA_INVALID_ID) &&
@@ -1380,8 +1310,8 @@ static void DisplayPicture(vout_display_t *vd, picture_t *pic, subpicture_t *sub
         }
     }
 
+    /* */
     sys->display.i_cache = surface->i_cache;
-    ReleaseSurface(surface);
 
     if (subpicture)
         subpicture_Delete(subpicture);
@@ -1389,6 +1319,67 @@ static void DisplayPicture(vout_display_t *vd, picture_t *pic, subpicture_t *sub
     picture_Release(pic);
 }
 
+/**
+ * PictureLock and PictureUnlock will be called by picture_pool_*
+ * when the picture is held and released. Both are called only
+ * once. PictureLock when the picture is retrieved from the pool
+ * and PictureUnlock when the picture is returned to the pool.
+ */
+static int PictureLock(picture_t *picture)
+{
+    vlc_va_surface_t *surface = picture->p_sys->surface;
+    if (surface)
+        surface->i_refcount++;
+    return VLC_SUCCESS;
+}
+
+static void PictureUnlock(picture_t *picture)
+{
+    assert(picture->p_sys);
+
+    vlc_va_surface_t *surface = (vlc_va_surface_t *) picture->p_sys->surface;
+    assert(surface);
+    if (surface == NULL)
+        return;
+
+    vout_display_t *vd = (vout_display_t*) picture->p_sys->obj;
+    vout_display_sys_t *sys = (vout_display_sys_t *) vd->sys;
+
+    if (surface->i_cache != VA_INVALID_ID)
+    {
+        vlc_mutex_lock(&sys->cache_lock);
+#ifdef VAAPI_DEBUG
+        msg_Dbg(vd, "unlink surface %d cache %d", surface->i_id, surface->i_cache);
+#endif
+        subpicture_cache_t *cache = cache_find(&sys->cache, surface->i_cache);
+        assert(cache);
+        const int count = vlc_array_count(&cache->subpictures);
+        for (int i = 0; i < count; i++)
+        {
+            vasubpicture_cache_t *sub = vlc_array_item_at_index(&cache->subpictures, i);
+            assert(sub);
+#ifdef VAAPI_DEBUG
+            msg_Dbg(vd, "- %u: subid %d", i, sub->i_id);
+#endif
+            VAStatus status = SubpictureUnlink(sys->conn, surface->i_id, sub->i_id);
+            if (status != VA_STATUS_SUCCESS)
+                msg_Err(vd, "failed unlinking (error %d)", status);
+        }
+        vlc_mutex_unlock(&sys->cache_lock);
+    }
+
+    /* */
+    if (surface->i_refcount > 0)
+        surface->i_refcount--;
+
+    assert(surface->i_refcount == 0);
+    surface->i_refcount = 0;
+    surface->i_cache = VA_INVALID_ID;
+    surface->i_state = FREE;
+
+    picture->p_sys->surface = NULL;
+}
+
 /**
  * Return a direct buffer
  */
@@ -1425,14 +1416,21 @@ static picture_pool_t *Pool (vout_display_t *vd, unsigned requested_count)
         pic_array[count] = pic;
         pic_array[count]->b_progressive = true;
         pic_array[count]->p_sys = sys;
-        pic_array[count]->pf_private_release = PicturePrivateRelease;
         pic_array[count]->pf_release = PictureRelease;
     }
 
     if (count == 0)
         goto error;
 
-    sys->pool = picture_pool_New(count, pic_array);
+    picture_pool_configuration_t cfg;
+
+    memset(&cfg, 0, sizeof(cfg));
+    cfg.picture_count = count;
+    cfg.picture       = pic_array;
+    cfg.lock          = PictureLock;
+    cfg.unlock        = PictureUnlock;
+
+    sys->pool = picture_pool_NewExtended(&cfg);
     if (!sys->pool)
         goto error;