vout_output: add an option to not recycle a vout.

The VA API output on AMD does not like to be recycled. Therefor add
an option to evade the vout recycle path.

include/vlc_vout.h

@@ -116,6 +116,16 @@ VLC_API vout_thread_t * vout_Request( vlc_object_t *object, const vout_configura
  */
 VLC_API void vout_Close( vout_thread_t *p_vout );
 
+/**
+ * This function will tell if a video output can be recycled during vout_Request.
+ * If the answer is false, then the vout object will be closed and released when
+ * the vout is about to be destroyed or when it is considered for reuse.
+ *
+ * \param p_vout the vout to close
+ * \return true when vout can be reused safely, false when not
+ */
+VLC_API bool vout_CanVoutBeRecycled( vout_thread_t *p_vout );
+
 /**
  * This function will close a vout created by vout_Create
  * and then release it.

include/vlc_vout_display.h

@@ -125,6 +125,7 @@ typedef struct {
     bool has_hide_mouse;                    /* Is mouse automatically hidden */
     bool has_pictures_invalid;              /* Will VOUT_DISPLAY_EVENT_PICTURES_INVALID be used */
     bool has_event_thread;                  /* Will events (key at least) be emitted using an independent thread */
+    bool can_recycle;                       /* Can reuse video output for next playlist item */
     const vlc_fourcc_t *subpicture_chromas; /* List of supported chromas for subpicture rendering. */
 } vout_display_info_t;
 

modules/codec/avcodec/vaapi_x11.c

@@ -289,6 +289,7 @@ int OpenVaapiX11(vlc_object_t *obj)
     info.has_pictures_invalid = false;
     info.has_event_thread = true;
     info.has_hide_mouse = false;
+    info.can_recycle = false;
 
     /* Setup vout_display_t once everything is fine */
     if (VASubtitleFourCC(sys->vaconn, VA_FOURCC_RGBA, &sys->sub_fmt, &sys->sflags) == VLC_SUCCESS)
@@ -383,6 +384,7 @@ void CloseVaapiX11(vlc_object_t *obj)
         vout_display_DeleteWindow (vd, sys->embed);
 
     free(vd->sys);
+    msg_Info(vd, "VAAPI XCB closed");
 }
 
 static void Manage(vout_display_t *vd)

src/input/decoder.c

@@ -2137,8 +2137,9 @@ static void DeleteDecoder( decoder_t * p_dec )
         vout_Reset( p_owner->p_vout );
 
         /* */
+        bool b_recycle = vout_CanVoutBeRecycled( p_owner->p_vout );
         input_resource_RequestVout( p_owner->p_resource, p_owner->p_vout, NULL,
-                                    0, true );
+                                    0, b_recycle );
         if( p_owner->p_input != NULL )
             input_SendEventVout( p_owner->p_input );
     }
@@ -2346,6 +2347,7 @@ static picture_t *vout_new_buffer( decoder_t *p_dec )
             !p_dec->fmt_out.video.i_height )
         {
             /* Can't create a new vout without display size */
+            msg_Err( p_dec, "Cannot create a new vout without display size" );
             return NULL;
         }
 
@@ -2430,6 +2432,7 @@ static picture_t *vout_new_buffer( decoder_t *p_dec )
             dpb_size = 2;
             break;
         }
+
         p_vout = input_resource_RequestVout( p_owner->p_resource,
                                              p_vout, &fmt,
                                              dpb_size +

src/video_output/display.c

@@ -115,6 +115,7 @@ static vout_display_t *vout_display_New(vlc_object_t *obj,
     vd->info.has_hide_mouse = false;
     vd->info.has_pictures_invalid = false;
     vd->info.has_event_thread = false;
+    vd->info.can_recycle = true;
     vd->info.subpicture_chromas = NULL;
 
     vd->cfg = cfg;

src/video_output/video_output.c

@@ -203,6 +203,15 @@ vout_thread_t *(vout_Request)(vlc_object_t *object,
 
     /* If a vout is provided, try reusing it */
     if (vout) {
+        if (!vout_CanVoutBeRecycled(vout))
+        {
+            vout_control_cmd_t cmd;
+            vout_control_cmd_Init(&cmd, VOUT_CONTROL_CLEAN);
+            cmd.u.cfg = cfg;
+
+            vout_control_Push(&vout->p->control, &cmd);
+            goto no_reuse;
+        }
         if (vout->p->input != cfg->input) {
             if (vout->p->input)
                 spu_Attach(vout->p->spu, vout->p->input, false);
@@ -224,6 +233,7 @@ vout_thread_t *(vout_Request)(vlc_object_t *object,
             msg_Dbg(object, "reusing provided vout");
             return vout;
         }
+no_reuse:
         vout_CloseAndRelease(vout);
 
         msg_Warn(object, "cannot reuse provided vout");
@@ -303,6 +313,12 @@ void vout_Reset(vout_thread_t *vout)
     vout_control_WaitEmpty(&vout->p->control);
 }
 
+bool vout_CanVoutBeRecycled(vout_thread_t *vout)
+{
+    assert( vout );
+    return vout->p->display.vd->info.can_recycle;
+}
+
 bool vout_IsEmpty(vout_thread_t *vout)
 {
     vlc_mutex_lock(&vout->p->picture_lock);