Fixed completely broken video filter 2 chain in vout. Now they have a

chance to get a frame when they ask (close #1738).

Decoders and filters share the same pool of video frames. But the
decoders try to decode as fast as possible, using all buffers. Then it
s pure luck if the filter can grab a frame...

include/vlc_vout.h

@@ -660,9 +660,14 @@ VLC_EXPORT( void,            vout_DatePicture,    ( vout_thread_t *, picture_t *
 VLC_EXPORT( void,            vout_LinkPicture,    ( vout_thread_t *, picture_t * ) );
 VLC_EXPORT( void,            vout_UnlinkPicture,  ( vout_thread_t *, picture_t * ) );
 VLC_EXPORT( void,            vout_PlacePicture,   ( vout_thread_t *, unsigned int, unsigned int, unsigned int *, unsigned int *, unsigned int *, unsigned int * ) );
+
+/* DO NOT use vout_RenderPicture unless you are in src/video_ouput */
 picture_t *     vout_RenderPicture  ( vout_thread_t *, picture_t *,
                                                        subpicture_t * );
 
+/* DO NOT use vout_CountPictureAvailable unless your are in src/input/dec.c (no exception) */
+int vout_CountPictureAvailable( vout_thread_t * );
+
 VLC_EXPORT( int, vout_vaControlDefault, ( vout_thread_t *, int, va_list ) );
 VLC_EXPORT( void *, vout_RequestWindow, ( vout_thread_t *, int *, int *, unsigned int *, unsigned int * ) );
 VLC_EXPORT( void,   vout_ReleaseWindow, ( vout_thread_t *, void * ) );

src/input/decoder.c

@@ -1235,6 +1235,9 @@ static void aout_del_buffer( decoder_t *p_dec, aout_buffer_t *p_buffer )
                           p_dec->p_owner->p_aout_input, p_buffer );
 }
 
+
+int vout_CountPictureAvailable( vout_thread_t *p_vout );
+
 static picture_t *vout_new_buffer( decoder_t *p_dec )
 {
     decoder_owner_sys_t *p_sys = (decoder_owner_sys_t *)p_dec->p_owner;
@@ -1317,14 +1320,25 @@ static picture_t *vout_new_buffer( decoder_t *p_dec )
             p_sys->p_vout->render.i_bmask = p_sys->video.i_bmask;
     }
 
-    /* Get a new picture */
-    while( !(p_pic = vout_CreatePicture( p_sys->p_vout, 0, 0, 0 ) ) )
+    /* Get a new picture
+     */
+    for( p_pic = NULL; ; )
     {
         int i_pic, i_ready_pic = 0;
 
         if( p_dec->b_die || p_dec->b_error )
-        {
             return NULL;
+
+        /* The video filter chain required that there is always 1 free buffer
+         * that it will use as temporary one. It will release the temporary
+         * buffer once its work is done, so this check is safe even if we don't
+         * lock around both count() and create().
+         */
+        if( vout_CountPictureAvailable( p_sys->p_vout ) >= 2 )
+        {
+            p_pic = vout_CreatePicture( p_sys->p_vout, 0, 0, 0 );
+            if( p_pic )
+                break;
         }
 
 #define p_pic p_dec->p_owner->p_vout->render.pp_picture[i_pic]

src/misc/filter_chain.c

@@ -353,7 +353,6 @@ picture_t *filter_chain_VideoFilter( filter_chain_t *p_chain, picture_t *p_pic )
         if( p_chain->p_this->i_object_type == VLC_OBJECT_VOUT )
         {
             vout_thread_t *p_vout = (vout_thread_t*)p_chain->p_this;
-
             vlc_mutex_lock( &p_vout->picture_lock );
             if( p_pic->i_refcount )
             {

src/video_output/vout_pictures.c

@@ -106,6 +106,35 @@ void vout_DatePicture( vout_thread_t *p_vout,
  * It needs locking since several pictures can be created by several producers
  * threads.
  */
+int vout_CountPictureAvailable( vout_thread_t *p_vout )
+{
+    int i_free = 0;
+    int i_pic;
+
+    vlc_mutex_lock( &p_vout->picture_lock );
+    for( i_pic = 0; i_pic < I_RENDERPICTURES; i_pic++ )
+    {
+        picture_t *p_pic = PP_RENDERPICTURE[(p_vout->render.i_last_used_pic + i_pic + 1) % I_RENDERPICTURES];
+
+        switch( p_pic->i_status )
+        {
+            case DESTROYED_PICTURE:
+                i_free++;
+                break;
+
+            case FREE_PICTURE:
+                i_free++;
+                break;
+
+            default:
+                break;
+        }
+    }
+    vlc_mutex_unlock( &p_vout->picture_lock );
+
+    return i_free;
+}
+
 picture_t *vout_CreatePicture( vout_thread_t *p_vout,
                                bool b_progressive,
                                bool b_top_field_first,