Fixed chain for chroma+resize.

This allows resizing non overlay output without swscale support.
(I have disabled chroma chain building until someone tests it.)

modules/video_filter/chain.c

@@ -34,28 +34,30 @@
 #include <vlc_filter.h>
 
 /*****************************************************************************
- * Local and extern prototypes.
+ * Module descriptor
  *****************************************************************************/
 static int       Activate   ( vlc_object_t * );
 static void      Destroy    ( vlc_object_t * );
-static picture_t *Chain( filter_t *, picture_t * );
-static int AllocInit( filter_t *p_filter, void *p_data );
 
-/*****************************************************************************
- * Module descriptor
- *****************************************************************************/
 vlc_module_begin();
     set_description( N_("Video filtering using a chain of video filter modules") );
     set_capability( "video filter2", 1 );
     set_callbacks( Activate, Destroy );
 vlc_module_end();
 
-#define MAX_FILTERS 4
+/*****************************************************************************
+ * Local prototypes.
+ *****************************************************************************/
+static picture_t *Chain         ( filter_t *, picture_t * );
+static int BufferAllocationInit ( filter_t *, void * );
 
-struct filter_sys_t
-{
-    filter_chain_t *p_chain;
-};
+static int BuildChromaResize( filter_t * );
+static int BuildChromaChain( filter_t *p_filter );
+
+static int CreateChain( filter_chain_t *p_chain, es_format_t *p_fmt_mid );
+static void EsFormatMergeSize( es_format_t *p_dst,
+                               const es_format_t *p_base,
+                               const es_format_t *p_size );
 
 static const vlc_fourcc_t pi_allowed_chromas[] = {
     VLC_FOURCC('I','4','2','0'),
@@ -65,28 +67,12 @@ static const vlc_fourcc_t pi_allowed_chromas[] = {
     0
 };
 
-static int CreateChain( filter_chain_t *p_chain, es_format_t *p_fmt_mid )
+struct filter_sys_t
 {
-    filter_t *p_filter1;
-    if( !( p_filter1 =
-           filter_chain_AppendFilter( p_chain, NULL, NULL, NULL, p_fmt_mid )) )
-        return VLC_EGENERIC;
-    if( !filter_chain_AppendFilter( p_chain, NULL, NULL, p_fmt_mid, NULL ) )
-    {
-        filter_chain_DeleteFilter( p_chain, p_filter1 );
-        return VLC_EGENERIC;
-    }
-    return VLC_SUCCESS;
-}
+    filter_chain_t *p_chain;
+};
 
-static int AllocInit( filter_t *p_filter, void *p_data )
-{
-    /* Not sure about all of this ... it should work */
-    p_filter->pf_vout_buffer_new = ((filter_t*)p_data)->pf_vout_buffer_new;
-    p_filter->pf_vout_buffer_del = ((filter_t*)p_data)->pf_vout_buffer_del;
-    p_filter->p_owner = ((filter_t*)p_data)->p_owner;
-    return VLC_SUCCESS;
-}
+#define CHAIN_LEVEL_MAX 4
 
 /*****************************************************************************
  * Activate: allocate a chroma function
@@ -96,30 +82,24 @@ static int AllocInit( filter_t *p_filter, void *p_data )
 static int Activate( vlc_object_t *p_this )
 {
     filter_t *p_filter = (filter_t *)p_this;
-    static int hack = 0; /* FIXME */
-    es_format_t fmt_mid;
+    filter_sys_t *p_sys;
+    int i_ret;
 
-    if( p_filter->fmt_in.video.i_chroma == p_filter->fmt_out.video.i_chroma )
-        return VLC_EGENERIC;
+    const bool b_chroma = p_filter->fmt_in.video.i_chroma != p_filter->fmt_out.video.i_chroma;
+    const bool b_resize = p_filter->fmt_in.video.i_width  != p_filter->fmt_out.video.i_width ||
+                          p_filter->fmt_in.video.i_height != p_filter->fmt_out.video.i_height;
 
-    hack++;
-    if( hack >= MAX_FILTERS )
-    {
-        msg_Err( p_this, "Preventing chain filter reccursion (already %d long)",
-                 hack );
+    /* XXX Remove check on b_resize to build chroma chain (untested) */
+    if( !b_chroma || !b_resize )
         return VLC_EGENERIC;
-    }
 
-    filter_sys_t *p_sys = (filter_sys_t *)malloc( sizeof( filter_sys_t ) );
+    p_sys = p_filter->p_sys = malloc( sizeof( *p_sys ) );
     if( !p_sys )
-    {
-        hack--;
         return VLC_ENOMEM;
-    }
-    memset( p_sys, 0, sizeof( filter_sys_t ) );
-    p_filter->p_sys = p_sys;
 
-    p_sys->p_chain = filter_chain_New( p_filter, "video filter2", false, AllocInit, NULL, p_filter );
+    memset( p_sys, 0, sizeof( *p_sys ) );
+
+    p_sys->p_chain = filter_chain_New( p_filter, "video filter2", false, BufferAllocationInit, NULL, p_filter );
     if( !p_sys->p_chain )
     {
         free( p_sys );
@@ -127,68 +107,162 @@ static int Activate( vlc_object_t *p_this )
     }
     filter_chain_Reset( p_sys->p_chain, &p_filter->fmt_in, &p_filter->fmt_out );
 
-    if( p_filter->fmt_in.video.i_width != p_filter->fmt_out.video.i_width ||
-        p_filter->fmt_in.video.i_height != p_filter->fmt_out.video.i_height ||
-        p_filter->fmt_in.video.i_visible_width != p_filter->fmt_out.video.i_visible_width ||
-        p_filter->fmt_in.video.i_visible_height != p_filter->fmt_out.video.i_visible_height )
+    if( b_chroma && b_resize )
+        i_ret = BuildChromaResize( p_filter );
+    else if( b_chroma )
+        i_ret = BuildChromaChain( p_filter );
+    else
+        i_ret = VLC_EGENERIC;
+
+    if( i_ret )
     {
+        /* Hum ... looks like this really isn't going to work. Too bad. */
+        filter_chain_Delete( p_sys->p_chain );
+        free( p_sys );
+        return VLC_EGENERIC;
+    }
+    /* */
+    p_filter->pf_video_filter = Chain;
+    return VLC_SUCCESS;
+}
+
+static void Destroy( vlc_object_t *p_this )
+{
+    filter_t *p_filter = (filter_t *)p_this;
+    filter_chain_Delete( p_filter->p_sys->p_chain );
+    free( p_filter->p_sys );
+}
+
+/*****************************************************************************
+ * Chain
+ *****************************************************************************/
+static picture_t *Chain( filter_t *p_filter, picture_t *p_pic )
+{
+    return filter_chain_VideoFilter( p_filter->p_sys->p_chain, p_pic );
+}
+
+/*****************************************************************************
+ * Builders
+ *****************************************************************************/
+static int BuildChromaResize( filter_t *p_filter )
+{
+    filter_sys_t *p_sys = p_filter->p_sys;
+    es_format_t fmt_mid;
+    int i_ret;
+
     /* Lets try resizing and then doing the chroma conversion */
-        es_format_Copy( &fmt_mid, &p_filter->fmt_out );
-        fmt_mid.video.i_chroma = p_filter->fmt_in.video.i_chroma;
-        if( CreateChain( p_sys->p_chain, &fmt_mid ) == VLC_SUCCESS )
-        {
+    msg_Dbg( p_filter, "Trying to build resize+chroma" );
+    EsFormatMergeSize( &fmt_mid, &p_filter->fmt_in, &p_filter->fmt_out );
+    i_ret = CreateChain( p_sys->p_chain, &fmt_mid );
     es_format_Clean( &fmt_mid );
-            p_filter->pf_video_filter = Chain;
+    if( i_ret == VLC_SUCCESS )
         return VLC_SUCCESS;
-        }
 
     /* Lets try it the other way arround (chroma and then resize) */
+    msg_Dbg( p_filter, "Trying to build chroma+resize" );
+    EsFormatMergeSize( &fmt_mid, &p_filter->fmt_out, &p_filter->fmt_in );
+    i_ret = CreateChain( p_sys->p_chain, &fmt_mid );
     es_format_Clean( &fmt_mid );
-        es_format_Copy( &fmt_mid, &p_filter->fmt_in );
-        fmt_mid.video.i_chroma = p_filter->fmt_out.video.i_chroma;
-        if( CreateChain( p_sys->p_chain, &fmt_mid ) == VLC_SUCCESS )
-        {
-            es_format_Clean( &fmt_mid );
-            p_filter->pf_video_filter = Chain;
+    if( i_ret == VLC_SUCCESS )
         return VLC_SUCCESS;
+
+    return VLC_EGENERIC;
+}
+
+static int BuildChromaChain( filter_t *p_filter )
+{
+    filter_sys_t *p_sys = p_filter->p_sys;
+    es_format_t fmt_mid;
+    int i_ret;
+    int i;
+
+    /* We have to protect ourself against a too high recursion */
+    const char *psz_option = MODULE_STRING"-level";
+    bool b_first = !var_Type( p_filter, psz_option );
+
+    if( var_Create( p_filter, MODULE_STRING"-level", VLC_VAR_INTEGER | (b_first ? VLC_VAR_DOINHERIT : 0 ) ) )
+    {
+        msg_Err( p_filter, "Failed to create %s variable\n", psz_option );
+        return VLC_EGENERIC;
     }
-    }
-    else
+    int i_level = var_GetInteger( p_filter, psz_option );
+    if( i_level >= CHAIN_LEVEL_MAX )
     {
-        /* Lets try doing a chroma chain */
-        int i;
-        es_format_Copy( &fmt_mid, &p_filter->fmt_in );
+        msg_Err( p_filter, "Too high level of recursion (%d)\n", i_level );
+        return VLC_EGENERIC;
+    }
+    var_SetInteger( p_filter, psz_option, i_level + 1 );
+
+    /* Now try chroma format list */
     for( i = 0; pi_allowed_chromas[i]; i++ )
     {
-            fmt_mid.video.i_chroma = pi_allowed_chromas[i];
-            if( CreateChain( p_sys->p_chain, &fmt_mid ) == VLC_SUCCESS )
-            {
+        const vlc_fourcc_t i_chroma = pi_allowed_chromas[i];
+
+        msg_Dbg( p_filter, "Trying to use chroma %4.4s as middle man",
+                 (char*)&i_chroma );
+
+        es_format_Copy( &fmt_mid, &p_filter->fmt_in );
+        fmt_mid.video.i_chroma = i_chroma;
+
+        i_ret = CreateChain( p_sys->p_chain, &fmt_mid );
         es_format_Clean( &fmt_mid );
-                p_filter->pf_video_filter = Chain;
+
+        if( i_ret == VLC_SUCCESS )
             return VLC_SUCCESS;
     }
-        }
-    }
-
-    /* Hum ... looks like this really isn't going to work. Too bad. */
-    es_format_Clean( &fmt_mid );
-    filter_chain_Delete( p_sys->p_chain );
-    free( p_sys );
-    hack--;
     return VLC_EGENERIC;
 }
 
-static void Destroy( vlc_object_t *p_this )
+/*****************************************************************************
+ * Buffer management
+ *****************************************************************************/
+static picture_t *BufferNew( filter_t *p_filter )
 {
-    filter_t *p_filter = (filter_t *)p_this;
-    filter_chain_Delete( p_filter->p_sys->p_chain );
-    free( p_filter->p_sys );
+    filter_t *p_parent = (filter_t*)p_filter->p_owner;
+
+    return p_parent->pf_vout_buffer_new( p_parent );
+}
+static void BufferDel( filter_t *p_filter, picture_t *p_pic )
+{
+    filter_t *p_parent = (filter_t*)p_filter->p_owner;
+
+    p_parent->pf_vout_buffer_del( p_parent, p_pic );
+}
+static int BufferAllocationInit ( filter_t *p_filter, void *p_data )
+{
+    p_filter->pf_vout_buffer_new = BufferNew;
+    p_filter->pf_vout_buffer_del = BufferDel;
+    p_filter->p_owner = p_data;
+    return VLC_SUCCESS;
 }
 
 /*****************************************************************************
- * Chain
+ *
  *****************************************************************************/
-static picture_t *Chain( filter_t *p_filter, picture_t *p_pic )
+static int CreateChain( filter_chain_t *p_chain, es_format_t *p_fmt_mid )
 {
-    return filter_chain_VideoFilter( p_filter->p_sys->p_chain, p_pic );
+    filter_t *p_filter1;
+    if( !( p_filter1 =
+           filter_chain_AppendFilter( p_chain, NULL, NULL, NULL, p_fmt_mid )) )
+        return VLC_EGENERIC;
+    if( !filter_chain_AppendFilter( p_chain, NULL, NULL, p_fmt_mid, NULL ) )
+    {
+        filter_chain_DeleteFilter( p_chain, p_filter1 );
+        return VLC_EGENERIC;
+    }
+    return VLC_SUCCESS;
+}
+
+static void EsFormatMergeSize( es_format_t *p_dst,
+                               const es_format_t *p_base,
+                               const es_format_t *p_size )
+{
+    es_format_Copy( p_dst, p_base );
+
+    p_dst->video.i_width  = p_size->video.i_width;
+    p_dst->video.i_height = p_size->video.i_height;
+
+    p_dst->video.i_visible_width  = p_size->video.i_visible_width;
+    p_dst->video.i_visible_height = p_size->video.i_visible_height;
 }
+