decoder: separate vout initialization from buffer allocation

Signed-off-by: Rémi Denis-Courmont <remi@remlab.net>

include/vlc_codec.h

@@ -97,6 +97,7 @@ struct decoder_t
     /* Video output callbacks
      * XXX use decoder_NewPicture/decoder_DeletePicture
      * and decoder_LinkPicture/decoder_UnlinkPicture */
+    int             (*pf_vout_format_update)( decoder_t * );
     picture_t      *(*pf_vout_buffer_new)( decoder_t * );
     void            (*pf_vout_buffer_del)( decoder_t *, picture_t * );
     void            (*pf_picture_link)   ( decoder_t *, picture_t * );
@@ -179,6 +180,19 @@ struct encoder_t
  */
 
 
+/**
+ * This function notifies the video output pipeline of a new video output
+ * format (fmt_out.video). If there is currently no video output or if the
+ * video output format has changed, a new audio video will be set up.
+ * @return 0 if the video output is working, -1 if not. */
+static inline int decoder_UpdateVideoFormat( decoder_t *dec )
+{
+    if( dec->pf_vout_format_update != NULL )
+        return dec->pf_vout_format_update( dec );
+    else
+        return -1;
+}
+
 /**
  * This function will return a new picture usable by a decoder as an output
  * buffer. You have to release it using decoder_DeletePicture or by returning

modules/stream_out/mosaic_bridge.c

@@ -80,10 +80,11 @@ static int               Send( sout_stream_t *, sout_stream_id_sys_t *, block_t
 inline static void video_del_buffer_decoder( decoder_t *, picture_t * );
 inline static void video_del_buffer_filter( filter_t *, picture_t * );
 
+inline static int video_update_format_decoder( decoder_t *p_dec );
 inline static picture_t *video_new_buffer_decoder( decoder_t * );
 inline static picture_t *video_new_buffer_filter( filter_t * );
-static picture_t *video_new_buffer( vlc_object_t *, decoder_owner_sys_t *,
-                                    es_format_t * );
+static int video_update_format( vlc_object_t *, decoder_owner_sys_t *,
+                                es_format_t * );
 
 static void video_link_picture_decoder( decoder_t *, picture_t * );
 static void video_unlink_picture_decoder( decoder_t *, picture_t * );
@@ -296,6 +297,7 @@ static sout_stream_id_sys_t * Add( sout_stream_t *p_stream, es_format_t *p_fmt )
     p_sys->p_decoder->fmt_out.i_extra = 0;
     p_sys->p_decoder->fmt_out.p_extra = 0;
     p_sys->p_decoder->pf_decode_video = 0;
+    p_sys->p_decoder->pf_vout_format_update = video_update_format_decoder;
     p_sys->p_decoder->pf_vout_buffer_new = video_new_buffer_decoder;
     p_sys->p_decoder->pf_vout_buffer_del = video_del_buffer_decoder;
     p_sys->p_decoder->pf_picture_link    = video_link_picture_decoder;
@@ -602,21 +604,30 @@ static int Send( sout_stream_t *p_stream, sout_stream_id_sys_t *id,
     return VLC_SUCCESS;
 }
 
+inline static int video_update_format_decoder( decoder_t *p_dec )
+{
+    return video_update_format( VLC_OBJECT( p_dec ),
+                                (decoder_owner_sys_t *)p_dec->p_owner,
+                                &p_dec->fmt_out );
+}
+
 inline static picture_t *video_new_buffer_decoder( decoder_t *p_dec )
 {
-    return video_new_buffer( VLC_OBJECT( p_dec ),
-                             (decoder_owner_sys_t *)p_dec->p_owner,
-                             &p_dec->fmt_out );
+    return picture_NewFromFormat( &p_dec->fmt_out.video );
 }
 
 inline static picture_t *video_new_buffer_filter( filter_t *p_filter )
 {
-    return video_new_buffer( VLC_OBJECT( p_filter ),
+    if( video_update_format( VLC_OBJECT( p_filter ),
                              (decoder_owner_sys_t *)p_filter->owner.sys,
-                             &p_filter->fmt_out );
+                             &p_filter->fmt_out ) ) {
+        msg_Warn( p_filter, "can't get output picture" );
+        return NULL;
+    }
+    return picture_NewFromFormat( &p_filter->fmt_out.video );
 }
 
-static picture_t *video_new_buffer( vlc_object_t *p_this,
+static int video_update_format( vlc_object_t *p_this,
                                     decoder_owner_sys_t *p_sys,
                                     es_format_t *fmt_out )
 {
@@ -645,8 +656,7 @@ static picture_t *video_new_buffer( vlc_object_t *p_this,
 
     /* */
     fmt_out->video.i_chroma = fmt_out->i_codec;
-
-    return picture_NewFromFormat( &fmt_out->video );
+    return 0;
 }
 
 inline static void video_del_buffer_decoder( decoder_t *p_this,

modules/stream_out/transcode/video.c

@@ -61,9 +61,14 @@ static void video_unlink_picture_decoder( decoder_t *p_dec, picture_t *p_pic )
     picture_Release( p_pic );
 }
 
-static picture_t *video_new_buffer_decoder( decoder_t *p_dec )
+static int video_update_format_decoder( decoder_t *p_dec )
 {
     p_dec->fmt_out.video.i_chroma = p_dec->fmt_out.i_codec;
+    return 0;
+}
+
+static picture_t *video_new_buffer_decoder( decoder_t *p_dec )
+{
     return picture_NewFromFormat( &p_dec->fmt_out.video );
 }
 
@@ -165,6 +170,7 @@ int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_sys_t *id )
     id->p_decoder->pf_decode_video = NULL;
     id->p_decoder->pf_get_cc = NULL;
     id->p_decoder->pf_get_cc = 0;
+    id->p_decoder->pf_vout_format_update = video_update_format_decoder;
     id->p_decoder->pf_vout_buffer_new = video_new_buffer_decoder;
     id->p_decoder->pf_vout_buffer_del = video_del_buffer_decoder;
     id->p_decoder->pf_picture_link    = video_link_picture_decoder;

src/input/decoder.c

@@ -66,6 +66,7 @@ static void       DecoderSignalWait( decoder_t *, bool );
 static void       DecoderUnsupportedCodec( decoder_t *, vlc_fourcc_t );
 
 /* Buffers allocation callbacks for the decoders */
+static int vout_update_format( decoder_t * );
 static picture_t *vout_new_buffer( decoder_t * );
 static void vout_del_buffer( decoder_t *, picture_t * );
 static void vout_link_picture( decoder_t *, picture_t * );
@@ -160,6 +161,9 @@ struct decoder_owner_sys_t
  *****************************************************************************/
 picture_t *decoder_NewPicture( decoder_t *p_decoder )
 {
+    if( decoder_UpdateVideoFormat( p_decoder ) )
+        return NULL;
+
     picture_t *p_picture = p_decoder->pf_vout_buffer_new( p_decoder );
     if( !p_picture )
         msg_Warn( p_decoder, "can't get output picture" );
@@ -778,6 +782,7 @@ static decoder_t * CreateDecoder( vlc_object_t *p_parent,
 
     /* Set buffers allocation callbacks for the decoders */
     p_dec->pf_aout_format_update = aout_update_format;
+    p_dec->pf_vout_format_update = vout_update_format;
     p_dec->pf_vout_buffer_new = vout_new_buffer;
     p_dec->pf_vout_buffer_del = vout_del_buffer;
     p_dec->pf_picture_link    = vout_link_picture;
@@ -2027,7 +2032,7 @@ static int aout_update_format( decoder_t *p_dec )
     return 0;
 }
 
-static picture_t *vout_new_buffer( decoder_t *p_dec )
+static int vout_update_format( decoder_t *p_dec )
 {
     decoder_owner_sys_t *p_owner = p_dec->p_owner;
 
@@ -2049,7 +2054,7 @@ static picture_t *vout_new_buffer( decoder_t *p_dec )
             !p_dec->fmt_out.video.i_height )
         {
             /* Can't create a new vout without display size */
-            return NULL;
+            return -1;
         }
 
         video_format_t fmt = p_dec->fmt_out.video;
@@ -2153,12 +2158,16 @@ static picture_t *vout_new_buffer( decoder_t *p_dec )
         {
             msg_Err( p_dec, "failed to create video output" );
             p_dec->b_error = true;
-            return NULL;
+            return -1;
         }
     }
+    return 0;
+}
+
+static picture_t *vout_new_buffer( decoder_t *p_dec )
+{
+    decoder_owner_sys_t *p_owner = p_dec->p_owner;
 
-    /* Get a new picture
-     */
     for( ;; )
     {
         if( DecoderIsExitRequested( p_dec ) || p_dec->b_error )

src/misc/image.c

@@ -585,10 +585,14 @@ vlc_fourcc_t image_Mime2Fourcc( const char *psz_mime )
     return 0;
 }
 
+static int video_update_format( decoder_t *p_dec )
+{
+    p_dec->fmt_out.video.i_chroma = p_dec->fmt_out.i_codec;
+    return 0;
+}
 
 static picture_t *video_new_buffer( decoder_t *p_dec )
 {
-    p_dec->fmt_out.video.i_chroma = p_dec->fmt_out.i_codec;
     return picture_NewFromFormat( &p_dec->fmt_out.video );
 }
 
@@ -624,6 +628,7 @@ static decoder_t *CreateDecoder( vlc_object_t *p_this, video_format_t *fmt )
     p_dec->fmt_in.video = *fmt;
     p_dec->b_pace_control = true;
 
+    p_dec->pf_vout_format_update = video_update_format;
     p_dec->pf_vout_buffer_new = video_new_buffer;
     p_dec->pf_vout_buffer_del = video_del_buffer;
     p_dec->pf_picture_link    = video_link_picture;