In video transcode, use the correct format for calculations

The current code doesn't properly take into account that filters
can change the format (including the frame rate) from that of the
decoder's output, and thus uncritically use the decoder's output
format where it should use the output of the decoder -> filter
-> user_filter chain.
Signed-off-by: Rémi Denis-Courmont <remi@remlab.net>

modules/stream_out/transcode/video.c

@@ -270,25 +270,24 @@ static void transcode_video_filter_init( sout_stream_t *p_stream,
     /* Deinterlace */
     if( p_stream->p_sys->b_deinterlace )
     {
-       filter_chain_AppendFilter( id->p_f_chain,
-                                  p_stream->p_sys->psz_deinterlace,
-                                  p_stream->p_sys->p_deinterlace_cfg,
-                                  &id->p_decoder->fmt_out,
-                                  &id->p_decoder->fmt_out );
+        filter_chain_AppendFilter( id->p_f_chain,
+                                   p_stream->p_sys->psz_deinterlace,
+                                   p_stream->p_sys->p_deinterlace_cfg,
+                                   &id->p_decoder->fmt_out,
+                                   &id->p_decoder->fmt_out );
+
+        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
     }
 
     /* Take care of the scaling and chroma conversions */
-    if( ( id->p_decoder->fmt_out.video.i_chroma !=
-          id->p_encoder->fmt_in.video.i_chroma ) ||
-        ( id->p_decoder->fmt_out.video.i_width !=
-          id->p_encoder->fmt_in.video.i_width ) ||
-        ( id->p_decoder->fmt_out.video.i_height !=
-          id->p_encoder->fmt_in.video.i_height ) )
+    if( ( p_fmt_out->video.i_chroma != id->p_encoder->fmt_in.video.i_chroma ) ||
+        ( p_fmt_out->video.i_width != id->p_encoder->fmt_in.video.i_width ) ||
+        ( p_fmt_out->video.i_height != id->p_encoder->fmt_in.video.i_height ) )
     {
-       filter_chain_AppendFilter( id->p_f_chain,
-                                  NULL, NULL,
-                                  &id->p_decoder->fmt_out,
-                                  &id->p_encoder->fmt_in );
+        filter_chain_AppendFilter( id->p_f_chain,
+                                   NULL, NULL,
+                                   p_fmt_out,
+                                   &id->p_encoder->fmt_in );
     }
 
     if( p_stream->p_sys->psz_vf2 )
@@ -321,10 +320,18 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
 {
     sout_stream_sys_t *p_sys = p_stream->p_sys;
 
+    const es_format_t *p_fmt_out = &id->p_decoder->fmt_out;
+    if( id->p_f_chain ) {
+        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
+    }
+    if( id->p_uf_chain ) {
+        p_fmt_out = filter_chain_GetFmtOut( id->p_uf_chain );
+    }
+
     /* Calculate scaling
      * width/height of source */
-    int i_src_width = id->p_decoder->fmt_out.video.i_width;
-    int i_src_height = id->p_decoder->fmt_out.video.i_height;
+    int i_src_width = p_fmt_out->video.i_width;
+    int i_src_height = p_fmt_out->video.i_height;
 
     /* with/height scaling */
     float f_scale_width = 1;
@@ -335,10 +342,10 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     int i_dst_height;
 
     /* aspect ratio */
-    float f_aspect = (double)id->p_decoder->fmt_out.video.i_sar_num *
-                     id->p_decoder->fmt_out.video.i_width /
-                     id->p_decoder->fmt_out.video.i_sar_den /
-                     id->p_decoder->fmt_out.video.i_height;
+    float f_aspect = (double)p_fmt_out->video.i_sar_num *
+                     p_fmt_out->video.i_width /
+                     p_fmt_out->video.i_sar_den /
+                     p_fmt_out->video.i_height;
 
     msg_Dbg( p_stream, "decoder aspect is %f:1", f_aspect );
 
@@ -437,13 +444,13 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     if( !id->p_encoder->fmt_out.video.i_frame_rate ||
         !id->p_encoder->fmt_out.video.i_frame_rate_base )
     {
-        if( id->p_decoder->fmt_out.video.i_frame_rate &&
-            id->p_decoder->fmt_out.video.i_frame_rate_base )
+        if( p_fmt_out->video.i_frame_rate &&
+            p_fmt_out->video.i_frame_rate_base )
         {
             id->p_encoder->fmt_out.video.i_frame_rate =
-                id->p_decoder->fmt_out.video.i_frame_rate;
+                p_fmt_out->video.i_frame_rate;
             id->p_encoder->fmt_out.video.i_frame_rate_base =
-                id->p_decoder->fmt_out.video.i_frame_rate_base;
+                p_fmt_out->video.i_frame_rate_base;
         }
         else
         {
@@ -468,8 +475,8 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     {
         vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
                      &id->p_encoder->fmt_out.video.i_sar_den,
-                     (uint64_t)id->p_decoder->fmt_out.video.i_sar_num * i_src_width  * i_dst_height,
-                     (uint64_t)id->p_decoder->fmt_out.video.i_sar_den * i_src_height * i_dst_width,
+                     (uint64_t)p_fmt_out->video.i_sar_num * i_src_width  * i_dst_height,
+                     (uint64_t)p_fmt_out->video.i_sar_den * i_src_height * i_dst_width,
                      0 );
     }
     else