Avoid potential integer overflow when computing AR when transcoding.

It was introduced by [99e262c4].
(cherry picked from commit 0c0c1a06)
Signed-off-by: Jean-Baptiste Kempf <jb@videolan.org>

modules/stream_out/transcode/video.c

@@ -493,20 +493,20 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     if( id->p_encoder->fmt_out.video.i_sar_num <= 0 ||
         id->p_encoder->fmt_out.video.i_sar_den <= 0 )
     {
-        vlc_ureduce( &id->p_decoder->fmt_out.video.i_sar_num,
-                     &id->p_decoder->fmt_out.video.i_sar_den,
-                     id->p_decoder->fmt_out.video.i_sar_num,
-                     id->p_decoder->fmt_out.video.i_sar_den,
+        vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
+                     &id->p_encoder->fmt_out.video.i_sar_den,
+                     (uint64_t)id->p_decoder->fmt_out.video.i_sar_num * i_src_width  * i_dst_height,
+                     (uint64_t)id->p_decoder->fmt_out.video.i_sar_den * i_src_height * i_dst_width,
+                     0 );
+    }
+    else
+    {
+        vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
+                     &id->p_encoder->fmt_out.video.i_sar_den,
+                     id->p_encoder->fmt_out.video.i_sar_num,
+                     id->p_encoder->fmt_out.video.i_sar_den,
                      0 );
-
-        id->p_encoder->fmt_out.video.i_sar_num = id->p_decoder->fmt_out.video.i_sar_num * i_src_width  * i_dst_height;
-        id->p_encoder->fmt_out.video.i_sar_den = id->p_decoder->fmt_out.video.i_sar_den * i_src_height * i_dst_width;
     }
-    vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
-                 &id->p_encoder->fmt_out.video.i_sar_den,
-                 id->p_encoder->fmt_out.video.i_sar_num,
-                 id->p_encoder->fmt_out.video.i_sar_den,
-                 0 );
 
     id->p_encoder->fmt_in.video.i_sar_num =
         id->p_encoder->fmt_out.video.i_sar_num;