canvas: use visible width/height on calculations

modules/video_filter/canvas.c

@@ -138,8 +138,8 @@ struct filter_sys_t
 static int Activate( vlc_object_t *p_this )
 {
     filter_t *p_filter = (filter_t *)p_this;
-    unsigned i_canvas_width; /* width of output canvas */
-    unsigned i_canvas_height; /* height of output canvas */
+    unsigned i_canvas_width; /* visible width of output canvas */
+    unsigned i_canvas_height; /* visible height of output canvas */
     unsigned i_canvas_aspect; /* canvas PictureAspectRatio */
     es_format_t fmt; /* target format after up/down conversion */
     char psz_croppadd[100];
@@ -182,14 +182,14 @@ static int Activate( vlc_object_t *p_this )
 
     if( p_filter->fmt_in.video.i_sar_num )
         i_fmt_in_aspect = (int64_t)p_filter->fmt_in.video.i_sar_num *
-                      p_filter->fmt_in.video.i_width *
+                      p_filter->fmt_in.video.i_visible_width *
                       VOUT_ASPECT_FACTOR /
                       p_filter->fmt_in.video.i_sar_den /
-                      p_filter->fmt_in.video.i_height;
+                      p_filter->fmt_in.video.i_visible_height;
     else
-        i_fmt_in_aspect = (int64_t)p_filter->fmt_in.video.i_width *
+        i_fmt_in_aspect = (int64_t)p_filter->fmt_in.video.i_visible_width *
                       VOUT_ASPECT_FACTOR /
-                      p_filter->fmt_in.video.i_height;
+                      p_filter->fmt_in.video.i_visible_height;
 
     psz_aspect = var_CreateGetNonEmptyString( p_filter, CFG_PREFIX "aspect" );
     if( psz_aspect )
@@ -213,10 +213,10 @@ static int Activate( vlc_object_t *p_this )
          * has the same sample aspect ratio as the subpicture */
         /* aspect = subpic_sar * canvas_width / canvas_height
          *  where subpic_sar = subpic_ph * subpic_par / subpic_pw */
-        i_canvas_aspect = (uint64_t) p_filter->fmt_in.video.i_height
+        i_canvas_aspect = (uint64_t) p_filter->fmt_in.video.i_visible_height
                         * i_fmt_in_aspect
                         * i_canvas_width
-                        / (i_canvas_height * p_filter->fmt_in.video.i_width);
+                        / (i_canvas_height * p_filter->fmt_in.video.i_visible_width);
     }
 
     b_padd = var_CreateGetBool( p_filter, CFG_PREFIX "padd" );
@@ -238,8 +238,8 @@ static int Activate( vlc_object_t *p_this )
     es_format_Copy( &fmt, &p_filter->fmt_in );
 
     /* one dimension will end up with one of the following: */
-    fmt.video.i_width = i_canvas_width;
-    fmt.video.i_height = i_canvas_height;
+    fmt.video.i_visible_width = i_canvas_width;
+    fmt.video.i_visible_height = i_canvas_height;
 
     if( b_padd )
     {
@@ -252,12 +252,12 @@ static int Activate( vlc_object_t *p_this )
              * width = upconverted_subpic_height * subpic_par / canvas_sar
              *  where canvas_sar = canvas_width / (canvas_height * canvas_par)
              * then simplify */
-            fmt.video.i_width = i_canvas_width
+            fmt.video.i_visible_width = i_canvas_width
                               * i_fmt_in_aspect
                               / i_canvas_aspect;
-            if( fmt.video.i_width & 1 ) fmt.video.i_width -= 1;
+            if( fmt.video.i_visible_width & 1 ) fmt.video.i_visible_width -= 1;
 
-            i_padd = (i_canvas_width - fmt.video.i_width) / 2;
+            i_padd = (i_canvas_width - fmt.video.i_visible_width) / 2;
             i_offset = (i_padd & 1);
             snprintf( psz_croppadd, 100, "croppadd{paddleft=%d,paddright=%d}",
                       i_padd - i_offset, i_padd + i_offset );
@@ -266,12 +266,12 @@ static int Activate( vlc_object_t *p_this )
         {
             /* The canvas has a taller aspect than the subpicture:
              *  ie, letterbox the [scaled] subpicture */
-            fmt.video.i_height = i_canvas_height
+            fmt.video.i_visible_height = i_canvas_height
                                * i_canvas_aspect
                                / i_fmt_in_aspect;
-            if( fmt.video.i_height & 1 ) fmt.video.i_height -= 1;
+            if( fmt.video.i_visible_height & 1 ) fmt.video.i_visible_height -= 1;
 
-            i_padd = (i_canvas_height - fmt.video.i_height ) / 2;
+            i_padd = (i_canvas_height - fmt.video.i_visible_height ) / 2;
             i_offset = (i_padd & 1);
             snprintf( psz_croppadd, 100, "croppadd{paddtop=%d,paddbottom=%d}",
                       i_padd - i_offset, i_padd + i_offset );
@@ -284,12 +284,12 @@ static int Activate( vlc_object_t *p_this )
         {
             /* The canvas has a narrower aspect than the subpicture:
              *  ie, crop the [scaled] subpicture horizontally */
-            fmt.video.i_width = i_canvas_width
+            fmt.video.i_visible_width = i_canvas_width
                               * i_fmt_in_aspect
                               / i_canvas_aspect;
-            if( fmt.video.i_width & 1 ) fmt.video.i_width -= 1;
+            if( fmt.video.i_visible_width & 1 ) fmt.video.i_visible_width -= 1;
 
-            i_padd = (fmt.video.i_width - i_canvas_width) / 2;
+            i_padd = (fmt.video.i_visible_width - i_canvas_width) / 2;
             i_offset = (i_padd & 1);
             snprintf( psz_croppadd, 100, "croppadd{cropleft=%d,cropright=%d}",
                       i_padd - i_offset, i_padd + i_offset );
@@ -298,12 +298,12 @@ static int Activate( vlc_object_t *p_this )
         {
             /* The canvas has a shorter aspect than the subpicture:
              *  ie, crop the [scaled] subpicture vertically */
-            fmt.video.i_height = i_canvas_height
+            fmt.video.i_visible_height = i_canvas_height
                                * i_canvas_aspect
                                / i_fmt_in_aspect;
-            if( fmt.video.i_height & 1 ) fmt.video.i_height -= 1;
+            if( fmt.video.i_visible_height & 1 ) fmt.video.i_visible_height -= 1;
 
-            i_padd = (fmt.video.i_height - i_canvas_height) / 2;
+            i_padd = (fmt.video.i_visible_height - i_canvas_height) / 2;
             i_offset = (i_padd & 1);
             snprintf( psz_croppadd, 100, "croppadd{croptop=%d,cropbottom=%d}",
                       i_padd - i_offset, i_padd + i_offset );
@@ -314,8 +314,8 @@ static int Activate( vlc_object_t *p_this )
      *  probably not, as some codecs can make use of that information
      *  and it should be a scaled version of the input clean area
      *   -- davidf */
-    fmt.video.i_visible_width = fmt.video.i_width;
-    fmt.video.i_visible_height = fmt.video.i_height;
+    fmt.video.i_width = p_filter->fmt_in.video.i_width * fmt.video.i_visible_width / p_filter->fmt_in.video.i_visible_width;
+    fmt.video.i_height = p_filter->fmt_in.video.i_height * fmt.video.i_visible_height / p_filter->fmt_in.video.i_visible_height;
 
     filter_chain_Reset( p_sys->p_chain, &p_filter->fmt_in, &fmt );
     /* Append scaling module */
@@ -326,18 +326,19 @@ static int Activate( vlc_object_t *p_this )
     fmt = *filter_chain_GetFmtOut( p_sys->p_chain );
     es_format_Copy( &p_filter->fmt_out, &fmt );
 
-    p_filter->fmt_out.video.i_sar_num =
-        i_canvas_aspect    * p_filter->fmt_out.video.i_height;
-    p_filter->fmt_out.video.i_sar_den =
-        VOUT_ASPECT_FACTOR * p_filter->fmt_out.video.i_width;
+    vlc_ureduce( &p_filter->fmt_out.video.i_sar_num,
+        &p_filter->fmt_out.video.i_sar_den,
+        i_canvas_aspect    * p_filter->fmt_out.video.i_visible_height,
+        VOUT_ASPECT_FACTOR * p_filter->fmt_out.video.i_visible_width,
+        0);
 
-    if( p_filter->fmt_out.video.i_width != i_canvas_width
-     || p_filter->fmt_out.video.i_height != i_canvas_height )
+    if( p_filter->fmt_out.video.i_visible_width != i_canvas_width
+     || p_filter->fmt_out.video.i_visible_height != i_canvas_height )
     {
         msg_Warn( p_filter, "Looks like something went wrong. "
                   "Output size is %dx%d while we asked for %dx%d",
-                  p_filter->fmt_out.video.i_width,
-                  p_filter->fmt_out.video.i_height,
+                  p_filter->fmt_out.video.i_visible_width,
+                  p_filter->fmt_out.video.i_visible_height,
                   i_canvas_width, i_canvas_height );
     }