Fixed segfault with invalid vout input aspect ratio.

src/video_output/video_output.c

@@ -227,6 +227,14 @@ vout_thread_t * __vout_Create( vlc_object_t *p_parent, video_format_t *p_fmt )
     char *psz_parser;
     char *psz_name;
 
+    if( i_width <= 0 || i_height <= 0 || i_aspect <= 0 )
+        return NULL;
+
+    vlc_ureduce( &p_fmt->i_sar_num, &p_fmt->i_sar_den,
+                 p_fmt->i_sar_num, p_fmt->i_sar_den, 50000 );
+    if( p_fmt->i_sar_num <= 0 || p_fmt->i_sar_den <= 0 )
+        return NULL;
+
     /* Allocate descriptor */
     static const char typename[] = "video output";
     p_vout = vlc_custom_create( p_parent, sizeof( *p_vout ), VLC_OBJECT_VOUT,
@@ -251,8 +259,6 @@ vout_thread_t * __vout_Create( vlc_object_t *p_parent, video_format_t *p_fmt )
     /* Initialize the rendering heap */
     I_RENDERPICTURES = 0;
 
-    vlc_ureduce( &p_fmt->i_sar_num, &p_fmt->i_sar_den,
-                 p_fmt->i_sar_num, p_fmt->i_sar_den, 50000 );
     p_vout->fmt_render        = *p_fmt;   /* FIXME palette */
     p_vout->fmt_in            = *p_fmt;   /* FIXME palette */
 

src/video_output/vout_pictures.c

@@ -471,20 +471,22 @@ void vout_PlacePicture( vout_thread_t *p_vout,
         *pi_height = __MIN( i_height, p_vout->fmt_in.i_visible_height );
     }
 
-    if( p_vout->fmt_in.i_visible_width * (int64_t)p_vout->fmt_in.i_sar_num *
-        *pi_height / p_vout->fmt_in.i_visible_height /
-        p_vout->fmt_in.i_sar_den > *pi_width )
+     int64_t i_scaled_width = p_vout->fmt_in.i_visible_width * (int64_t)p_vout->fmt_in.i_sar_num *
+                              *pi_height / p_vout->fmt_in.i_visible_height / p_vout->fmt_in.i_sar_den;
+     int64_t i_scaled_height = p_vout->fmt_in.i_visible_height * (int64_t)p_vout->fmt_in.i_sar_den *
+                               *pi_width / p_vout->fmt_in.i_visible_width / p_vout->fmt_in.i_sar_num;
+
+    if( i_scaled_width <= 0 || i_scaled_height <= 0 )
     {
-        *pi_height = p_vout->fmt_in.i_visible_height *
-            (int64_t)p_vout->fmt_in.i_sar_den * *pi_width /
-            p_vout->fmt_in.i_visible_width / p_vout->fmt_in.i_sar_num;
+        msg_Warn( p_vout, "ignoring broken aspect ratio" );
+        i_scaled_width = *pi_width;
+        i_scaled_height = *pi_height;
     }
+
+    if( i_scaled_width > *pi_width )
+        *pi_height = i_scaled_height;
     else
-    {
-        *pi_width = p_vout->fmt_in.i_visible_width *
-            (int64_t)p_vout->fmt_in.i_sar_num * *pi_height /
-            p_vout->fmt_in.i_visible_height / p_vout->fmt_in.i_sar_den;
-    }
+        *pi_width = i_scaled_width;
 
     switch( p_vout->i_alignment & VOUT_ALIGN_HMASK )
     {