From 52b77526210ff7b1aeb9f30b4ab023e393ffdcbd Mon Sep 17 00:00:00 2001
From: Laurent Aimar <fenrir@videolan.org>
Date: Wed, 27 Aug 2008 13:31:26 +0200
Subject: [PATCH] Fixed segfault with invalid vout input aspect ratio.
---
src/video_output/video_output.c | 10 ++++++++--
src/video_output/vout_pictures.c | 24 +++++++++++++-----------
2 files changed, 21 insertions(+), 13 deletions(-)
diff --git a/src/video_output/video_output.c b/src/video_output/video_output.c
index a68d7fdae4..d2d4719fe7 100644
--- a/src/video_output/video_output.c
+++ b/src/video_output/video_output.c
@@ -227,6 +227,14 @@ vout_thread_t * __vout_Create( vlc_object_t *p_parent, video_format_t *p_fmt )
char *psz_parser;
char *psz_name;
+ if( i_width <= 0 || i_height <= 0 || i_aspect <= 0 )
+ return NULL;
+
+ vlc_ureduce( &p_fmt->i_sar_num, &p_fmt->i_sar_den,
+ p_fmt->i_sar_num, p_fmt->i_sar_den, 50000 );
+ if( p_fmt->i_sar_num <= 0 || p_fmt->i_sar_den <= 0 )
+ return NULL;
+
/* Allocate descriptor */
static const char typename[] = "video output";
p_vout = vlc_custom_create( p_parent, sizeof( *p_vout ), VLC_OBJECT_VOUT,
@@ -251,8 +259,6 @@ vout_thread_t * __vout_Create( vlc_object_t *p_parent, video_format_t *p_fmt )
/* Initialize the rendering heap */
I_RENDERPICTURES = 0;
- vlc_ureduce( &p_fmt->i_sar_num, &p_fmt->i_sar_den,
- p_fmt->i_sar_num, p_fmt->i_sar_den, 50000 );
p_vout->fmt_render = *p_fmt; /* FIXME palette */
p_vout->fmt_in = *p_fmt; /* FIXME palette */
diff --git a/src/video_output/vout_pictures.c b/src/video_output/vout_pictures.c
index 8c63e33b3f..9c620895b8 100644
--- a/src/video_output/vout_pictures.c
+++ b/src/video_output/vout_pictures.c
@@ -471,20 +471,22 @@ void vout_PlacePicture( vout_thread_t *p_vout,
*pi_height = __MIN( i_height, p_vout->fmt_in.i_visible_height );
}
- if( p_vout->fmt_in.i_visible_width * (int64_t)p_vout->fmt_in.i_sar_num *
- *pi_height / p_vout->fmt_in.i_visible_height /
- p_vout->fmt_in.i_sar_den > *pi_width )
+ int64_t i_scaled_width = p_vout->fmt_in.i_visible_width * (int64_t)p_vout->fmt_in.i_sar_num *
+ *pi_height / p_vout->fmt_in.i_visible_height / p_vout->fmt_in.i_sar_den;
+ int64_t i_scaled_height = p_vout->fmt_in.i_visible_height * (int64_t)p_vout->fmt_in.i_sar_den *
+ *pi_width / p_vout->fmt_in.i_visible_width / p_vout->fmt_in.i_sar_num;
+
+ if( i_scaled_width <= 0 || i_scaled_height <= 0 )
{
- *pi_height = p_vout->fmt_in.i_visible_height *
- (int64_t)p_vout->fmt_in.i_sar_den * *pi_width /
- p_vout->fmt_in.i_visible_width / p_vout->fmt_in.i_sar_num;
+ msg_Warn( p_vout, "ignoring broken aspect ratio" );
+ i_scaled_width = *pi_width;
+ i_scaled_height = *pi_height;
}
+
+ if( i_scaled_width > *pi_width )
+ *pi_height = i_scaled_height;
else
- {
- *pi_width = p_vout->fmt_in.i_visible_width *
- (int64_t)p_vout->fmt_in.i_sar_num * *pi_height /
- p_vout->fmt_in.i_visible_height / p_vout->fmt_in.i_sar_den;
- }
+ *pi_width = i_scaled_width;
switch( p_vout->i_alignment & VOUT_ALIGN_HMASK )
{
--
2.25.4