Real: fix heap buffer overflow (CVE-2010-3907)

Malformatted files may have a zero i_subpackets value. In this case,
we cannot use the array, but we still have to free it (calloc(0)).

This should fix LP#690173.
Reported-by: Dan Rosenberg <drosenberg@vsecurity.com>

modules/demux/real.c

@@ -251,11 +251,8 @@ static void Close( vlc_object_t *p_this )
             if( tk->p_subpackets[ j ] )
                 block_Release( tk->p_subpackets[ j ] );
         }
-        if( tk->i_subpackets )
-        {
         free( tk->p_subpackets );
         free( tk->p_subpackets_timecode );
-        }
         if( tk->p_sipr_packet )
             block_Release( tk->p_sipr_packet );
         free( tk );
@@ -636,6 +633,11 @@ static void DemuxAudioMethod1( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
 
         for( int i = 0; i < i_num; i++ )
         {
+            int i_index = tk->i_subpacket_h * i +
+                          ((tk->i_subpacket_h + 1) / 2) * (y&1) + (y>>1);
+            if( i_index >= tk->i_subpackets )
+                return;
+
             block_t *p_block = block_New( p_demux, tk->i_subpacket_size );
             if( !p_block )
                 return;
@@ -648,9 +650,6 @@ static void DemuxAudioMethod1( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
 
             p_buf += tk->i_subpacket_size;
 
-            int i_index = tk->i_subpacket_h * i +
-                          ((tk->i_subpacket_h + 1) / 2) * (y&1) + (y>>1);
-
             if( tk->p_subpackets[i_index] != NULL )
             {
                 msg_Dbg(p_demux, "p_subpackets[ %d ] not null!",  i_index );
@@ -670,14 +669,16 @@ static void DemuxAudioMethod1( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
 
         for( int i = 0; i < tk->i_subpacket_h / 2; i++ )
         {
+            int i_index = (i * 2 * tk->i_frame_size / tk->i_coded_frame_size) + y;
+            if( i_index >= tk->i_subpackets )
+                return;
+
             block_t *p_block = block_New( p_demux, tk->i_coded_frame_size);
             if( !p_block )
                 return;
             if( &p_buf[tk->i_coded_frame_size] > &p_sys->buffer[p_sys->i_buffer] )
                 return;
 
-            int i_index = (i * 2 * tk->i_frame_size / tk->i_coded_frame_size) + y;
-
             memcpy( p_block->p_buffer, p_buf, tk->i_coded_frame_size );
             p_block->i_dts =
             p_block->i_pts = i_index == 0 ? i_pts : VLC_TS_INVALID;