Real: fix heap buffer overflow (CVE-2010-3907)

Malformatted files may have a zero i_subpackets value. In this case,
we cannot use the array, but we still have to free it (calloc(0)).

This should fix LP#690173.
Reported-by: Dan Rosenberg <drosenberg@vsecurity.com>
(cherry picked from commit 6568965770f906d34d4aef83237842a5376adb55)

modules/demux/real.c

@@ -252,11 +252,8 @@ static void Close( vlc_object_t *p_this )
             if( tk->p_subpackets[ j ] )
                 block_Release( tk->p_subpackets[ j ] );
         }
-        if( tk->i_subpackets )
-        {
         free( tk->p_subpackets );
         free( tk->p_subpackets_timecode );
-        }
         if( tk->p_sipr_packet )
             block_Release( tk->p_sipr_packet );
         free( tk );
@@ -637,6 +634,11 @@ static void DemuxAudioMethod1( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
 
         for( int i = 0; i < i_num; i++ )
         {
+            int i_index = tk->i_subpacket_h * i +
+                          ((tk->i_subpacket_h + 1) / 2) * (y&1) + (y>>1);
+            if( i_index >= tk->i_subpackets )
+                return;
+
             block_t *p_block = block_New( p_demux, tk->i_subpacket_size );
             if( !p_block )
                 return;
@@ -649,9 +651,6 @@ static void DemuxAudioMethod1( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
 
             p_buf += tk->i_subpacket_size;
 
-            int i_index = tk->i_subpacket_h * i +
-                          ((tk->i_subpacket_h + 1) / 2) * (y&1) + (y>>1);
-
             if( tk->p_subpackets[i_index] != NULL )
             {
                 msg_Dbg(p_demux, "p_subpackets[ %d ] not null!",  i_index );
@@ -671,14 +670,16 @@ static void DemuxAudioMethod1( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
 
         for( int i = 0; i < tk->i_subpacket_h / 2; i++ )
         {
+            int i_index = (i * 2 * tk->i_frame_size / tk->i_coded_frame_size) + y;
+            if( i_index >= tk->i_subpackets )
+                return;
+
             block_t *p_block = block_New( p_demux, tk->i_coded_frame_size);
             if( !p_block )
                 return;
             if( &p_buf[tk->i_coded_frame_size] > &p_sys->buffer[p_sys->i_buffer] )
                 return;
 
-            int i_index = (i * 2 * tk->i_frame_size / tk->i_coded_frame_size) + y;
-
             memcpy( p_block->p_buffer, p_buf, tk->i_coded_frame_size );
             p_block->i_dts =
             p_block->i_pts = i_index == 0 ? i_pts : VLC_TS_INVALID;