Set i_bitspersample to correct value when changing an audio codec. (Not

sure if I didn't miss some spots where that was missing). Thanks to
Kevin DuBois for finding the bug. (This should fix some audio
transcoding crashes)

include/vlc_aout.h

@@ -365,6 +365,7 @@ VLC_EXPORT( int, aout_CheckChannelReorder, ( const uint32_t *, const uint32_t *,
 VLC_EXPORT( void, aout_ChannelReorder, ( uint8_t *, int, int, const int *, int ) );
 
 VLC_EXPORT( unsigned int, aout_FormatNbChannels, ( const audio_sample_format_t * p_format ) );
+VLC_EXPORT( unsigned int, aout_BitsPerSample, ( vlc_fourcc_t i_format ) );
 VLC_EXPORT( void, aout_FormatPrepare, ( audio_sample_format_t * p_format ) );
 VLC_EXPORT( void, aout_FormatPrint, ( aout_instance_t * p_aout, const char * psz_text, const audio_sample_format_t * p_format ) );
 VLC_EXPORT( const char *, aout_FormatPrintChannels, ( const audio_sample_format_t * ) );

modules/audio_filter/converter/a52tofloat32.c

@@ -438,6 +438,8 @@ static int OpenFilter( vlc_object_t *p_this )
 #else
         p_filter->fmt_out.i_codec = VLC_FOURCC('f','l','3','2');
 #endif
+    p_filter->fmt_out.audio.i_bitspersample =
+        aout_BitsPerSample( p_filter->fmt_out.i_codec );
 
     /* Allocate the memory needed to store the module's structure */
     p_filter->p_sys = p_sys = malloc( sizeof(filter_sys_t) );

modules/audio_filter/converter/dtstofloat32.c

@@ -396,6 +396,8 @@ static int OpenFilter( vlc_object_t *p_this )
 
     p_filter->fmt_out.audio.i_format =
         p_filter->fmt_out.i_codec = VLC_FOURCC('f','l','3','2');
+    p_filter->fmt_out.audio.i_bitspersample =
+        aout_BitsPerSample( p_filter->fmt_out.i_codec );
 
     /* Allocate the memory needed to store the module's structure */
     p_sys = p_filter->p_sys = malloc( sizeof(filter_sys_t) );

modules/audio_filter/converter/mpgatofixed32.c

@@ -335,6 +335,8 @@ static int OpenFilter( vlc_object_t *p_this )
     else
         p_filter->fmt_out.i_codec = VLC_FOURCC('f','i','3','2');
     p_filter->fmt_out.audio.i_format = p_filter->fmt_out.i_codec;
+    p_filter->fmt_out.audio.i_bitspersample =
+        aout_BitsPerSample( p_filter->fmt_out.i_codec );
 
     p_filter->fmt_out.audio.i_rate = p_filter->fmt_in.audio.i_rate;
 

src/audio_output/common.c

@@ -109,52 +109,54 @@ unsigned int aout_FormatNbChannels( const audio_sample_format_t * p_format )
 }
 
 /*****************************************************************************
- * aout_FormatPrepare : compute the number of bytes per frame & frame length
+ * aout_BitsPerSample : get the number of bits per sample
  *****************************************************************************/
-void aout_FormatPrepare( audio_sample_format_t * p_format )
+unsigned int aout_BitsPerSample( vlc_fourcc_t i_format )
 {
-    int i_result;
-
-    switch ( p_format->i_format )
+    switch( i_format )
     {
     case VLC_FOURCC('u','8',' ',' '):
     case VLC_FOURCC('s','8',' ',' '):
-        i_result = 1;
-        break;
+        return 8;
 
     case VLC_FOURCC('u','1','6','l'):
     case VLC_FOURCC('s','1','6','l'):
     case VLC_FOURCC('u','1','6','b'):
     case VLC_FOURCC('s','1','6','b'):
-        i_result = 2;
-        break;
+        return 16;
 
     case VLC_FOURCC('u','2','4','l'):
     case VLC_FOURCC('s','2','4','l'):
     case VLC_FOURCC('u','2','4','b'):
     case VLC_FOURCC('s','2','4','b'):
-        i_result = 3;
-        break;
+        return 24;
 
     case VLC_FOURCC('f','l','3','2'):
     case VLC_FOURCC('f','i','3','2'):
-        i_result = 4;
-        break;
-
-    case VLC_FOURCC('s','p','d','i'):
-    case VLC_FOURCC('s','p','d','b'): /* Big endian spdif output */
-    case VLC_FOURCC('a','5','2',' '):
-    case VLC_FOURCC('d','t','s',' '):
-    case VLC_FOURCC('m','p','g','a'):
-    case VLC_FOURCC('m','p','g','3'):
+        return 32;
+
+    case VLC_FOURCC('f','l','6','4'):
+        return 64;
+
     default:
         /* For these formats the caller has to indicate the parameters
          * by hand. */
-        return;
+        return 0;
     }
+}
 
-    p_format->i_bytes_per_frame = i_result * aout_FormatNbChannels( p_format );
+/*****************************************************************************
+ * aout_FormatPrepare : compute the number of bytes per frame & frame length
+ *****************************************************************************/
+void aout_FormatPrepare( audio_sample_format_t * p_format )
+{
+    p_format->i_bitspersample = aout_BitsPerSample( p_format->i_format );
+    if( p_format->i_bitspersample > 0 )
+    {
+        p_format->i_bytes_per_frame = ( p_format->i_bitspersample / 8 )
+                                    * aout_FormatNbChannels( p_format );
         p_format->i_frame_length = 1;
+    }
 }
 
 /*****************************************************************************