aout: robustify channel reordering

include/vlc_aout.h

@@ -193,7 +193,7 @@ static const uint32_t pi_vlc_chan_order_wg4[] =
  */
 VLC_API unsigned aout_CheckChannelReorder( const uint32_t *, const uint32_t *,
                                            uint32_t mask, uint8_t *table );
-VLC_API void aout_ChannelReorder(void *, size_t, unsigned, const uint8_t *, vlc_fourcc_t);
+VLC_API void aout_ChannelReorder(void *, size_t, uint8_t, const uint8_t *, vlc_fourcc_t);
 
 VLC_API void aout_Interleave(void *dst, const void *const *planes,
                              unsigned samples, unsigned channels,

src/audio_output/common.c

@@ -286,14 +286,14 @@ unsigned aout_CheckChannelReorder( const uint32_t *chans_in,
  * \param fourcc sample format (must be a linear sample format)
  * \note The samples must be naturally aligned in memory.
  */
-void aout_ChannelReorder( void *ptr, size_t bytes, unsigned channels,
+void aout_ChannelReorder( void *ptr, size_t bytes, uint8_t channels,
                           const uint8_t *restrict chans_table, vlc_fourcc_t fourcc )
 {
+    if( unlikely(bytes == 0) )
+        return;
+
     assert( channels != 0 );
-    assert( channels <= AOUT_CHAN_MAX );
 
-    if ( channels == 0 || channels >= AOUT_CHAN_MAX )
-        return;
     /* The audio formats supported in audio output are inlined. For other
      * formats (used in demuxers and muxers), memcpy() is used to avoid
      * breaking type punning. */
@@ -313,36 +313,32 @@ do { \
     } \
 } while(0)
 
-    switch( fourcc )
+    if( likely(channels <= AOUT_CHAN_MAX) )
     {
-        case VLC_CODEC_U8:   REORDER_TYPE(uint8_t); break;
-        case VLC_CODEC_S16N: REORDER_TYPE(int16_t); break;
-        case VLC_CODEC_FL32: REORDER_TYPE(float);   break;
-        case VLC_CODEC_S32N: REORDER_TYPE(int32_t); break;
-        case VLC_CODEC_FL64: REORDER_TYPE(double);  break;
-
-        default:
+        switch( fourcc )
         {
-            unsigned size = aout_BitsPerSample( fourcc ) / 8;
-            assert( size != 0 );
-            if ( size == 0 )
-                return;
+            case VLC_CODEC_U8:   REORDER_TYPE(uint8_t); return;
+            case VLC_CODEC_S16N: REORDER_TYPE(int16_t); return;
+            case VLC_CODEC_FL32: REORDER_TYPE(float);   return;
+            case VLC_CODEC_S32N: REORDER_TYPE(int32_t); return;
+            case VLC_CODEC_FL64: REORDER_TYPE(double);  return;
+        }
+    }
 
-            const size_t frames = bytes / (size * channels);
-            unsigned char *buf = ptr;
+    unsigned size = aout_BitsPerSample( fourcc ) / 8;
+    assert( size != 0 && size <= 8 );
 
-            assert( bytes != 0 );
-            for( size_t i = 0; i < frames; i++ )
-            {
-                unsigned char tmp[AOUT_CHAN_MAX * size];
+    const size_t frames = bytes / (size * channels);
+    unsigned char *buf = ptr;
 
-                for( size_t j = 0; j < channels; j++ )
-                    memcpy( tmp + size * chans_table[j], buf + size * j, size );
-                memcpy( buf, tmp, size * channels );
-                buf += size * channels;
-            }
-            break;
-        }
+    for( size_t i = 0; i < frames; i++ )
+    {
+        unsigned char tmp[256 * 8];
+
+        for( size_t j = 0; j < channels; j++ )
+             memcpy( tmp + size * chans_table[j], buf + size * j, size );
+         memcpy( buf, tmp, size * channels );
+         buf += size * channels;
     }
 }