aout: robustify channel reordering

include/vlc_aout.h

@@ -193,7 +193,7 @@ static const uint32_t pi_vlc_chan_order_wg4[] =
  */
 VLC_API unsigned aout_CheckChannelReorder( const uint32_t *, const uint32_t *,
                                            uint32_t mask, uint8_t *table );
-VLC_API void aout_ChannelReorder(void *, size_t, unsigned, const uint8_t *, vlc_fourcc_t);
+VLC_API void aout_ChannelReorder(void *, size_t, uint8_t, const uint8_t *, vlc_fourcc_t);
 
 VLC_API void aout_Interleave(void *dst, const void *const *planes,
                              unsigned samples, unsigned channels,

src/audio_output/common.c

@@ -286,14 +286,14 @@ unsigned aout_CheckChannelReorder( const uint32_t *chans_in,
  * \param fourcc sample format (must be a linear sample format)
  * \note The samples must be naturally aligned in memory.
  */
-void aout_ChannelReorder( void *ptr, size_t bytes, unsigned channels,
+void aout_ChannelReorder( void *ptr, size_t bytes, uint8_t channels,
                           const uint8_t *restrict chans_table, vlc_fourcc_t fourcc )
 {
+    if( unlikely(bytes == 0) )
+        return;
+
     assert( channels != 0 );
-    assert( channels <= AOUT_CHAN_MAX );
 
-    if ( channels == 0 || channels >= AOUT_CHAN_MAX )
-        return;
     /* The audio formats supported in audio output are inlined. For other
      * formats (used in demuxers and muxers), memcpy() is used to avoid
      * breaking type punning. */
@@ -313,37 +313,33 @@ do { \
     } \
 } while(0)
 
+    if( likely(channels <= AOUT_CHAN_MAX) )
+    {
         switch( fourcc )
         {
-        case VLC_CODEC_U8:   REORDER_TYPE(uint8_t); break;
-        case VLC_CODEC_S16N: REORDER_TYPE(int16_t); break;
-        case VLC_CODEC_FL32: REORDER_TYPE(float);   break;
-        case VLC_CODEC_S32N: REORDER_TYPE(int32_t); break;
-        case VLC_CODEC_FL64: REORDER_TYPE(double);  break;
+            case VLC_CODEC_U8:   REORDER_TYPE(uint8_t); return;
+            case VLC_CODEC_S16N: REORDER_TYPE(int16_t); return;
+            case VLC_CODEC_FL32: REORDER_TYPE(float);   return;
+            case VLC_CODEC_S32N: REORDER_TYPE(int32_t); return;
+            case VLC_CODEC_FL64: REORDER_TYPE(double);  return;
+        }
+    }
 
-        default:
-        {
     unsigned size = aout_BitsPerSample( fourcc ) / 8;
-            assert( size != 0 );
-            if ( size == 0 )
-                return;
+    assert( size != 0 && size <= 8 );
 
     const size_t frames = bytes / (size * channels);
     unsigned char *buf = ptr;
 
-            assert( bytes != 0 );
     for( size_t i = 0; i < frames; i++ )
     {
-                unsigned char tmp[AOUT_CHAN_MAX * size];
+        unsigned char tmp[256 * 8];
 
         for( size_t j = 0; j < channels; j++ )
              memcpy( tmp + size * chans_table[j], buf + size * j, size );
          memcpy( buf, tmp, size * channels );
          buf += size * channels;
     }
-            break;
-        }
-    }
 }
 
 /**