codec: dts: fix custom channel mapping detection/tautology

(cherry picked from commit 7e19ed8b7c7a833b091f9127bac38d7c4a6e958c)
Signed-off-by: Jean-Baptiste Kempf <jb@videolan.org>

modules/codec/dts.c

@@ -548,18 +548,20 @@ static int SyncInfo( const uint8_t *p_buf,
                                 AOUT_CHAN_REARRIGHT | AOUT_CHAN_LFE;
             break;
 
+        case 0xF:
         default:
-            if( i_audio_mode <= 63 )
+            if( (i_audio_mode & 0xFFFF) >= 0x10 )
             {
                 /* User defined */
                 *pi_channels = 0;
                 *pi_channels_conf = 0;
             }
             else return 0;
+
             break;
     }
 
-    if( i_audio_mode & 0x10000 )
+    if( *pi_channels && (i_audio_mode & 0x10000) )
     {
         (*pi_channels)++;
         *pi_channels_conf |= AOUT_CHAN_LFE;