m3u: rewrite probing (fix #9237)

modules/demux/playlist/m3u.c

@@ -67,30 +67,44 @@ int Import_M3U( vlc_object_t *p_this )
 {
     demux_t *p_demux = (demux_t *)p_this;
     const uint8_t *p_peek;
-    CHECK_PEEK( p_peek, 8 );
-    char *(*pf_dup) (const char *);
+    char *(*pf_dup) (const char *) = GuessEncoding;
+    int offset = 0;
 
-    if( POKE( p_peek, "\xef\xbb\xbf", 3) )/* BOM at start */
+    if( stream_Peek( p_demux->s, &p_peek, 3 ) == 3
+     && !memcmp( p_peek, "\xef\xbb\xbf", 3) )
     {
-        pf_dup = CheckUnicode; /* UTF-8 */
-        stream_Seek( p_demux->s, 3 );
+        pf_dup = CheckUnicode; /* UTF-8 Byte Order Mark */
+        offset = 3;
     }
-    else
-    if( POKE( p_peek, "RTSPtext", 8 ) /* QuickTime */
-     || demux_IsPathExtension( p_demux, ".m3u8" )
+
+    if( demux_IsPathExtension( p_demux, ".m3u8" )
      || demux_IsForced( p_demux, "m3u8" )
      || CheckContentType( p_demux->s, "application/vnd.apple.mpegurl" ) )
-        pf_dup = CheckUnicode; /* UTF-8 */
+        pf_dup = CheckUnicode; /* UTF-8 file type */
     else
-    if( POKE( p_peek, "#EXTM3U", 7 )
-     || demux_IsPathExtension( p_demux, ".m3u" )
+    if( demux_IsPathExtension( p_demux, ".m3u" )
      || demux_IsPathExtension( p_demux, ".vlc" )
      || demux_IsForced( p_demux, "m3u" )
      || ContainsURL( p_demux )
      || CheckContentType( p_demux->s, "audio/x-mpegurl") )
-        pf_dup = GuessEncoding;
+        ; /* Guess encoding */
     else
-        return VLC_EGENERIC;
+    {
+        if( stream_Peek( p_demux->s, &p_peek, 8 + offset ) < (8 + offset) )
+            return VLC_EGENERIC;
+
+        p_peek += offset;
+
+        if( !memcmp( p_peek, "RTSPtext", 8 ) ) /* QuickTime */
+            pf_dup = CheckUnicode; /* UTF-8 */
+        else
+        if( !memcmp( p_peek, "#EXTM3U", 7 ) )
+            ; /* Guess encoding */
+        else
+            return VLC_EGENERIC;
+    }
+
+    stream_Seek( p_demux->s, offset );
 
     STANDARD_DEMUX_INIT_MSG( "found valid M3U playlist" );
     p_demux->p_sys->psz_prefix = FindPrefix( p_demux );