demux: mkv: handle 0 trailed DTS samples (fix #12725)

Sounds like a broken encoder (growing frame size from N-1 to N twice)

modules/demux/Makefile.am

@@ -180,6 +180,7 @@ libmkv_plugin_la_SOURCES = \
 	demux/mp4/libmp4.c demux/vobsub.h \
 	demux/mkv/mkv.hpp demux/mkv/mkv.cpp \
 	demux/windows_audio_commons.h
+libmkv_plugin_la_SOURCES += codec/dts_header.h codec/dts_header.c
 libmkv_plugin_la_CPPFLAGS = $(AM_CPPFLAGS)
 libmkv_plugin_la_LDFLAGS = $(AM_LDFLAGS) -rpath '$(demuxdir)'
 libmkv_plugin_la_LIBADD = $(LIBS_mkv)

modules/demux/mkv/mkv.cpp

@@ -33,6 +33,10 @@
 
 #include "stream_io_callback.hpp"
 
+extern "C" {
+#include "../../modules/codec/dts_header.h"
+}
+
 #include <vlc_fs.h>
 #include <vlc_url.h>
 
@@ -606,6 +610,20 @@ void BlockDecode( demux_t *p_demux, KaxBlock *block, KaxSimpleBlock *simpleblock
                 VLC_TS_INVALID;
             continue;
          }
+
+        case VLC_CODEC_DTS:
+            /* Check if packetization is correct and without padding.
+             * example: Test_mkv_div3_DTS_1920x1080_1785Kbps_23,97fps.mkv */
+            if( p_block->i_buffer > 6 )
+            {
+                unsigned int a, b, c, d;
+                bool e;
+                int i_frame_size = GetSyncInfo( p_block->p_buffer, &e, &a, &b, &c, &d );
+                if( i_frame_size > 0 )
+                    p_block->i_buffer = __MIN(p_block->i_buffer, (size_t)i_frame_size);
+            }
+            break;
+
          case VLC_CODEC_OPUS:
             mtime_t i_length = i_duration * tk-> f_timecodescale *
                     (double) p_segment->i_timescale / 1000.0;