videotoolbox: use generic AnnexB to xVC converter

modules/codec/Makefile.am

@@ -285,7 +285,9 @@ libvorbis_plugin_la_LIBADD = $(VORBIS_LIBS)
 EXTRA_LTLIBRARIES += libvorbis_plugin.la
 codec_LTLIBRARIES += $(LTLIBvorbis)
 
-libvideotoolbox_plugin_la_SOURCES = video_chroma/copy.c video_chroma/copy.h packetizer/h264_nal.c packetizer/h264_nal.h codec/videotoolbox.m
+libvideotoolbox_plugin_la_SOURCES = video_chroma/copy.c video_chroma/copy.h codec/videotoolbox.m \
+                                    packetizer/h264_nal.c packetizer/h264_nal.h \
+                                    packetizer/hxxx_nal.c packetizer/hxxx_nal.h
 libvideotoolbox_plugin_la_CFLAGS = $(AM_CFLAGS) -fobjc-arc
 libvideotoolbox_plugin_la_LDFLAGS = $(AM_LDFLAGS) -rpath '$(codecdir)' -Wl,-framework,Foundation -Wl,-framework,VideoToolbox -Wl,-framework,CoreMedia -Wl,-framework,CoreVideo
 EXTRA_LTLIBRARIES += libvideotoolbox_plugin.la

modules/codec/videotoolbox.m

@@ -886,7 +886,7 @@ static block_t *H264ProcessBlock(decoder_t *p_dec, block_t *p_block)
         }
     }
 
-    return (p_block) ? h264_AnnexB_to_AVC(p_block, p_sys->i_nal_length_size) : NULL;
+    return (p_block) ? hxxx_AnnexB_to_xVC(p_block, p_sys->i_nal_length_size) : NULL;
 }
 
 static CMSampleBufferRef VTSampleBufferCreate(decoder_t *p_dec,

modules/packetizer/h264_nal.c

@@ -154,171 +154,6 @@ void h264_AVC_to_AnnexB( uint8_t *p_buf, uint32_t i_len,
     }
 }
 
-static block_t *h264_increase_startcode_size( block_t *p_block,
-                                              size_t i_start_ofs )
-{
-    block_t *p_new;
-    uint32_t i_buf = p_block->i_buffer - i_start_ofs;
-    uint8_t *p_buf = p_block->p_buffer;
-    uint8_t *p_new_buf;
-    size_t i_ofs = i_start_ofs;
-    size_t i_grow = 0;
-    size_t i_new_ofs;
-
-    /* Search all startcode of size 3 */
-    unsigned i_bitflow = 0;
-    unsigned i_nalcount = 0;
-    while( i_buf-- )
-    {
-        i_bitflow <<= 1;
-        if( *(p_buf++) != 0x01 )
-        {
-            i_bitflow |= 1;
-        }
-        else if( (i_bitflow & 0x06) == 0x06 ) /* two zero prefixed 1 */
-        {
-            i_nalcount++;
-            if( !(i_bitflow & 0x08) ) /* max two zero prefixed 1 */
-                i_grow++;
-        }
-   }
-
-    if( i_grow == 0 )
-        return p_block;
-
-
-
-    /* Alloc a bigger buffer */
-    p_new = block_Alloc( p_block->i_buffer + i_grow );
-    if( !p_new )
-        return NULL;
-    i_buf = p_block->i_buffer - i_start_ofs;
-    p_buf = p_block->p_buffer;
-    p_new_buf = p_new->p_buffer;
-    i_new_ofs = i_ofs = i_start_ofs;
-
-    /* Copy the beginning of the buffer (same data) */
-    if( i_start_ofs )
-        memcpy( p_new_buf, p_buf, i_start_ofs );
-
-    /* Copy the rest of the buffer and append a 0 before each 000001 */
-    while( i_buf > 0 )
-    {
-        if( i_buf > 3 && memcmp( &p_buf[i_ofs], annexb_startcode3, 3 ) == 0 )
-        {
-            if( i_ofs == 0 || p_buf[i_ofs - 1] != 0 )
-                p_new_buf[i_new_ofs++] = 0;
-            for( int i = 0; i < 3; ++i )
-                p_new_buf[i_new_ofs++] = p_buf[i_ofs++];
-            i_buf -= 3;
-        } else
-        {
-            p_new_buf[i_new_ofs++] = p_buf[i_ofs++];
-            i_buf--;
-        }
-   }
-
-    block_Release( p_block );
-    return p_new;
-}
-
-static int h264_replace_startcode( uint8_t *p_buf,
-                                   uint8_t i_nal_length_size,
-                                   size_t i_startcode_ofs,
-                                   size_t i_nal_size )
-{
-    if( i_nal_size < (unsigned) 1 << ( 8 * i_nal_length_size) )
-    {
-        /* NAL is too big to fit in i_nal_length_size */
-        return -1;
-    }
-
-    p_buf[i_startcode_ofs++] = i_nal_size >> (--i_nal_length_size * 8);
-    if( !i_nal_length_size )
-        return 0;
-    p_buf[i_startcode_ofs++] = i_nal_size >> (--i_nal_length_size * 8);
-    if( !i_nal_length_size )
-        return 0;
-    p_buf[i_startcode_ofs++] = i_nal_size >> (--i_nal_length_size * 8);
-    p_buf[i_startcode_ofs] = i_nal_size;
-    return 0;
-}
-
-block_t *h264_AnnexB_to_AVC( block_t *p_block, uint8_t i_nal_length_size )
-{
-    size_t i_startcode_ofs = 0;
-    size_t i_startcode_size = 0;
-    uint32_t i_buf = p_block->i_buffer;
-    uint8_t *p_buf = p_block->p_buffer;
-    size_t i_ofs = 0;
-
-    /* The length of the NAL size is encoded using 1, 2 or 4 bytes */
-    if( i_nal_length_size != 1 && i_nal_length_size != 2
-     && i_nal_length_size != 4 )
-        goto error;
-
-    /* Replace the Annex B start code with the size of the NAL. */
-    while( i_buf > 0 )
-    {
-        if( i_buf > 3 && memcmp( &p_buf[i_ofs], annexb_startcode3, 3 ) == 0 )
-        {
-            if( i_startcode_size )
-            {
-                size_t i_nal_size = i_ofs - i_startcode_ofs - i_startcode_size;
-
-                if( i_ofs > 0 && p_buf[i_ofs - 1] == 0 )
-                    i_nal_size--;
-                if( h264_replace_startcode( p_buf, i_nal_length_size,
-                                            i_startcode_ofs,
-                                            i_nal_size ) )
-                    goto error;
-            }
-            if( i_ofs > 0 && p_buf[i_ofs - 1] == 0 )
-            {
-                /* startcode of size 3 */
-                i_startcode_ofs = i_ofs - 1;
-                i_startcode_size = 4;
-            }
-            else
-            {
-                i_startcode_ofs = i_ofs;
-                i_startcode_size = 3;
-            }
-
-            if( i_startcode_size < i_nal_length_size )
-            {
-                /* i_nal_length_size can't fit in i_startcode_size. Therefore,
-                 * reallocate a buffer in order to increase all startcode that
-                 * are smaller than i_nal_length_size. This is not efficient but
-                 * it's a corner case that won't happen often */
-                p_block = h264_increase_startcode_size( p_block, i_startcode_ofs );
-                if( !p_block )
-                    return NULL;
-
-                p_buf = p_block->p_buffer;
-                i_startcode_size++;
-            }
-            i_buf -= 3;
-            i_ofs += 3;
-        }
-        else
-        {
-            i_buf--;
-            i_ofs++;
-        }
-    }
-
-    if( i_startcode_size
-     && h264_replace_startcode( p_buf, i_nal_length_size, i_startcode_ofs,
-                                i_ofs - i_startcode_ofs - i_startcode_size) )
-        return NULL;
-    else
-        return p_block;
-error:
-    block_Release( p_block );
-    return NULL;
-}
-
 int h264_get_spspps( uint8_t *p_buf, size_t i_buf,
                      uint8_t **pp_sps, size_t *p_sps_size,
                      uint8_t **pp_pps, size_t *p_pps_size )

modules/packetizer/h264_nal.h

@@ -149,12 +149,6 @@ bool h264_isavcC( const uint8_t *, size_t );
 void h264_AVC_to_AnnexB( uint8_t *p_buf, uint32_t i_len,
                          uint8_t i_nal_length_size );
 
-/* Convert Annex B to AVC Sample format in-place
- * Returns the same p_block or a new p_block if there is not enough room to put
- * the NAL size. In case of error, NULL is returned and p_block is released.
- * */
-block_t *h264_AnnexB_to_AVC( block_t *p_block, uint8_t i_nal_length_size );
-
 /* Get the SPS/PPS pointers from an Annex B buffer
  * Returns 0 if a SPS and/or a PPS is found */
 int h264_get_spspps( uint8_t *p_buf, size_t i_buf,

modules/packetizer/hxxx_nal.c

@@ -38,7 +38,7 @@ static inline void hxxx_WritePrefix( uint8_t i_nal_length_size, uint8_t *p_dest,
         *p_dest = i_payload;
 }
 
-static block_t *hxxx_AnnexB_to_xVC( block_t *p_block, uint8_t i_nal_length_size )
+block_t *hxxx_AnnexB_to_xVC( block_t *p_block, uint8_t i_nal_length_size )
 {
     unsigned i_nalcount = 0;
     unsigned i_list = 16;