packetizer: h264: use 2pass exact size buffer from Avcc->annexB

Useless then suspicious use in omxil.c might incur a regression.

modules/codec/crystalhd.c

@@ -108,7 +108,7 @@ struct decoder_sys_t
     HANDLE bcm_handle;       /* Device Handle */
 
     uint8_t *p_sps_pps_buf;  /* SPS/PPS buffer */
-    uint32_t i_sps_pps_size; /* SPS/PPS size */
+    size_t   i_sps_pps_size; /* SPS/PPS size */
 
     uint8_t i_nal_size;     /* NAL header size */
 
@@ -607,22 +607,11 @@ static int crystal_insert_sps_pps( decoder_t *p_dec,
                                    uint32_t i_buf_size)
 {
     decoder_sys_t *p_sys = p_dec->p_sys;
-    int ret;
 
     p_sys->i_sps_pps_size = 0;
+    p_sys->p_sps_pps_buf = h264_avcC_to_AnnexB_NAL( p_buf, i_buf_size,
+                           &p_sys->i_sps_pps_size, &p_sys->i_nal_size );
 
-    p_sys->p_sps_pps_buf = malloc( p_dec->fmt_in.i_extra * 2 );
-    if( !p_sys->p_sps_pps_buf )
-        return VLC_ENOMEM;
-
-    ret = h264_avcC_to_AnnexB_NAL( p_dec, p_buf, i_buf_size, p_sys->p_sps_pps_buf,
-                           p_dec->fmt_in.i_extra * 2, &p_sys->i_sps_pps_size,
-                           &p_sys->i_nal_size );
-    if( !ret )
-        return ret;
-
-    free( p_sys->p_sps_pps_buf );
-    p_sys->p_sps_pps_buf = NULL;
-    return ret;
+    return (p_sys->p_sps_pps_buf) ? VLC_SUCCESS : VLC_EGENERIC;
 }
 

modules/codec/mft.c

@@ -989,16 +989,19 @@ static int InitializeMFT(decoder_t *p_dec)
 
         if (p_dec->fmt_in.i_extra)
         {
-            int buf_size = p_dec->fmt_in.i_extra + 20;
-            uint32_t size = p_dec->fmt_in.i_extra;
-            uint8_t *buf = malloc(buf_size);
             if (h264_isavcC((uint8_t*)p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra))
             {
-                h264_avcC_to_AnnexB_NAL(p_dec, p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra,
-                                buf, buf_size,
-                                &size, &p_sys->nal_length_size);
+                size_t i_buf;
+                uint8_t *buf = h264_avcC_to_AnnexB_NAL(p_dec->fmt_in.p_extra,
+                                                       p_dec->fmt_in.i_extra,
+                                                      &i_buf, &p_sys->nal_length_size);
+                if(buf)
+                {
+                    free(p_dec->fmt_in.p_extra);
+                    p_dec->fmt_in.p_extra = buf;
+                    p_dec->fmt_in.i_extra = i_buf;
+                }
             }
-            free(buf);
         }
     }
     return VLC_SUCCESS;

modules/codec/omxil/mediacodec.c

@@ -300,26 +300,34 @@ static int ParseVideoExtra(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
 
     if (p_dec->fmt_in.i_codec == VLC_CODEC_H264
      || p_dec->fmt_in.i_codec == VLC_CODEC_HEVC)
+    {
+        if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
+        {
+            if (h264_isavcC(p_extra, i_extra))
+            {
+                size_t i_size = 0;
+                uint8_t *p_buf = h264_avcC_to_AnnexB_NAL(p_extra, i_extra, &i_size,
+                                                         &p_sys->u.video.i_nal_length_size);
+                if(p_buf)
+                {
+                    H264SetCSD(p_dec, p_buf, i_size, NULL);
+                    free(p_buf);
+                }
+            }
+            else
+                H264SetCSD(p_dec, p_extra, i_extra, NULL);
+        }
+        else
         {
             int buf_size = i_extra + 20;
             uint32_t size = i_extra;
             void *p_buf = malloc(buf_size);
-
             if (!p_buf)
             {
                 msg_Warn(p_dec, "extra buffer allocation failed");
                 return VLC_EGENERIC;
             }
 
-        if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
-        {
-            if ( h264_isavcC(p_extra, i_extra)
-             && h264_avcC_to_AnnexB_NAL(p_dec, p_extra, i_extra,
-                                p_buf, buf_size, &size,
-                                &p_sys->u.video.i_nal_length_size) == VLC_SUCCESS)
-                H264SetCSD(p_dec, p_buf, size, NULL);
-        } else
-        {
             if ( hevc_ishvcC(p_extra, i_extra) &&
                     hevc_hvcC_to_AnnexB_NAL(p_dec, p_extra, i_extra,
                                        p_buf, buf_size, &size,
@@ -331,9 +339,9 @@ static int ParseVideoExtra(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
                 csd.i_size = size;
                 CSDDup(p_dec, &csd, 1);
             }
-        }
             free(p_buf);
         }
+    }
     return VLC_SUCCESS;
 }
 

modules/codec/omxil/omxil.c

@@ -1173,12 +1173,14 @@ static int OpenGeneric( vlc_object_t *p_this, bool b_encode )
         p_header->nFilledLen = p_dec->fmt_in.i_extra;
 
         /* Convert H.264 NAL format to annex b */
-        if( p_sys->i_nal_size_length && !p_sys->in.b_direct )
-        {
-            p_header->nFilledLen = 0;
-            h264_avcC_to_AnnexB_NAL( p_dec, p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra,
-                             p_header->pBuffer, p_header->nAllocLen,
-                             (uint32_t*) &p_header->nFilledLen, NULL );
+        if( p_sys->i_nal_size_length && !p_sys->in.b_direct &&
+            h264_isavcC(p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra) )
+        {
+            size_t i_filled_len = 0;
+            p_header->pBuffer = h264_avcC_to_AnnexB_NAL(
+                        p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra,
+                        &i_filled_len, NULL );
+            p_header->nFilledLen = i_filled_len;
         }
         else if( p_dec->fmt_in.i_codec == VLC_CODEC_HEVC && !p_sys->in.b_direct )
         {

modules/codec/videotoolbox.m

@@ -302,57 +302,43 @@ static int StartVideoToolbox(decoder_t *p_dec, block_t *p_block)
             return VLC_SUCCESS;
         }
 
-        uint32_t size;
-        void *p_buf, *p_alloc_buf = NULL;
+        size_t i_buf;
+        const uint8_t *p_buf = NULL;
+        uint8_t *p_alloc_buf = NULL;
         int i_ret = 0;
 
         if (p_block == NULL) {
-            int buf_size = p_dec->fmt_in.i_extra + 20;
-            size = p_dec->fmt_in.i_extra;
-
-            p_alloc_buf = p_buf = malloc(buf_size);
-            if (!p_buf) {
-                msg_Warn(p_dec, "extra buffer allocation failed");
-                return VLC_ENOMEM;
-            }
-
             /* we need to convert the SPS and PPS units we received from the
             * demuxer's avvC atom so we can process them further */
-            i_ret = h264_avcC_to_AnnexB_NAL(p_dec,
-                                    p_dec->fmt_in.p_extra,
+            if(h264_isavcC(p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra))
+            {
+                p_alloc_buf = h264_avcC_to_AnnexB_NAL(p_dec->fmt_in.p_extra,
                                                       p_dec->fmt_in.i_extra,
-                                    p_buf,
-                                    buf_size,
-                                    &size,
+                                                      &i_buf,
                                                       &p_sys->i_nal_length_size);
-            p_sys->b_is_avcc = i_ret == VLC_SUCCESS;
+                buf = p_alloc_buf;
+                p_sys->b_is_avcc = !!p_buf;
+            }
         } else {
             /* we are mid-stream, let's have the h264_get helper see if it
              * can find a NAL unit */
-            size = p_block->i_buffer;
+            i_buf = p_block->i_buffer;
             p_buf = p_block->p_buffer;
             p_sys->i_nal_length_size = 4; /* default to 4 bytes */
             i_ret = VLC_SUCCESS;
         }
 
-        if (i_ret != VLC_SUCCESS) {
-            free(p_alloc_buf);
-            msg_Warn(p_dec, "extra buffer allocation failed");
-            return VLC_EGENERIC;
-        }
-
         uint8_t *p_sps_buf = NULL, *p_pps_buf = NULL;
         size_t i_sps_size = 0, i_pps_size = 0;
         if (!p_buf) {
-            free(p_alloc_buf);
-            msg_Warn(p_dec, "extra buffer allocation failed");
+            msg_Warn(p_dec, "no valid extradata or conversion failed");
             return VLC_EGENERIC;
         }
 
         /* get the SPS and PPS units from the NAL unit which is either
          * part of the demuxer's avvC atom or the mid stream data block */
         i_ret = h264_get_spspps(p_buf,
-                                size,
+                                i_buf,
                                 &p_sps_buf,
                                 &i_sps_size,
                                 &p_pps_buf,
@@ -385,6 +371,7 @@ static int StartVideoToolbox(decoder_t *p_dec, block_t *p_block)
         p_sys->codec_profile = sps_data.i_profile;
         p_sys->codec_level = sps_data.i_level;
 
+        /* FIXME: Reuse p_extra avcC is p_sys->b_is_avcc */
         /* create avvC atom to forward to the HW decoder */
         block_t *p_block = h264_AnnexB_NAL_to_avcC(
                                 p_sys->i_nal_length_size,

modules/packetizer/h264_nal.c

@@ -29,8 +29,6 @@
  * For Annex B specification, see ISO/IEC 14496-10
  */
 
-static const uint8_t annexb_startcode[] = { 0x00, 0x00, 0x01 };
-
 bool h264_isavcC( const uint8_t *p_buf, size_t i_buf )
 {
     return ( i_buf > H264_MIN_AVCC_SIZE &&
@@ -47,12 +45,12 @@ static inline bool strip_AnnexB_startcode( const uint8_t **pp_data, size_t *pi_d
     {
         return false;
     }
-    else if( p_data[0] == !!memcmp(&p_data[1], annexb_startcode, 3) )
+    else if( p_data[0] == !!memcmp(&p_data[1], annexb_startcode3, 3) )
     {
         *pp_data += 4;
         *pi_data -= 4;
     }
-    else if( !memcmp(p_data, annexb_startcode, 3) )
+    else if( !memcmp(p_data, annexb_startcode4, 3) )
     {
         *pp_data += 3;
         *pi_data -= 3;
@@ -61,79 +59,72 @@ static inline bool strip_AnnexB_startcode( const uint8_t **pp_data, size_t *pi_d
     return true;
 }
 
-int h264_avcC_to_AnnexB_NAL( decoder_t *p_dec, const uint8_t *p_buf,
-                     uint32_t i_buf_size, uint8_t *p_out_buf,
-                     uint32_t i_out_buf_size, uint32_t *p_sps_pps_size,
-                     uint8_t *pi_nal_length_size)
+static size_t get_avcC_to_AnnexB_NAL_size( const uint8_t *p_buf, size_t i_buf )
 {
-    uint32_t i_data_size = i_buf_size, i_nal_size, i_sps_pps_size = 0;
-    unsigned int i_loop_end;
+    size_t i_total = 0;
+
+    p_buf += 5;
+    i_buf -= 5;
+
+    if( i_buf < H264_MIN_AVCC_SIZE )
+        return 0;
+
+    for ( unsigned int j = 0; j < 2; j++ )
+    {
+        /* First time is SPS, Second is PPS */
+        const unsigned int i_loop_end = p_buf[0] & (j == 0 ? 0x1f : 0xff);
+        p_buf++; i_buf--;
 
-    /* */
-    if( i_data_size < 7 )
+        for ( unsigned int i = 0; i < i_loop_end; i++ )
         {
-        msg_Err( p_dec, "avcC too small" );
-        return VLC_ENOMEM;
+            uint16_t i_nal_size = (p_buf[0] << 8) | p_buf[1];
+            if(i_nal_size > i_buf - 2)
+                return 0;
+            i_total += i_nal_size + 4;
+            i_buf -= i_nal_size + 2;
         }
+    }
+    return i_total;
+}
+
+uint8_t *h264_avcC_to_AnnexB_NAL( const uint8_t *p_buf, size_t i_buf,
+                                  size_t *pi_result, uint8_t *pi_nal_length_size )
+{
+    *pi_result = get_avcC_to_AnnexB_NAL_size( p_buf, i_buf ); /* Does check min size */
+    if( *pi_result == 0 )
+        return NULL;
 
     /* Read infos in first 6 bytes */
-    if (pi_nal_length_size)
+    if ( pi_nal_length_size )
         *pi_nal_length_size = (p_buf[4] & 0x03) + 1;
+
+    uint8_t *p_ret;
+    uint8_t *p_out_buf = p_ret = malloc( *pi_result );
+    if( !p_out_buf )
+        return NULL;
+
     p_buf += 5;
-    i_data_size -= 5;
 
     for ( unsigned int j = 0; j < 2; j++ )
     {
-        /* First time is SPS, Second is PPS */
-        if( i_data_size < 1 )
-        {
-            msg_Err( p_dec, "PPS too small after processing SPS/PPS %u",
-                    i_data_size );
-            return VLC_ENOMEM;
-        }
-        i_loop_end = p_buf[0] & (j == 0 ? 0x1f : 0xff);
-        p_buf++; i_data_size--;
+        const unsigned int i_loop_end = p_buf[0] & (j == 0 ? 0x1f : 0xff);
+        p_buf++;
 
         for ( unsigned int i = 0; i < i_loop_end; i++)
         {
-            if( i_data_size < 2 )
-            {
-                msg_Err( p_dec, "SPS is too small %u", i_data_size );
-                return VLC_ENOMEM;
-            }
-
-            i_nal_size = (p_buf[0] << 8) | p_buf[1];
+            uint16_t i_nal_size = (p_buf[0] << 8) | p_buf[1];
             p_buf += 2;
-            i_data_size -= 2;
 
-            if( i_data_size < i_nal_size )
-            {
-                msg_Err( p_dec, "SPS size does not match NAL specified size %u",
-                        i_data_size );
-                return VLC_ENOMEM;
-            }
-            if( i_sps_pps_size + 4 + i_nal_size > i_out_buf_size )
-            {
-                msg_Err( p_dec, "Output SPS/PPS buffer too small" );
-                return VLC_ENOMEM;
-            }
-
-            p_out_buf[i_sps_pps_size++] = 0;
-            p_out_buf[i_sps_pps_size++] = 0;
-            p_out_buf[i_sps_pps_size++] = 0;
-            p_out_buf[i_sps_pps_size++] = 1;
-
-            memcpy( p_out_buf + i_sps_pps_size, p_buf, i_nal_size );
-            i_sps_pps_size += i_nal_size;
+            memcpy( p_out_buf, annexb_startcode4, 4 );
+            p_out_buf += 4;
 
+            memcpy( p_out_buf, p_buf, i_nal_size );
+            p_out_buf += i_nal_size;
             p_buf += i_nal_size;
-            i_data_size -= i_nal_size;
         }
     }
 
-    *p_sps_pps_size = i_sps_pps_size;
-
-    return VLC_SUCCESS;
+    return p_ret;
 }
 
 void h264_AVC_to_AnnexB( uint8_t *p_buf, uint32_t i_len,
@@ -192,7 +183,7 @@ static block_t *h264_increase_startcode_size( block_t *p_block,
     /* Search all startcode of size 3 */
     while( i_buf > 0 )
     {
-        if( i_buf > 3 && memcmp( &p_buf[i_ofs], annexb_startcode, 3 ) == 0 )
+        if( i_buf > 3 && memcmp( &p_buf[i_ofs], annexb_startcode3, 3 ) == 0 )
         {
             if( i_ofs == 0 || p_buf[i_ofs - 1] != 0 )
                 i_grow++;
@@ -224,7 +215,7 @@ static block_t *h264_increase_startcode_size( block_t *p_block,
     /* Copy the rest of the buffer and append a 0 before each 000001 */
     while( i_buf > 0 )
     {
-        if( i_buf > 3 && memcmp( &p_buf[i_ofs], annexb_startcode, 3 ) == 0 )
+        if( i_buf > 3 && memcmp( &p_buf[i_ofs], annexb_startcode3, 3 ) == 0 )
         {
             if( i_ofs == 0 || p_buf[i_ofs - 1] != 0 )
                 p_new_buf[i_new_ofs++] = 0;
@@ -280,7 +271,7 @@ block_t *h264_AnnexB_to_AVC( block_t *p_block, uint8_t i_nal_length_size )
     /* Replace the Annex B start code with the size of the NAL. */
     while( i_buf > 0 )
     {
-        if( i_buf > 3 && memcmp( &p_buf[i_ofs], annexb_startcode, 3 ) == 0 )
+        if( i_buf > 3 && memcmp( &p_buf[i_ofs], annexb_startcode3, 3 ) == 0 )
         {
             if( i_startcode_size )
             {
@@ -354,7 +345,7 @@ int h264_get_spspps( uint8_t *p_buf, size_t i_buf,
         unsigned int i_move = 1;
 
         /* cf B.1.1: a NAL unit starts and ends with 0x000001 or 0x00000001 */
-        if( i_buf > 3 && !memcmp( p_buf, annexb_startcode, 3 ) )
+        if( i_buf > 3 && !memcmp( p_buf, annexb_startcode3, 3 ) )
         {
             if( i_nal_type != H264_NAL_UNKNOWN )
             {

modules/packetizer/h264_nal.h

@@ -152,10 +152,8 @@ block_t *h264_AnnexB_NAL_to_avcC( uint8_t i_nal_length_size,
                                   size_t i_pps_size );
 
 /* Convert AVCDecoderConfigurationRecord SPS/PPS to Annex B format */
-int h264_avcC_to_AnnexB_NAL( decoder_t *p_dec, const uint8_t *p_buf,
-                             uint32_t i_buf_size, uint8_t *p_out_buf,
-                             uint32_t i_out_buf_size, uint32_t *p_sps_pps_size,
-                             uint8_t *p_nal_length_size);
+uint8_t * h264_avcC_to_AnnexB_NAL( const uint8_t *p_buf, size_t i_buf,
+                                   size_t *pi_result, uint8_t *pi_nal_length_size );
 
 /* Get level and Profile */
 bool h264_get_profile_level(const es_format_t *p_fmt, size_t *p_profile,

modules/packetizer/hxxx_nal.h

@@ -26,6 +26,9 @@
 
 #include <vlc_common.h>
 
+static const uint8_t  annexb_startcode4[] = { 0x00, 0x00, 0x00, 0x01 };
+#define annexb_startcode3 (&annexb_startcode4[1])
+
 /* Discards emulation prevention three bytes */
 static inline uint8_t * hxxx_ep3b_to_rbsp(const uint8_t *p_src, size_t i_src, size_t *pi_ret)
 {