packetizer: hevc: use 2pass exact size buffer from hvcC->annexB

modules/codec/omxil/mediacodec.c

@@ -317,29 +317,20 @@ static int ParseVideoExtra(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
             else
                 H264SetCSD(p_dec, p_extra, i_extra, NULL);
         }
-        else
-        {
-            int buf_size = i_extra + 20;
-            uint32_t size = i_extra;
-            void *p_buf = malloc(buf_size);
-            if (!p_buf)
+        else  /* FIXME and refactor: CSDDup vs CSDless SetCSD */
         {
-                msg_Warn(p_dec, "extra buffer allocation failed");
-                return VLC_EGENERIC;
-            }
-
-            if ( hevc_ishvcC(p_extra, i_extra) &&
-                    hevc_hvcC_to_AnnexB_NAL(p_dec, p_extra, i_extra,
-                                       p_buf, buf_size, &size,
-                                       &p_sys->u.video.i_nal_length_size) == VLC_SUCCESS)
+            if (hevc_ishvcC(p_extra, i_extra))
             {
                 struct csd csd;
-
-                csd.p_buf = p_buf;
-                csd.i_size = size;
+                csd.p_buf = hevc_hvcC_to_AnnexB_NAL(p_extra, i_extra, &csd.i_size,
+                                                    &p_sys->u.video.i_nal_length_size);
+                if(csd.p_buf)
+                {
                     CSDDup(p_dec, &csd, 1);
+                    free(csd.p_buf);
                 }
-            free(p_buf);
+            }
+            /* FIXME: what to do with AnnexB ? */
         }
     }
     return VLC_SUCCESS;

modules/codec/omxil/omxil.c

@@ -1184,12 +1184,11 @@ static int OpenGeneric( vlc_object_t *p_this, bool b_encode )
         }
         else if( p_dec->fmt_in.i_codec == VLC_CODEC_HEVC && !p_sys->in.b_direct )
         {
-            p_header->nFilledLen = 0;
-            hevc_hvcC_to_AnnexB_NAL( p_dec, p_dec->fmt_in.p_extra,
-                                    p_dec->fmt_in.i_extra,
-                                    p_header->pBuffer, p_header->nAllocLen,
-                                    (uint32_t*) &p_header->nFilledLen,
-                                    &p_sys->i_nal_size_length );
+            size_t i_filled_len = 0;
+            p_header->pBuffer = hevc_hvcC_to_AnnexB_NAL(
+                        p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra,
+                        &i_filled_len, &p_sys->i_nal_size_length );
+            p_header->nFilledLen = i_filled_len;
         }
         else if(p_sys->in.b_direct)
         {

modules/packetizer/hevc.c

@@ -108,30 +108,18 @@ static int Open(vlc_object_t *p_this)
     /* Check if we have hvcC as extradata */
     if(hevc_ishvcC(p_extra, i_extra))
     {
-        p_sys->i_nal_length_size = 1 + (p_extra[21] & 0x03);
         p_dec->pf_packetize = PacketizeHVC1;
 
         /* Clear hvcC/HVC1 extra, to be replaced with AnnexB */
         free(p_dec->fmt_out.p_extra);
         p_dec->fmt_out.i_extra = 0;
 
-        size_t i_new_extra = i_extra + 40 * (4 - p_sys->i_nal_length_size);
-        uint8_t *p_new_extra = malloc(i_new_extra);
-        if(p_new_extra)
-        {
-            uint32_t i_total = 0;
-            if( hevc_hvcC_to_AnnexB_NAL( p_dec, p_extra, i_extra,
-                                        p_new_extra, i_new_extra,
-                                        &i_total, NULL ) == VLC_SUCCESS )
-            {
-                p_dec->fmt_out.p_extra = p_new_extra;
-                p_dec->fmt_out.i_extra = i_total;
-            }
-            else
-            {
-                free(p_new_extra);
-            }
-        }
+        size_t i_new_extra = 0;
+        p_dec->fmt_out.p_extra =
+                hevc_hvcC_to_AnnexB_NAL(p_extra, i_extra,
+                                        &i_new_extra, &p_sys->i_nal_length_size);
+        if(p_dec->fmt_out.p_extra)
+            p_dec->fmt_out.i_extra = i_new_extra;
     }
     else
     {

modules/packetizer/hevc_nal.c

@@ -19,89 +19,85 @@
  *****************************************************************************/
 
 #include "hevc_nal.h"
+#include "hxxx_nal.h"
 
 #include <limits.h>
 
-/* Inspired by libavcodec/hevc.c */
-int hevc_hvcC_to_AnnexB_NAL(decoder_t *p_dec, const uint8_t *p_buf,
-                           uint32_t i_buf_size, uint8_t *p_out_buf,
-                           uint32_t i_out_buf_size, uint32_t *p_sps_pps_size,
-                           uint8_t *p_nal_size)
+/* Computes size and does check the whole struct integrity */
+static size_t get_hvcC_to_AnnexB_NAL_size( const uint8_t *p_buf, size_t i_buf )
 {
-    int i, num_arrays;
-    const uint8_t *p_end = p_buf + i_buf_size;
-    uint32_t i_sps_pps_size = 0;
+    size_t i_total = 0;
 
-    if( i_buf_size <= 3 || ( !p_buf[0] && !p_buf[1] && p_buf[2] <= 1 ) )
-        return VLC_EGENERIC;
+    if( i_buf < HEVC_MIN_HVCC_SIZE )
+        return 0;
 
-    if( p_end - p_buf < 23 )
-    {
-        msg_Err( p_dec, "Input Metadata too small" );
-        return VLC_ENOMEM;
-    }
+    const uint8_t i_nal_length_size = (p_buf[21] & 0x03) + 1;
+    if(i_nal_length_size == 3)
+        return 0;
 
-    p_buf += 21;
+    const uint8_t i_num_array = p_buf[22];
+    p_buf += 23; i_buf -= 23;
 
-    if( p_nal_size )
-        *p_nal_size = (*p_buf & 0x03) + 1;
-    p_buf++;
+    for( uint8_t i = 0; i < i_num_array; i++ )
+    {
+        if(i_buf < 3)
+            return 0;
 
-    num_arrays = *p_buf++;
+        const uint16_t i_num_nalu = p_buf[1] << 8 | p_buf[2];
+        p_buf += 3; i_buf -= 3;
 
-    for( i = 0; i < num_arrays; i++ )
+        for( uint16_t j = 0; j < i_num_nalu; j++ )
         {
-        int type, cnt, j;
+            if(i_buf < 2)
+                return 0;
 
-        if( p_end - p_buf < 3 )
-        {
-            msg_Err( p_dec, "Input Metadata too small" );
-            return VLC_ENOMEM;
+            const uint16_t i_nalu_length = p_buf[0] << 8 | p_buf[1];
+            if(i_buf < i_nalu_length)
+                return 0;
+
+            i_total += i_nalu_length + i_nal_length_size;
+            p_buf += i_nalu_length + 2;
+            i_buf -= i_nalu_length + 2;
+        }
     }
-        type = *(p_buf++) & 0x3f;
-        VLC_UNUSED(type);
 
-        cnt = p_buf[0] << 8 | p_buf[1];
-        p_buf += 2;
+    return i_total;
+}
 
-        for( j = 0; j < cnt; j++ )
-        {
-            int i_nal_size;
+uint8_t * hevc_hvcC_to_AnnexB_NAL( const uint8_t *p_buf, size_t i_buf,
+                                   size_t *pi_result, uint8_t *pi_nal_length_size )
+{
+    *pi_result = get_hvcC_to_AnnexB_NAL_size( p_buf, i_buf ); /* Does all checks */
+    if( *pi_result == 0 )
+        return NULL;
 
-            if( p_end - p_buf < 2 )
-            {
-                msg_Err( p_dec, "Input Metadata too small" );
-                return VLC_ENOMEM;
-            }
+    if( pi_nal_length_size )
+        *pi_nal_length_size = (p_buf[21] & 0x03) + 1;
 
-            i_nal_size = p_buf[0] << 8 | p_buf[1];
-            p_buf += 2;
+    uint8_t *p_ret;
+    uint8_t *p_out_buf = p_ret = malloc( *pi_result );
+    if( !p_out_buf )
+        return NULL;
 
-            if( i_nal_size < 0 || p_end - p_buf < i_nal_size )
-            {
-                msg_Err( p_dec, "NAL unit size does not match Input Metadata size" );
-                return VLC_ENOMEM;
-            }
+    const uint8_t i_num_array = p_buf[22];
+    p_buf += 23;
 
-            if( i_sps_pps_size + 4 + i_nal_size > i_out_buf_size )
+    for( uint8_t i = 0; i < i_num_array; i++ )
     {
-                msg_Err( p_dec, "Output buffer too small" );
-                return VLC_ENOMEM;
-            }
+        const uint16_t i_num_nalu = p_buf[1] << 8 | p_buf[2];
+        p_buf += 3;
 
-            p_out_buf[i_sps_pps_size++] = 0;
-            p_out_buf[i_sps_pps_size++] = 0;
-            p_out_buf[i_sps_pps_size++] = 0;
-            p_out_buf[i_sps_pps_size++] = 1;
+        for( uint16_t j = 0; j < i_num_nalu; j++ )
+        {
+            const uint16_t i_nalu_length = p_buf[0] << 8 | p_buf[1];
 
-            memcpy(p_out_buf + i_sps_pps_size, p_buf, i_nal_size);
-            p_buf += i_nal_size;
+            memcpy( p_out_buf, annexb_startcode4, 4 );
+            memcpy( &p_out_buf[4], &p_buf[2], i_nalu_length );
 
-            i_sps_pps_size += i_nal_size;
+            p_out_buf += 4 + i_nalu_length;
+            p_buf += 2 + i_nalu_length;
         }
     }
 
-    *p_sps_pps_size = i_sps_pps_size;
-
-    return VLC_SUCCESS;
+    return p_ret;
 }

modules/packetizer/hevc_nal.h

@@ -99,7 +99,7 @@ enum hevc_nal_unit_type_e
     HEVC_NAL_UNKNOWN
 };
 
-#define HEVC_MIN_HVCC_SIZE 22
+#define HEVC_MIN_HVCC_SIZE 23
 
 /* checks if data is an HEVCDecoderConfigurationRecord */
 static inline bool hevc_ishvcC( const uint8_t *p_buf, size_t i_buf )
@@ -116,10 +116,8 @@ static inline bool hevc_ishvcC( const uint8_t *p_buf, size_t i_buf )
 }
 
 /* Converts HEVCDecoderConfigurationRecord to Annex B format */
-int hevc_hvcC_to_AnnexB_NAL( decoder_t *p_dec, const uint8_t *p_buf,
-                            uint32_t i_buf_size, uint8_t *p_out_buf,
-                            uint32_t i_out_buf_size, uint32_t *p_sps_pps_size,
-                            uint8_t *p_nal_size);
+uint8_t * hevc_hvcC_to_AnnexB_NAL( const uint8_t *p_buf, size_t i_buf,
+                                   size_t *pi_res, uint8_t *pi_nal_length_size );
 
 
 #endif /* HEVC_NAL_H */