transcoding video, sends NULL when ending. Should fix sending all the encoded frames to sout

Sends NULL-pict to encode when it's closing time, then encoder knows to
flush buffers and check that it has outputted all the frames.

Contains changes to x264/avcodec-module to implement that on encoder-side.
Add quick check on omxil/dirac/theora for that, so they don't crash.
And add likely-macro to x264/avcodec in check of NULL

If someone more familiar with dirac/theora/omxil encoder-modules could
check if they have buffers that need to be outputted, would be nice.

Fixes some tickets, but I failed to find any of those in trac.

modules/codec/avcodec/encoder.c

@@ -720,101 +720,108 @@ static block_t *EncodeVideo( encoder_t *p_enc, picture_t *p_pict )
 
 
     memset( &frame, 0, sizeof( AVFrame ) );
-    for( i_plane = 0; i_plane < p_pict->i_planes; i_plane++ )
-    {
-        frame.data[i_plane] = p_pict->p[i_plane].p_pixels;
-        frame.linesize[i_plane] = p_pict->p[i_plane].i_pitch;
-    }
-
-    /* Let ffmpeg select the frame type */
-    frame.pict_type = 0;
-
-    frame.repeat_pict = p_pict->i_nb_fields - 2;
-    frame.interlaced_frame = !p_pict->b_progressive;
-    frame.top_field_first = !!p_pict->b_top_field_first;
-
-    /* Set the pts of the frame being encoded (segfaults with mpeg4!)*/
-    if( p_enc->fmt_out.i_codec != VLC_CODEC_MP4V )
-    {
-        frame.pts = p_pict->date ? p_pict->date : (int64_t)AV_NOPTS_VALUE;
-
-        if ( p_sys->b_hurry_up && frame.pts != (int64_t)AV_NOPTS_VALUE )
-        {
-            mtime_t current_date = mdate();
-
-            if ( current_date + HURRY_UP_GUARD3 > frame.pts )
-            {
-                p_sys->p_context->mb_decision = FF_MB_DECISION_SIMPLE;
-                p_sys->p_context->trellis = 0;
-                msg_Dbg( p_enc, "hurry up mode 3" );
-            }
-            else
-            {
-                p_sys->p_context->mb_decision = p_sys->i_hq;
-
-                if ( current_date + HURRY_UP_GUARD2 > frame.pts )
-                {
-                    p_sys->p_context->trellis = 0;
-                    p_sys->p_context->noise_reduction = p_sys->i_noise_reduction
-                         + (HURRY_UP_GUARD2 + current_date - frame.pts) / 500;
-                    msg_Dbg( p_enc, "hurry up mode 2" );
-                }
-                else
-                {
-                    p_sys->p_context->trellis = p_sys->b_trellis;
-
-                    p_sys->p_context->noise_reduction =
-                        p_sys->i_noise_reduction;
-                }
-            }
-
-            if ( current_date + HURRY_UP_GUARD1 > frame.pts )
-            {
-                frame.pict_type = FF_P_TYPE;
-                /* msg_Dbg( p_enc, "hurry up mode 1 %lld", current_date + HURRY_UP_GUARD1 - frame.pts ); */
-            }
-        }
+    if( likely(p_pict) ) {
+       for( i_plane = 0; i_plane < p_pict->i_planes; i_plane++ )
+       {
+           frame.data[i_plane] = p_pict->p[i_plane].p_pixels;
+           frame.linesize[i_plane] = p_pict->p[i_plane].i_pitch;
+       }
+
+       /* Let ffmpeg select the frame type */
+       frame.pict_type = 0;
+
+       frame.repeat_pict = p_pict->i_nb_fields - 2;
+       frame.interlaced_frame = !p_pict->b_progressive;
+       frame.top_field_first = !!p_pict->b_top_field_first;
+
+       /* Set the pts of the frame being encoded (segfaults with mpeg4!)*/
+       if( p_enc->fmt_out.i_codec != VLC_CODEC_MP4V )
+       {
+           frame.pts = p_pict->date ? p_pict->date : (int64_t)AV_NOPTS_VALUE;
+
+           if ( p_sys->b_hurry_up && frame.pts != (int64_t)AV_NOPTS_VALUE )
+           {
+               mtime_t current_date = mdate();
+
+               if ( current_date + HURRY_UP_GUARD3 > frame.pts )
+               {
+                   p_sys->p_context->mb_decision = FF_MB_DECISION_SIMPLE;
+                   p_sys->p_context->trellis = 0;
+                   msg_Dbg( p_enc, "hurry up mode 3" );
+               }
+               else
+               {
+                   p_sys->p_context->mb_decision = p_sys->i_hq;
+
+                   if ( current_date + HURRY_UP_GUARD2 > frame.pts )
+                   {
+                       p_sys->p_context->trellis = 0;
+                       p_sys->p_context->noise_reduction = p_sys->i_noise_reduction
+                            + (HURRY_UP_GUARD2 + current_date - frame.pts) / 500;
+                       msg_Dbg( p_enc, "hurry up mode 2" );
+                   }
+                   else
+                   {
+                       p_sys->p_context->trellis = p_sys->b_trellis;
+
+                       p_sys->p_context->noise_reduction =
+                           p_sys->i_noise_reduction;
+                   }
+               }
+
+               if ( current_date + HURRY_UP_GUARD1 > frame.pts )
+               {
+                   frame.pict_type = FF_P_TYPE;
+                   /* msg_Dbg( p_enc, "hurry up mode 1 %lld", current_date + HURRY_UP_GUARD1 - frame.pts ); */
+               }
+           }
+       }
+       else
+       {
+           frame.pts = (int64_t)AV_NOPTS_VALUE;
+       }
+
+       if ( frame.pts != (int64_t)AV_NOPTS_VALUE && frame.pts != 0 )
+       {
+           if ( p_sys->i_last_pts == frame.pts )
+           {
+               msg_Warn( p_enc, "almost fed libavcodec with two frames with the "
+                         "same PTS (%"PRId64 ")", frame.pts );
+               return NULL;
+           }
+           else if ( p_sys->i_last_pts > frame.pts )
+           {
+               msg_Warn( p_enc, "almost fed libavcodec with a frame in the "
+                         "past (current: %"PRId64 ", last: %"PRId64")",
+                         frame.pts, p_sys->i_last_pts );
+               return NULL;
+           }
+           else
+           {
+               p_sys->i_last_pts = frame.pts;
+           }
+       }
+
+       frame.quality = p_sys->i_quality;
+
+       /* Ugly work-around for stupid libavcodec behaviour */
+       p_sys->i_framenum++;
+       p_sys->pi_delay_pts[p_sys->i_framenum % MAX_FRAME_DELAY] = frame.pts;
+       frame.pts = p_sys->i_framenum * AV_TIME_BASE *
+           p_enc->fmt_in.video.i_frame_rate_base;
+       frame.pts += p_enc->fmt_in.video.i_frame_rate - 1;
+       frame.pts /= p_enc->fmt_in.video.i_frame_rate;
+       /* End work-around */
+
+       i_out = avcodec_encode_video( p_sys->p_context, p_sys->p_buffer_out,
+                                     p_sys->i_buffer_out, &frame );
     }
     else
     {
-        frame.pts = (int64_t)AV_NOPTS_VALUE;
+       i_out = avcodec_encode_video( p_sys->p_context, p_sys->p_buffer_out,
+                                     p_sys->i_buffer_out, NULL);
     }
 
-    if ( frame.pts != (int64_t)AV_NOPTS_VALUE && frame.pts != 0 )
-    {
-        if ( p_sys->i_last_pts == frame.pts )
-        {
-            msg_Warn( p_enc, "almost fed libavcodec with two frames with the "
-                      "same PTS (%"PRId64 ")", frame.pts );
-            return NULL;
-        }
-        else if ( p_sys->i_last_pts > frame.pts )
-        {
-            msg_Warn( p_enc, "almost fed libavcodec with a frame in the "
-                      "past (current: %"PRId64 ", last: %"PRId64")",
-                      frame.pts, p_sys->i_last_pts );
-            return NULL;
-        }
-        else
-        {
-            p_sys->i_last_pts = frame.pts;
-        }
-    }
-
-    frame.quality = p_sys->i_quality;
-
-    /* Ugly work-around for stupid libavcodec behaviour */
-    p_sys->i_framenum++;
-    p_sys->pi_delay_pts[p_sys->i_framenum % MAX_FRAME_DELAY] = frame.pts;
-    frame.pts = p_sys->i_framenum * AV_TIME_BASE *
-        p_enc->fmt_in.video.i_frame_rate_base;
-    frame.pts += p_enc->fmt_in.video.i_frame_rate - 1;
-    frame.pts /= p_enc->fmt_in.video.i_frame_rate;
-    /* End work-around */
-
-    i_out = avcodec_encode_video( p_sys->p_context, p_sys->p_buffer_out,
-                                  p_sys->i_buffer_out, &frame );
-
     if( i_out > 0 )
     {
         block_t *p_block = block_New( p_enc, i_out );

modules/codec/dirac.c

@@ -781,6 +781,7 @@ static block_t *Encode( encoder_t *p_enc, picture_t *p_pic )
     int i_plane, i_line, i_width, i_src_stride;
     uint8_t *p_dst;
 
+    if( !p_pic ) return NULL;
     /* we only know if the sequence is interlaced when the first
      * picture arrives, so final setup is done here */
     /* XXX todo, detect change of interlace */

modules/codec/omxil/omxil.c

@@ -1302,11 +1302,13 @@ static block_t *EncodeVideo( encoder_t *p_enc, picture_t *p_pic )
     OMX_BUFFERHEADERTYPE *p_header;
     block_t *p_block = 0;
 
+    if( !p_pic ) return NULL;
+
     /* Check for errors from codec */
     if(p_sys->b_error)
     {
         msg_Dbg(p_dec, "error during encoding");
-        return 0;
+        return NULL;
     }
 
     /* Send the input buffer to the component */

modules/codec/theora.c

@@ -676,6 +676,7 @@ static block_t *Encode( encoder_t *p_enc, picture_t *p_pict )
     yuv_buffer yuv;
     int i;
 
+    if( !p_pict ) return NULL;
     /* Sanity check */
     if( p_pict->p[0].i_pitch < (int)p_sys->i_width ||
         p_pict->p[0].i_lines < (int)p_sys->i_height )

modules/codec/x264.c

@@ -1292,20 +1292,26 @@ static block_t *Encode( encoder_t *p_enc, picture_t *p_pict )
     x264_picture_t pic;
     x264_nal_t *nal;
     block_t *p_block;
-    int i_nal, i_out, i;
+    int i_nal=0, i_out=0, i=0;
 
     /* init pic */
     memset( &pic, 0, sizeof( x264_picture_t ) );
-    pic.i_pts = p_pict->date;
-    pic.img.i_csp = X264_CSP_I420;
-    pic.img.i_plane = p_pict->i_planes;
-    for( i = 0; i < p_pict->i_planes; i++ )
-    {
-        pic.img.plane[i] = p_pict->p[i].p_pixels;
-        pic.img.i_stride[i] = p_pict->p[i].i_pitch;
-    }
+    if( likely(p_pict) ) {
+       pic.i_pts = p_pict->date;
+       pic.img.i_csp = X264_CSP_I420;
+       pic.img.i_plane = p_pict->i_planes;
+       for( i = 0; i < p_pict->i_planes; i++ )
+       {
+           pic.img.plane[i] = p_pict->p[i].p_pixels;
+           pic.img.i_stride[i] = p_pict->p[i].i_pitch;
+       }
 
-    x264_encoder_encode( p_sys->h, &nal, &i_nal, &pic, &pic );
+       x264_encoder_encode( p_sys->h, &nal, &i_nal, &pic, &pic );
+    } else {
+       if( x264_encoder_delayed_frames( p_sys->h ) ) {
+           x264_encoder_encode( p_sys->h, &nal, &i_nal, NULL, &pic );
+       }
+    }
 
     if( !i_nal ) return NULL;
 

modules/stream_out/transcode/transcode.c

@@ -637,6 +637,7 @@ static int Del( sout_stream_t *p_stream, sout_stream_id_t *id )
             transcode_audio_close( id );
             break;
         case VIDEO_ES:
+            Send( p_stream, id, NULL );
             transcode_video_close( p_stream, id );
             break;
         case SPU_ES:

modules/stream_out/transcode/video.c

@@ -535,6 +535,19 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_t *id,
     picture_t *p_pic, *p_pic2 = NULL;
     *out = NULL;
 
+    if( in == NULL )
+    {
+       block_t *p_block;
+       do {
+           video_timer_start( id->p_encoder );
+           p_block = id->p_encoder->pf_encode_video(id->p_encoder, NULL );
+           video_timer_stop( id->p_encoder );
+           block_ChainAppend( out, p_block );
+       } while( p_block );
+       return VLC_SUCCESS;
+    }
+
+
     while( (p_pic = id->p_decoder->pf_decode_video( id->p_decoder, &in )) )
     {
         subpicture_t *p_subpic = NULL;