Decode Teletext subtitles into fourcc 'RGBA', then convert the chroma to...

Decode Teletext subtitles into fourcc 'RGBA', then convert the chroma to fourcc 'YUVA' using video blending and encode YUVA into YUVP and SPU. To be sent to the muxer. However the last step fails. The data never arrives in the muxer.

The transcode_spu_process() only works OK when used with overlay to burn the subtitle in the video. This needs a complete rewrite in the direction of 'osdmenu' subtitles.

modules/stream_out/transcode.c

@@ -438,6 +438,7 @@ struct decoder_owner_sys_t
     picture_t *pp_pics[PICTURE_RING_SIZE];
     sout_stream_sys_t *p_sys;
 };
+
 struct filter_owner_sys_t
 {
     picture_t *pp_pics[PICTURE_RING_SIZE];
@@ -1222,7 +1223,7 @@ static filter_t *transcode_audio_filter_new( sout_stream_t *p_stream,
     {
         vlc_object_detach( p_filter );
         vlc_object_destroy( p_filter );
-        p_filter = 0;
+        p_filter = NULL;
     }
 
     return p_filter;
@@ -1859,9 +1860,7 @@ static int transcode_video_encoder_open( sout_stream_t *p_stream,
          }
      }
 
-
      /* Change aspect ratio from scaled pixel to output frame */
-
      f_aspect = f_aspect * i_dst_width / i_dst_height;
 
      /* Store calculated values */
@@ -2563,7 +2562,71 @@ static void video_unlink_picture_decoder( decoder_t *p_dec, picture_t *p_pic )
  */
 static subpicture_t *spu_new_buffer( decoder_t * );
 static void spu_del_buffer( decoder_t *, subpicture_t * );
+#if 0
+static filter_t *transcode_spu_filter_new( sout_stream_t *p_stream,
+                                           sout_stream_id_t *id,
+                                           es_format_t *p_fmt_in,
+                                           es_format_t *p_fmt_out )
+{
+    int i;
+    sout_stream_sys_t *p_sys = p_stream->p_sys;
+    filter_t *p_filter = vlc_object_create( p_stream, VLC_OBJECT_FILTER );
+
+    vlc_object_attach( p_filter, p_stream );
+    p_filter->pf_vout_buffer_new = video_new_buffer_filter;
+    p_filter->pf_vout_buffer_del = video_del_buffer_filter;
+
+    p_filter->fmt_in = *p_fmt_in;
+    p_filter->fmt_out = *p_fmt_out;
+
+    p_filter->p_module = module_Need( p_filter, "video filter2", 0, 0 );
+    if( p_filter->p_module )
+    {
+        p_filter->p_owner = malloc( sizeof(filter_owner_sys_t) );
+        if( !p_filter->p_owner )
+        {
+            msg_Err( p_stream, "out of memory" );
+
+            module_Unneed( p_filter, p_filter->p_module );
+            vlc_object_detach( p_filter );
+            vlc_object_destroy( p_filter );
+            return NULL;
+        }
+        for( i = 0; i < PICTURE_RING_SIZE; i++ )
+             p_filter->p_owner->pp_pics[i] = NULL;
+        p_filter->p_owner->p_sys = p_sys;
 
+        *p_fmt_in = p_filter->fmt_out;
+    }
+    else
+    {
+        vlc_object_detach( p_filter );
+        vlc_object_destroy( p_filter );
+        p_filter = NULL;
+    }
+    return p_filter;
+}
+
+static void transcode_spu_filter_del( sout_stream_t *p_stream, filter_t *p_filter )
+{
+    int j;
+    vlc_object_detach( p_filter );
+    if( p_filter->p_module )
+        module_Unneed( p_filter, p_filter->p_module );
+
+    /* Clean-up pictures ring buffer */
+    for( j = 0; j < PICTURE_RING_SIZE; j++ )
+    {
+        if( p_filter->p_owner->pp_pics[j] )
+            video_del_buffer( VLC_OBJECT(p_filter),
+                              p_filter->p_owner->pp_pics[j] );
+    }
+    free( p_filter->p_owner );
+
+    vlc_object_destroy( p_filter );
+    p_filter = NULL;
+}
+#endif
 static int transcode_spu_new( sout_stream_t *p_stream, sout_stream_id_t *id )
 {
     sout_stream_sys_t *p_sys = p_stream->p_sys;
@@ -2594,9 +2657,13 @@ static int transcode_spu_new( sout_stream_t *p_stream, sout_stream_id_t *id )
          */
 
         /* Initialization of encoder format structures */
-        es_format_Init( &id->p_encoder->fmt_in, id->p_decoder->fmt_in.i_cat,
+        es_format_Init( &id->p_encoder->fmt_in, SPU_ES,
                         id->p_decoder->fmt_in.i_codec );
 
+        id->p_encoder->fmt_out.i_id    = 0xbd1;
+        id->p_encoder->fmt_out.i_group = 3;
+        id->p_encoder->fmt_out.psz_language = strdup( "unknown" );
+
         id->p_encoder->p_cfg = p_sys->p_spu_cfg;
 
         id->p_encoder->p_module =
@@ -2640,6 +2707,7 @@ static int transcode_spu_process( sout_stream_t *p_stream,
 
     p_subpic = id->p_decoder->pf_decode_sub( id->p_decoder, &in );
     if( !p_subpic ) return VLC_EGENERIC;
+    if( !p_subpic->p_region ) return VLC_EGENERIC;
 
     if( p_sys->b_master_sync && p_sys->i_master_drift )
     {
@@ -2647,22 +2715,134 @@ static int transcode_spu_process( sout_stream_t *p_stream,
         if( p_subpic->i_stop ) p_subpic->i_stop -= p_sys->i_master_drift;
     }
 
+    /* Do we need a chroma conversion? */
+    if( ( id->p_decoder->fmt_out.video.i_chroma != VLC_FOURCC('Y','U','V','A') ) ||
+        ( id->p_decoder->fmt_out.video.i_chroma != VLC_FOURCC('Y','U','V','P') ) )
+    {
+        subpicture_region_t *p_region = p_subpic->p_region;
+        subpicture_region_t *p_list = NULL, *p_tail = NULL, *p_old = p_region;
+
+        if( id->p_decoder->fmt_out.video.i_chroma == VLC_FOURCC('R','G','B','A') )
+            id->p_encoder->fmt_in.video.i_chroma = VLC_FOURCC('Y','U','V','A');
+        else
+            id->p_encoder->fmt_in.video.i_chroma = VLC_FOURCC('Y','U','V','P');
+
+        memcpy( &id->p_decoder->fmt_out, &p_region->fmt, sizeof(video_format_t) );
+        id->p_encoder->fmt_in.video.i_aspect = VOUT_ASPECT_FACTOR;
+
+        /* Load the blending module */
+        if( !p_sys->p_spu->p_blend && p_region )
+        {
+            p_sys->p_spu->p_blend = vlc_object_create( p_sys->p_spu, VLC_OBJECT_FILTER );
+            vlc_object_attach( p_sys->p_spu->p_blend, p_sys->p_spu );
+            p_sys->p_spu->p_blend->fmt_out.video.i_x_offset =
+                p_sys->p_spu->p_blend->fmt_out.video.i_y_offset = 0;
+            p_sys->p_spu->p_blend->fmt_out.video.i_aspect =
+                id->p_encoder->fmt_in.video.i_aspect;
+            p_sys->p_spu->p_blend->fmt_out.video.i_chroma =
+                id->p_encoder->fmt_in.video.i_chroma;
+            p_sys->p_spu->p_blend->fmt_in.video.i_chroma =
+                p_region->fmt.i_chroma; // id->p_decoder->fmt_out.video.i_chroma;
+
+            p_sys->p_spu->p_blend->p_module =
+                module_Need( p_sys->p_spu->p_blend, "video blending", NULL, VLC_FALSE );
+        }
+
+        while( p_region && p_sys->p_spu->p_blend &&
+               p_sys->p_spu->p_blend->pf_video_blend )
+        {
+            subpicture_region_t *p_new = NULL;
+            video_format_t  fmt;
+
+            /* Create a new subpicture region */
+            memset( &fmt, 0, sizeof(video_format_t) );
+            fmt.i_chroma = VLC_FOURCC('Y','U','V','A');
+            fmt.i_aspect = VOUT_ASPECT_FACTOR;
+            fmt.i_sar_num = fmt.i_sar_den = 1;
+            fmt.i_width = fmt.i_visible_width = p_region->fmt.i_visible_width;
+            fmt.i_height = fmt.i_visible_height = p_region->fmt.i_visible_height;
+            fmt.i_bits_per_pixel = p_region->fmt.i_bits_per_pixel;
+            fmt.i_y_offset = p_region->fmt.i_y_offset;
+            fmt.i_x_offset = p_region->fmt.i_x_offset;
+
+            p_new = p_subpic->pf_create_region( VLC_OBJECT(id->p_decoder), &fmt );
+            if( p_new == NULL )
+            {
+                msg_Err( p_stream, "out of memory" );
+                break;
+            }
+            p_new->i_x = 0;
+            p_new->i_y = 0;
+            p_new->i_align = SUBPICTURE_ALIGN_TOP;
+
+            /* Force palette if requested */
+            if( p_sys->p_spu->b_force_palette &&
+                (VLC_FOURCC('Y','U','V','P') == p_region->fmt.i_chroma) )
+            {
+                memcpy( p_region->fmt.p_palette->palette,
+                        p_sys->p_spu->palette, 16 );
+            }
+            p_sys->p_spu->p_blend->fmt_in.video = p_region->fmt;
+
+            /* Update the output picture size */
+            p_sys->p_spu->p_blend->fmt_out.video.i_width =
+                p_sys->p_spu->p_blend->fmt_out.video.i_visible_width =
+                    id->p_decoder->fmt_out.video.i_width;
+            p_sys->p_spu->p_blend->fmt_out.video.i_height =
+                p_sys->p_spu->p_blend->fmt_out.video.i_visible_height =
+                    id->p_decoder->fmt_out.video.i_height;
+
+            p_sys->p_spu->p_blend->pf_video_blend( p_sys->p_spu->p_blend,
+                    &p_new->picture, &p_new->picture, &p_region->picture,
+                    p_subpic->i_x, p_subpic->i_y, 0 );
+
+            if( !p_list )
+            {
+                p_list = p_new;
+                p_tail = p_new;
+            }
+            else
+            {
+                p_tail->p_next = p_new;
+                p_tail = p_new;
+            }
+            p_region = p_region->p_next;
+        }
+        if( p_list )
+        {
+            p_subpic->pf_destroy_region( VLC_OBJECT(id->p_decoder), p_old );
+            p_subpic->p_region = p_list;
+        }
+    }
+
     if( p_sys->b_soverlay )
     {
         spu_DisplaySubpicture( p_sys->p_spu, p_subpic );
     }
     else
     {
-        block_t *p_block;
-
+        block_t *p_block = NULL;
+#if 1
         p_block = id->p_encoder->pf_encode_sub( id->p_encoder, p_subpic );
         spu_del_buffer( id->p_decoder, p_subpic );
-
         if( p_block )
         {
             block_ChainAppend( out, p_block );
             return VLC_SUCCESS;
         }
+#else
+        if( p_block )
+        {
+            p_block->i_dts = p_block->i_pts = p_subpic->i_start;
+            block_ChainAppend( out, p_block );
+            if( *out )
+            {
+                if( p_sys->p_out->pf_send( p_sys->p_out, id, *out ) == VLC_SUCCESS )
+                    spu_del_buffer( id->p_decoder, p_subpic );
+            }
+            return VLC_SUCCESS;
+        }
+#endif
     }
 
     return VLC_EGENERIC;
@@ -2695,6 +2875,11 @@ static int transcode_osd_new( sout_stream_t *p_stream, sout_stream_id_t *id )
     fmt.psz_language = strdup( "osd" );
 
     id = malloc( sizeof( sout_stream_id_t ) );
+    if( !id )
+    {
+        msg_Err( p_stream, "out of memory" );
+        goto error;
+    }
     memset( id, 0, sizeof(sout_stream_id_t) );
 
     id->id = NULL;