Preliminary changes to fix postprocessing. Decoder quantizer settings can now...

Preliminary changes to fix postprocessing. Decoder quantizer settings can now be exported in the picture_t struct. Currently only works if the video is being decoded by ffmpeg since other codecs don't expose quantizer stuff. Please review.

include/vlc_vout.h

@@ -100,6 +100,9 @@ struct picture_t
     bool            b_progressive;          /**< is it a progressive frame ? */
     unsigned int    i_nb_fields;                  /**< # of displayed fields */
     bool            b_top_field_first;             /**< which field is first */
+    uint8_t        *p_q;                           /**< quantification table */
+    int             i_qstride;                    /**< quantification stride */
+    int             i_qtype;                       /**< quantification style */
     /**@}*/
 
     /** The picture heap we are attached to */
@@ -168,6 +171,8 @@ static inline void picture_CopyProperties( picture_t *p_dst, const picture_t *p_
     p_dst->b_progressive = p_src->b_progressive;
     p_dst->i_nb_fields = p_src->i_nb_fields;
     p_dst->b_top_field_first = p_src->b_top_field_first;
+
+    /* FIXME: copy ->p_q and ->p_qstride */
 }
 
 /**
@@ -240,6 +245,11 @@ struct picture_heap_t
 #define DISPLAYED_PICTURE       5            /* been displayed but is linked */
 #define DESTROYED_PICTURE       6              /* allocated but no more used */
 
+/* Quantification type */
+#define QTYPE_MPEG1            0
+#define QTYPE_MPEG2            1
+#define QTYPE_H264             2
+
 /*****************************************************************************
  * Shortcuts to access image components
  *****************************************************************************/

modules/codec/avcodec/video.c

@@ -552,7 +552,7 @@ picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
         p_sys->i_buffer -= i_used;
         p_sys->p_buffer += i_used;
 
-    p_sys->b_first_frame = true;
+        p_sys->b_first_frame = true;
 
         /* Nothing to display */
         if( !b_gotpicture )
@@ -636,6 +636,7 @@ picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
         /* Send decoded frame to vout */
         if( p_sys->i_pts )
         {
+            int i;
             p_pic->date = p_sys->i_pts;
 
             ffmpeg_NextPts( p_dec, p_block->i_rate );
@@ -651,6 +652,29 @@ picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
             p_pic->b_progressive = !p_sys->p_ff_pic->interlaced_frame;
             p_pic->b_top_field_first = p_sys->p_ff_pic->top_field_first;
 
+            p_pic->i_qstride = p_sys->p_ff_pic->qstride;
+#if 1
+            p_pic->p_q = p_sys->p_ff_pic->qscale_table; /* XXX: is this dangerous? shouldn't be since the ff pics are never freed ... but you never know */
+#else
+            /* FIXME: this leaks p_q */
+            int i_mb_h = ( p_pic->format.i_height + 15 ) / 16;
+            p_pic->p_q = malloc( p_pic->i_qstride * i_mb_h );
+            memcpy( p_pic->p_q, p_sys->p_ff_pic->qscale_table,
+                    p_pic->i_qstride * i_mb_h );
+#endif
+            switch( p_sys->p_ff_pic->qscale_type )
+            {
+                case FF_QSCALE_TYPE_MPEG1:
+                    p_pic->i_qtype = QTYPE_MPEG1;
+                    break;
+                case FF_QSCALE_TYPE_MPEG2:
+                    p_pic->i_qtype = QTYPE_MPEG2;
+                    break;
+                case FF_QSCALE_TYPE_H264:
+                    p_pic->i_qtype = QTYPE_H264;
+                    break;
+            }
+
             return p_pic;
         }
         else

modules/video_filter/postproc.c

@@ -102,6 +102,8 @@ struct filter_sys_t
     pp_context_t *pp_context; /* Never changes after init */
     pp_mode_t    *pp_mode; /* Set to NULL if post processing is disabled */
 
+    bool b_had_matrix; /* Set to true if previous pic had a quant matrix (used to prevent spamming warning messages */
+
     vlc_mutex_t lock; /* Lock when using or changing pp_mode */
 };
 
@@ -139,6 +141,7 @@ static int OpenPostproc( vlc_object_t *p_this )
     {
         case VLC_FOURCC('I','4','4','4'):
         case VLC_FOURCC('J','4','4','4'):
+        /* case VLC_FOURCC('Y','U','V','A'): FIXME Should work but alpha plane needs to be copied manually and I'm kind of feeling too lazy to write the code to do that ATM (i_pitch vs i_visible_pitch...). */
             i_flags |= PP_FORMAT_444;
             break;
         case VLC_FOURCC('I','4','2','2'):
@@ -152,7 +155,6 @@ static int OpenPostproc( vlc_object_t *p_this )
         case VLC_FOURCC('I','Y','U','V'):
         case VLC_FOURCC('J','4','2','0'):
         case VLC_FOURCC('Y','V','1','2'):
-        /* case VLC_FOURCC('Y','U','V','A'): FIXME Should work but alpha plane needs to be copied manually and I'm kind of feeling too lazy to write the code to do that ATM (i_pitch vs i_visible_pitch...). */
             i_flags |= PP_FORMAT_420;
             break;
         default:
@@ -238,6 +240,7 @@ static int OpenPostproc( vlc_object_t *p_this )
     vlc_mutex_init( &p_sys->lock );
 
     p_filter->pf_video_filter = PostprocPict;
+    p_sys->b_had_matrix = true;
 
     return VLC_SUCCESS;
 }
@@ -293,13 +296,22 @@ static picture_t *PostprocPict( filter_t *p_filter, picture_t *p_pic )
         i_dst_stride[i_plane] = p_outpic->p[i_plane].i_pitch;
     }
 
+    if( !p_pic->p_q && p_sys->b_had_matrix )
+    {
+        msg_Warn( p_filter, "Quantification table was not set by video decoder. Postprocessing won't look good." );
+        p_sys->b_had_matrix = false;
+    }
+    else if( p_pic->p_q )
+    {
+        p_sys->b_had_matrix = true;
+    }
+
     pp_postprocess( src, i_src_stride, dst, i_dst_stride,
                     p_filter->fmt_in.video.i_width,
                     p_filter->fmt_in.video.i_height,
-                    NULL /* FIXME ? works by selecting a default table. But maybe setting our own might help improve post processing quality ... */,
-                    0 /* FIXME */,
+                    p_pic->p_q, p_pic->i_qstride,
                     p_sys->pp_mode, p_sys->pp_context,
-                    PP_PICT_TYPE_QP2 /* FIXME ? This should be set only for mpeg2 type codecs if I understand correctly. */ );
+                    p_pic->i_qtype == QTYPE_MPEG2 ? PP_PICT_TYPE_QP2 : 0 );
     vlc_mutex_unlock( &p_sys->lock );
 
     return CopyInfoAndRelease( p_outpic, p_pic );

src/video_output/vout_pictures.c

@@ -681,6 +681,10 @@ int __vout_InitPicture( vlc_object_t *p_this, picture_t *p_pic,
     p_pic->pf_unlock = 0;
     p_pic->i_refcount = 0;
 
+    p_pic->p_q = NULL;
+    p_pic->i_qstride = 0;
+    p_pic->i_qtype = 0;
+
     vout_InitFormat( &p_pic->format, i_chroma, i_width, i_height, i_aspect );
 
     /* Make sure the real dimensions are a multiple of 16 */