Resizer: fix coeffs, based on TI input

Signed-off-by: Jean-Paul Saman <jean-paul.saman@m2x.nl>

modules/codec/davinci/viddec.c

@@ -135,22 +135,24 @@ static void Resize_coeff( decoder_t *p_dec,
     /* choose the minimum for both axes */
     i_hratio = i_wratio = i_hratio < i_wratio ? i_hratio : i_wratio;
 
-    msg_Dbg( p_dec, "Final resolution %dx%d (%.2f , %.2f)",
-        i_vidw * i_wratio / VOUT_ASPECT_FACTOR,
-        i_vidh * i_hratio / VOUT_ASPECT_FACTOR,
-        (float)i_wratio / VOUT_ASPECT_FACTOR,
-        (float)i_hratio / VOUT_ASPECT_FACTOR );
-
-    /* and set the new resolution accordingly */
-    *i_height = (i_vidh * i_hratio) / VOUT_ASPECT_FACTOR;
-    *i_width = (i_vidw * i_wratio) / VOUT_ASPECT_FACTOR;
-
-    /* width must be a factor of 8 : davinci resizer limitation when doing
-     * vertical upscaling, 16 bytes per output line and Bpp is 2 */
-    if( i_wratio > 1 && *i_width % 8 )
-        *i_width = ( *i_width + 7 ) & ~7;
-
-    msg_Dbg( p_dec, "Width fixed to %d\n", *i_width );
+    /* Now calculate hrsz and vrsz */
+    int i_hrsz = 256 * VOUT_ASPECT_FACTOR / i_wratio;
+    int i_vrsz = 256 * VOUT_ASPECT_FACTOR / i_hratio;
+
+    /* Then recalculate final resolution */
+    int i_htap = i_hrsz > 512 ? 7 : 4;
+    int i_vtap = i_vrsz > 512 ? 7 : 4;
+
+    *i_width = ((i_vidw - 7) * 256 - (i_htap ==  7 ? 32 : 16)) / i_hrsz + 1;
+    *i_height = ((i_vidh - i_vtap) * 256 - (i_vtap ==  7 ? 32 : 16)) / i_vrsz + 1;
+
+    /* width must be a factor of 16 : davinci resizer limitation when doing
+     * vertical upscaling, 16 bytes per output line
+     * the bpp is 2 so a factor of 8 should be good but not in practice
+     */
+    *i_width = ( *i_width + 15 ) & ~15;
+
+    msg_Dbg( p_dec, "Destination fixed to %dx%d", *i_width, *i_height );
 }
 
 /* open() a framebuffer by its name (NOT device path) */
@@ -583,10 +585,8 @@ static void Resize( decoder_t *p_dec, vlc_bool_t b_scale )
 
     if( !p_sys->i_yuyv )
     {
-        p_sys->i_yuyv = p_dec->fmt_out.video.i_width * 
-                p_dec->fmt_out.video.i_height * BPP;
-
-        msg_Dbg( p_dec, "Allocating %d bytes buffer", p_sys->i_yuyv );
+        /* use screen resolution for the buffer, we may be off by a few */
+        p_sys->i_yuyv = p_sys->var_info.xres * p_sys->var_info.yres_virtual;
         assert(p_sys->i_yuyv);
 
         rsz_reqbufs_t reqbufs;
@@ -615,9 +615,6 @@ static void Resize( decoder_t *p_dec, vlc_bool_t b_scale )
 
         p_sys->offset = buffer.offset;
 
-        printf( "mmap(NULL, %d, PROT_READ|PROT_WRITE, MAP_SHARED, %d, %zd)\n",
-            reqbufs.size, p_sys->i_fd_resizer, buffer.offset
-            );
         p_sys->p_yuyv = mmap( NULL, reqbufs.size, PROT_READ|PROT_WRITE, 
                  MAP_SHARED, p_sys->i_fd_resizer, buffer.offset );
 
@@ -642,9 +639,9 @@ static void Resize( decoder_t *p_dec, vlc_bool_t b_scale )
         rsz_params.inptyp = RSZ_INTYPE_YCBCR422_16BIT;
         rsz_params.pix_fmt = RSZ_PIX_FMT_YUYV;
 
+        /* Sets the desired maximum destination resolution */
         if( b_scale )
         {
-            /* Sets the destination resolution */
             rsz_params.out_hsize = p_sys->var_info.xres;
             rsz_params.out_vsize = p_sys->var_info.yres;
         }
@@ -654,7 +651,8 @@ static void Resize( decoder_t *p_dec, vlc_bool_t b_scale )
             rsz_params.out_vsize = rsz_params.in_vsize;
         }
 
-        /* And then modify it to keep the same aspect ratio */
+        /* And then modify it to keep the same aspect ratio, and make sure
+         * that the hardware is able to handle it */
         Resize_coeff( p_dec, &rsz_params.out_hsize, &rsz_params.out_vsize );
 
         /* RSZ_PIX_FMT_YUYV is 2 bytes per pixel */
@@ -733,7 +731,7 @@ static void Resize( decoder_t *p_dec, vlc_bool_t b_scale )
     rsz.out_buf.index    = -1;
     rsz.out_buf.buf_type = RSZ_BUF_OUT;
 
-    if( b_scale && p_sys->b_direct )
+    if( b_scale && p_sys->b_direct /* output video res is screen res */ )
     {
         /* FIXME: use NUM_BUFFERS if needed */
         rsz.out_buf.offset   = p_sys->p_physbufs[0];