dxva2: convert D3D9 textures to planes using an external filter

Signed-off-by: Jean-Baptiste Kempf <jb@videolan.org>

modules/MODULES_LIST

@@ -119,6 +119,7 @@ $Id$
  * dvbsub: decoder module for subs in dvb streams
  * dvdnav: access module for DVDs with libdvdnav
  * dvdread: input module for accessing DVDs, uses libdvdread
+ * dxa9: Convert D3D9 GPU textures to YUV planes
  * dxva2: DxVA2 hardware-accelerated decoding
  * dynamicoverlay: subpicture filter using shared memory that can be written to by external applications
  * edummy: dummy encoder

modules/codec/avcodec/dxva2.c

@@ -31,6 +31,8 @@
 
 #include <vlc_common.h>
 #include <vlc_picture.h>
+#include <vlc_filter.h> /* need for the DXA9 to YV12 conversion */
+#include <vlc_modules.h>
 #include <vlc_plugin.h>
 
 #include "directx_va.h"
@@ -124,6 +126,9 @@ struct vlc_va_sys_t
     /* Video decoder */
     DXVA2_ConfigPictureDecode    cfg;
 
+    /* Option conversion */
+    filter_t                 *filter;
+
     /* avcodec internals */
     struct dxva_context hw;
 };
@@ -155,6 +160,49 @@ static void DxDestroyVideoDecoder(vlc_va_t *);
 static int DxResetVideoDecoder(vlc_va_t *);
 static void SetupAVCodecContext(vlc_va_t *);
 
+static void DeleteFilter( filter_t * p_filter )
+{
+    if( p_filter->p_module )
+        module_unneed( p_filter, p_filter->p_module );
+
+    es_format_Clean( &p_filter->fmt_in );
+    es_format_Clean( &p_filter->fmt_out );
+
+    vlc_object_release( p_filter );
+}
+
+static picture_t *video_new_buffer(filter_t *p_filter)
+{
+    return p_filter->owner.sys;
+}
+
+static filter_t *CreateFilter( vlc_object_t *p_this, const es_format_t *p_fmt_in,
+                               vlc_fourcc_t fmt_out )
+{
+    filter_t *p_filter;
+
+    p_filter = vlc_object_create( p_this, sizeof(filter_t) );
+    if (unlikely(p_filter == NULL))
+        return NULL;
+
+    p_filter->owner.video.buffer_new = (picture_t *(*)(filter_t *))video_new_buffer;
+
+    es_format_InitFromVideo( &p_filter->fmt_in,  &p_fmt_in->video );
+    es_format_InitFromVideo( &p_filter->fmt_out, &p_fmt_in->video );
+    p_filter->fmt_in.i_codec  = p_filter->fmt_in.video.i_chroma  = VLC_CODEC_D3D9_OPAQUE;
+    p_filter->fmt_out.i_codec = p_filter->fmt_out.video.i_chroma = fmt_out;
+    p_filter->p_module = module_need( p_filter, "video filter2", NULL, false );
+
+    if( !p_filter->p_module )
+    {
+        msg_Dbg( p_filter, "no video filter found" );
+        DeleteFilter( p_filter );
+        return NULL;
+    }
+
+    return p_filter;
+}
+
 /* */
 static int Setup(vlc_va_t *va, AVCodecContext *avctx, vlc_fourcc_t *chroma)
 {
@@ -163,7 +211,7 @@ static int Setup(vlc_va_t *va, AVCodecContext *avctx, vlc_fourcc_t *chroma)
         return VLC_EGENERIC;
 
     avctx->hwaccel_context = &sys->hw;
-    *chroma = VLC_CODEC_D3D9_OPAQUE;
+    *chroma = sys->filter == NULL ? VLC_CODEC_D3D9_OPAQUE : VLC_CODEC_YV12;
 
     return VLC_SUCCESS;
 }
@@ -186,26 +234,38 @@ static int Extract(vlc_va_t *va, picture_t *picture, uint8_t *data)
 {
     directx_sys_t *dx_sys = &va->sys->dx_sys;
     LPDIRECT3DSURFACE9 d3d = (LPDIRECT3DSURFACE9)(uintptr_t)data;
-    picture_sys_t *p_sys = picture->p_sys;
-    LPDIRECT3DSURFACE9 output = p_sys->surface;
+    if (picture->format.i_chroma == VLC_CODEC_D3D9_OPAQUE)
+    {
+        picture_sys_t *p_sys = picture->p_sys;
+        LPDIRECT3DSURFACE9 output = p_sys->surface;
 
-    assert(d3d != output);
+        assert(d3d != output);
 #ifndef NDEBUG
-    LPDIRECT3DDEVICE9 srcDevice, dstDevice;
-    IDirect3DSurface9_GetDevice(d3d, &srcDevice);
-    IDirect3DSurface9_GetDevice(output, &dstDevice);
-    assert(srcDevice == dstDevice);
+        LPDIRECT3DDEVICE9 srcDevice, dstDevice;
+        IDirect3DSurface9_GetDevice(d3d, &srcDevice);
+        IDirect3DSurface9_GetDevice(output, &dstDevice);
+        assert(srcDevice == dstDevice);
 #endif
 
-    HRESULT hr;
-    RECT visibleSource;
-    visibleSource.left = 0;
-    visibleSource.top = 0;
-    visibleSource.right = picture->format.i_visible_width;
-    visibleSource.bottom = picture->format.i_visible_height;
-    hr = IDirect3DDevice9_StretchRect( (IDirect3DDevice9*) dx_sys->d3ddev, d3d, &visibleSource, output, &visibleSource, D3DTEXF_NONE);
-    if (FAILED(hr)) {
-        msg_Err(va, "Failed to copy the hw surface to the decoder surface (hr=0x%0lx)", hr );
+        HRESULT hr;
+        RECT visibleSource;
+        visibleSource.left = 0;
+        visibleSource.top = 0;
+        visibleSource.right = picture->format.i_visible_width;
+        visibleSource.bottom = picture->format.i_visible_height;
+        hr = IDirect3DDevice9_StretchRect( (IDirect3DDevice9*) dx_sys->d3ddev, d3d, &visibleSource, output, &visibleSource, D3DTEXF_NONE);
+        if (FAILED(hr)) {
+            msg_Err(va, "Failed to copy the hw surface to the decoder surface (hr=0x%0lx)", hr );
+            return VLC_EGENERIC;
+        }
+    }
+    else if (va->sys->filter != NULL) {
+        va->sys->filter->owner.sys = picture;
+        vlc_va_surface_t *surface = picture->context;
+        picture_Hold( surface->p_pic );
+        va->sys->filter->pf_video_filter( va->sys->filter, surface->p_pic );
+    } else {
+        msg_Err(va, "Unsupported output picture format %08X", picture->format.i_chroma );
         return VLC_EGENERIC;
     }
 
@@ -239,6 +299,12 @@ static void Close(vlc_va_t *va, AVCodecContext *ctx)
 
     (void) ctx;
 
+    if (sys->filter)
+    {
+        DeleteFilter( sys->filter );
+        sys->filter = NULL;
+    }
+
     directx_va_Close(va, &sys->dx_sys);
 
     if (sys->hd3d9_dll)
@@ -257,8 +323,6 @@ static int Open(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
     if (pix_fmt != AV_PIX_FMT_DXVA2_VLD)
         return VLC_EGENERIC;
 
-    (void) p_sys;
-
     vlc_va_sys_t *sys = calloc(1, sizeof (*sys));
     if (unlikely(sys == NULL))
         return VLC_ENOMEM;
@@ -297,6 +361,12 @@ static int Open(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
     if (err!=VLC_SUCCESS)
         goto error;
 
+    if (p_sys == NULL)
+    {
+        sys->filter = CreateFilter( VLC_OBJECT(va), fmt, VLC_CODEC_YV12);
+        if (sys->filter == NULL)
+            goto error;
+    }
 
     /* TODO print the hardware name/vendor for debugging purposes */
     va->description = DxDescribe(sys);

modules/codec/avcodec/va.c

@@ -41,7 +41,8 @@ vlc_fourcc_t vlc_va_GetChroma(enum PixelFormat hwfmt, enum PixelFormat swfmt)
             return VLC_CODEC_YV12;
 
         case AV_PIX_FMT_DXVA2_VLD:
-            return VLC_CODEC_YV12;
+            return VLC_CODEC_D3D9_OPAQUE;
+
 #if (LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(53, 14, 0))
         case AV_PIX_FMT_VDA_VLD:
             return VLC_CODEC_UYVY;

modules/video_chroma/Makefile.am

@@ -97,3 +97,12 @@ chroma_LTLIBRARIES += \
 	libi420_yuy2_sse2_plugin.la \
 	libi422_yuy2_sse2_plugin.la
 endif
+
+# DXVA2
+libdxa9_plugin_la_SOURCES = video_chroma/dxa9.c \
+	video_chroma/copy.c video_chroma/copy.h
+
+if HAVE_AVCODEC_DXVA2
+chroma_LTLIBRARIES += \
+	libdxa9_plugin.la
+endif

modules/video_chroma/dxa9.c

+/*****************************************************************************
+ * dxa9.c : DXVA2 GPU surface conversion module for vlc
+ *****************************************************************************
+ * Copyright (C) 2015 VLC authors, VideoLAN and VideoLabs
+ * $Id$
+ *
+ * Authors: Steve Lhomme <robux4@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+
+/*****************************************************************************
+ * Preamble
+ *****************************************************************************/
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <vlc_common.h>
+#include <vlc_plugin.h>
+#include <vlc_filter.h>
+
+#include "copy.h"
+
+static int  OpenConverter( vlc_object_t * );
+static void CloseConverter( vlc_object_t * );
+
+/*****************************************************************************
+ * Module descriptor.
+ *****************************************************************************/
+vlc_module_begin ()
+    set_description( N_("Conversions from DxVA2 to YUV") )
+    set_capability( "video filter2", 10 )
+    set_callbacks( OpenConverter, CloseConverter )
+vlc_module_end ()
+
+#include <windows.h>
+#include <d3d9.h>
+
+struct picture_sys_t
+{
+    LPDIRECT3DSURFACE9 surface;
+};
+
+static bool GetLock(filter_t *p_filter, LPDIRECT3DSURFACE9 d3d,
+                    D3DLOCKED_RECT *p_lock, D3DSURFACE_DESC *p_desc)
+{
+    if (FAILED( IDirect3DSurface9_GetDesc(d3d, p_desc)))
+        return false;
+
+    /* */
+    if (FAILED(IDirect3DSurface9_LockRect(d3d, p_lock, NULL, D3DLOCK_READONLY))) {
+        msg_Err(p_filter, "Failed to lock surface");
+        return false;
+    }
+
+    return true;
+}
+
+static void DXA9_YV12(filter_t *p_filter, picture_t *src, picture_t *dst)
+{
+    copy_cache_t *p_copy_cache = (copy_cache_t*) p_filter->p_sys;
+
+    D3DSURFACE_DESC desc;
+    D3DLOCKED_RECT lock;
+    if (!GetLock(p_filter, src->p_sys->surface, &lock, &desc))
+        return;
+
+    if (dst->format.i_chroma == VLC_CODEC_I420) {
+        uint8_t *tmp = dst->p[1].p_pixels;
+        dst->p[1].p_pixels = dst->p[2].p_pixels;
+        dst->p[2].p_pixels = tmp;
+    }
+
+    if (desc.Format == MAKEFOURCC('Y','V','1','2') ||
+        desc.Format == MAKEFOURCC('I','M','C','3')) {
+        bool imc3 = desc.Format == MAKEFOURCC('I','M','C','3');
+        size_t chroma_pitch = imc3 ? lock.Pitch : (lock.Pitch / 2);
+
+        size_t pitch[3] = {
+            lock.Pitch,
+            chroma_pitch,
+            chroma_pitch,
+        };
+
+        uint8_t *plane[3] = {
+            (uint8_t*)lock.pBits,
+            (uint8_t*)lock.pBits + pitch[0] * src->format.i_height,
+            (uint8_t*)lock.pBits + pitch[0] * src->format.i_height
+                                 + pitch[1] * src->format.i_height / 2,
+        };
+
+        if (imc3) {
+            uint8_t *V = plane[1];
+            plane[1] = plane[2];
+            plane[2] = V;
+        }
+        CopyFromYv12(dst, plane, pitch, src->format.i_width,
+                     src->format.i_height, p_copy_cache);
+    } else if (desc.Format == MAKEFOURCC('N','V','1','2')) {
+        uint8_t *plane[2] = {
+            lock.pBits,
+            (uint8_t*)lock.pBits + lock.Pitch * src->format.i_height
+        };
+        size_t  pitch[2] = {
+            lock.Pitch,
+            lock.Pitch,
+        };
+        CopyFromNv12(dst, plane, pitch, src->format.i_width,
+                     src->format.i_height, p_copy_cache);
+    } else {
+        msg_Err(p_filter, "Unsupported DXA9 conversion from 0x%08X to YV12", desc.Format);
+    }
+
+    if (dst->format.i_chroma == VLC_CODEC_I420) {
+        uint8_t *tmp = dst->p[1].p_pixels;
+        dst->p[1].p_pixels = dst->p[2].p_pixels;
+        dst->p[2].p_pixels = tmp;
+    }
+
+    /* */
+    IDirect3DSurface9_UnlockRect(src->p_sys->surface);
+}
+
+static void DXA9_NV12(filter_t *p_filter, picture_t *src, picture_t *dst)
+{
+    copy_cache_t *p_copy_cache = (copy_cache_t*) p_filter->p_sys;
+
+    D3DSURFACE_DESC desc;
+    D3DLOCKED_RECT lock;
+    if (!GetLock(p_filter, src->p_sys->surface, &lock, &desc))
+        return;
+
+    if (desc.Format == MAKEFOURCC('N','V','1','2')) {
+        uint8_t *plane[2] = {
+            lock.pBits,
+            (uint8_t*)lock.pBits + lock.Pitch * src->format.i_height
+        };
+        size_t  pitch[2] = {
+            lock.Pitch,
+            lock.Pitch,
+        };
+        CopyFromNv12ToNv12(dst, plane, pitch, src->format.i_width,
+                           src->format.i_height, p_copy_cache);
+    } else {
+        msg_Err(p_filter, "Unsupported DXA9 conversion from 0x%08X to NV12", desc.Format);
+    }
+
+    /* */
+    IDirect3DSurface9_UnlockRect(src->p_sys->surface);
+}
+
+VIDEO_FILTER_WRAPPER (DXA9_YV12)
+VIDEO_FILTER_WRAPPER (DXA9_NV12)
+
+static int OpenConverter( vlc_object_t *obj )
+{
+    filter_t *p_filter = (filter_t *)obj;
+    if ( p_filter->fmt_in.video.i_chroma != VLC_CODEC_D3D9_OPAQUE )
+        return VLC_EGENERIC;
+
+    if ( p_filter->fmt_in.video.i_height != p_filter->fmt_out.video.i_height
+         || p_filter->fmt_in.video.i_width != p_filter->fmt_out.video.i_width )
+        return VLC_EGENERIC;
+
+    switch( p_filter->fmt_out.video.i_chroma ) {
+    case VLC_CODEC_I420:
+    case VLC_CODEC_YV12:
+        p_filter->pf_video_filter = DXA9_YV12_Filter;
+        break;
+    case VLC_CODEC_NV12:
+        p_filter->pf_video_filter = DXA9_NV12_Filter;
+        break;
+    default:
+        return VLC_EGENERIC;
+    }
+
+    copy_cache_t *p_copy_cache = calloc(1, sizeof(*p_copy_cache));
+    if (!p_copy_cache)
+         return VLC_ENOMEM;
+    CopyInitCache(p_copy_cache, p_filter->fmt_in.video.i_width );
+    p_filter->p_sys = (filter_sys_t*) p_copy_cache;
+
+    return VLC_SUCCESS;
+}
+
+static void CloseConverter( vlc_object_t *obj )
+{
+    filter_t *p_filter = (filter_t *)obj;
+    copy_cache_t *p_copy_cache = (copy_cache_t*) p_filter->p_sys;
+    CopyCleanCache(p_copy_cache);
+    free( p_copy_cache );
+    p_filter->p_sys = NULL;
+}

po/POTFILES.in

@@ -1060,6 +1060,7 @@ modules/text_renderer/svg.c
 modules/text_renderer/tdummy.c
 modules/text_renderer/win32text.c
 modules/video_chroma/chain.c
+modules/video_chroma/dxa9.c
 modules/video_chroma/grey_yuv.c
 modules/video_chroma/i420_rgb16.c
 modules/video_chroma/i420_rgb8.c