Used a CPU memory fallback when Direct3DLockSurface/DirectXLock fails.

It fixes a segfault when the directx/direct3d device is lost at the wrong time
or cannot be restored soon enough (close #3647).

modules/video_output/msw/common.c

@@ -217,9 +217,24 @@ void CommonDisplay(vout_display_t *vd)
 /**
  * It updates a picture data/pitches.
  */
-void CommonUpdatePicture(picture_t *picture,
-                         uint8_t *data, unsigned pitch)
+int CommonUpdatePicture(picture_t *picture, picture_t **fallback,
+                        uint8_t *data, unsigned pitch)
 {
+    if (fallback) {
+        if (*fallback == NULL) {
+            *fallback = picture_NewFromFormat(&picture->format);
+            if (*fallback == NULL)
+                return VLC_EGENERIC;
+        }
+        for (int n = 0; n < picture->i_planes; n++) {
+            const plane_t *src = &(*fallback)->p[n];
+            plane_t       *dst = &picture->p[n];
+            dst->p_pixels = src->p_pixels;
+            dst->i_pitch  = src->i_pitch;
+            dst->i_lines  = src->i_lines;
+        }
+        return VLC_SUCCESS;
+    }
     /* fill in buffer info in first plane */
     picture->p->p_pixels = data;
     picture->p->i_pitch  = pitch;
@@ -245,6 +260,7 @@ void CommonUpdatePicture(picture_t *picture,
             picture->p[2].p_pixels = p_tmp;
         }
     }
+    return VLC_SUCCESS;
 }
 
 void AlignRect(RECT *r, int align_boundary, int align_size)

modules/video_output/msw/common.h

@@ -237,7 +237,7 @@ void CommonClean(vout_display_t *);
 void CommonManage(vout_display_t *);
 int  CommonControl(vout_display_t *, int , va_list );
 void CommonDisplay(vout_display_t *);
-void CommonUpdatePicture(picture_t *, uint8_t *, unsigned);
+int  CommonUpdatePicture(picture_t *, picture_t **, uint8_t *, unsigned);
 
 void UpdateRects (vout_display_t *,
                   const vout_display_cfg_t *,

modules/video_output/msw/direct3d.c

@@ -87,6 +87,7 @@ vlc_module_end ()
 struct picture_sys_t
 {
     LPDIRECT3DSURFACE9 surface;
+    picture_t          *fallback;
 };
 
 static int  Open(vlc_object_t *);
@@ -758,10 +759,10 @@ static int Direct3DLockSurface(picture_t *picture)
     HRESULT hr = IDirect3DSurface9_LockRect(picture->p_sys->surface, &d3drect, NULL, 0);
     if (FAILED(hr)) {
         //msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
-        return VLC_EGENERIC;
+        return CommonUpdatePicture(picture, &picture->p_sys->fallback, NULL, 0);
     }
 
-    CommonUpdatePicture(picture, d3drect.pBits, d3drect.Pitch);
+    CommonUpdatePicture(picture, NULL, d3drect.pBits, d3drect.Pitch);
     return VLC_SUCCESS;
 }
 /**
@@ -832,6 +833,7 @@ static int Direct3DCreatePool(vout_display_t *vd, video_format_t *fmt)
         return VLC_ENOMEM;
     }
     rsc->p_sys->surface = surface;
+    rsc->p_sys->fallback = NULL;
     for (int i = 0; i < PICTURE_PLANE_MAX; i++) {
         rsc->p[i].p_pixels = NULL;
         rsc->p[i].i_pitch = 0;
@@ -870,7 +872,8 @@ static void Direct3DDestroyPool(vout_display_t *vd)
     if (sys->pool) {
         picture_resource_t *rsc = &sys->resource;
         IDirect3DSurface9_Release(rsc->p_sys->surface);
-
+        if (rsc->p_sys->fallback)
+            picture_Release(rsc->p_sys->fallback);
         picture_pool_Delete(sys->pool);
     }
     sys->pool = NULL;

modules/video_output/msw/directx.c

@@ -127,6 +127,7 @@ vlc_module_end()
 struct picture_sys_t {
     LPDIRECTDRAWSURFACE2 surface;
     LPDIRECTDRAWSURFACE2 front_surface;
+    picture_t            *fallback;
 };
 
 /*****************************************************************************
@@ -1046,6 +1047,7 @@ static int DirectXCreatePictureResourceYuvOverlay(vout_display_t *vd,
     picture_resource_t *rsc = &sys->resource;
     rsc->p_sys->front_surface = front_surface;
     rsc->p_sys->surface       = surface;
+    rsc->p_sys->fallback      = NULL;
     return VLC_SUCCESS;
 }
 static int DirectXCreatePictureResourceYuv(vout_display_t *vd,
@@ -1101,6 +1103,7 @@ static int DirectXCreatePictureResourceYuv(vout_display_t *vd,
     picture_resource_t *rsc = &sys->resource;
     rsc->p_sys->front_surface = surface;
     rsc->p_sys->surface       = surface;
+    rsc->p_sys->fallback      = NULL;
     return VLC_SUCCESS;
 }
 static int DirectXCreatePictureResourceRgb(vout_display_t *vd,
@@ -1160,6 +1163,7 @@ static int DirectXCreatePictureResourceRgb(vout_display_t *vd,
     picture_resource_t *rsc = &sys->resource;
     rsc->p_sys->front_surface = surface;
     rsc->p_sys->surface       = surface;
+    rsc->p_sys->fallback      = NULL;
     return VLC_SUCCESS;
 }
 
@@ -1214,6 +1218,8 @@ static void DirectXDestroyPictureResource(vout_display_t *vd)
     vout_display_sys_t *sys = vd->sys;
 
     DirectXDestroySurface(sys->resource.p_sys->front_surface);
+    if (sys->resource.p_sys->fallback)
+        picture_Release(sys->resource.p_sys->fallback);
 }
 
 static int DirectXLock(picture_t *picture)
@@ -1221,9 +1227,9 @@ static int DirectXLock(picture_t *picture)
     DDSURFACEDESC ddsd;
     if (DirectXLockSurface(picture->p_sys->front_surface,
                            picture->p_sys->surface, &ddsd))
-        return VLC_EGENERIC;
+        return CommonUpdatePicture(picture, &picture->p_sys->fallback, NULL, 0);
 
-    CommonUpdatePicture(picture, ddsd.lpSurface, ddsd.lPitch);
+    CommonUpdatePicture(picture, NULL, ddsd.lpSurface, ddsd.lPitch);
     return VLC_SUCCESS;
 }
 static void DirectXUnlock(picture_t *picture)