- directx: avoid mixing GDI and Direct3D/OpenGL/DirectX (non-overlay) in video...

- directx: avoid mixing GDI and Direct3D/OpenGL/DirectX (non-overlay) in video window, should help Vista compatibility

modules/video_output/directx/direct3d.c

@@ -57,6 +57,7 @@ static int  Init      ( vout_thread_t * );
 static void End       ( vout_thread_t * );
 static int  Manage    ( vout_thread_t * );
 static void Display   ( vout_thread_t *, picture_t * );
+static void FirstDisplay( vout_thread_t *, picture_t * );
 
 static int Direct3DVoutCreate     ( vout_thread_t * );
 static void Direct3DVoutRelease   ( vout_thread_t * );
@@ -164,7 +165,7 @@ static int OpenVideo( vlc_object_t *p_this )
     p_vout->pf_end = End;
     p_vout->pf_manage = Manage;
     p_vout->pf_render = Direct3DVoutRenderScene;
-    p_vout->pf_display = Display;
+    p_vout->pf_display = FirstDisplay;
 
     p_vout->p_sys->hwnd = p_vout->p_sys->hvideownd = NULL;
     p_vout->p_sys->hparent = p_vout->p_sys->hfswnd = NULL;
@@ -624,12 +625,32 @@ static void Display( vout_thread_t *p_vout, picture_t *p_pic )
     HRESULT hr = IDirect3DDevice9_Present(p_d3ddev, NULL, NULL, NULL, NULL);
     if( FAILED(hr) )
         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
+}
 
-    /* if set, remove the black brush to avoid flickering in repaint operations */
-    if( 0UL != GetClassLong( p_vout->p_sys->hvideownd, GCL_HBRBACKGROUND) )
-    {
-        SetClassLong( p_vout->p_sys->hvideownd, GCL_HBRBACKGROUND, 0UL);
-    }
+/*
+** this function is only used once when the first picture is received
+** this function will show the video window once a picture is ready
+*/
+
+static void FirstDisplay( vout_thread_t *p_vout, picture_t *p_pic )
+{
+    /* get initial picture presented through D3D */
+    Display(p_vout, p_pic);
+
+    /*
+    ** Video window is initially hidden, show it now since we got a 
+    ** picture to show.
+    */
+    SetWindowPos( p_vout->p_sys->hvideownd, NULL, 0, 0, 0, 0, 
+        SWP_ASYNCWINDOWPOS|
+        SWP_FRAMECHANGED|
+        SWP_SHOWWINDOW|
+        SWP_NOMOVE|
+        SWP_NOSIZE|
+        SWP_NOZORDER );
+
+    /* use and restores proper display function for further pictures */
+    p_vout->pf_display = Display;
 }
 
 /*****************************************************************************

modules/video_output/directx/directx.c

@@ -92,7 +92,7 @@ static int  Init      ( vout_thread_t * );
 static void End       ( vout_thread_t * );
 static int  Manage    ( vout_thread_t * );
 static void Display   ( vout_thread_t *, picture_t * );
-static void OverlayDisplay( vout_thread_t *, picture_t * );
+static void FirstDisplay( vout_thread_t *, picture_t * );
 static void SetPalette( vout_thread_t *, uint16_t *, uint16_t *, uint16_t * );
 
 static int  NewPictureVec  ( vout_thread_t *, picture_t *, int );
@@ -218,7 +218,7 @@ static int OpenVideo( vlc_object_t *p_this )
     p_vout->pf_end = End;
     p_vout->pf_manage = Manage;
     p_vout->pf_render = NULL;
-    p_vout->pf_display = Display;
+    p_vout->pf_display = FirstDisplay;
 
     p_vout->p_sys->p_ddobject = NULL;
     p_vout->p_sys->p_display = NULL;
@@ -454,11 +454,6 @@ static int Init( vout_thread_t *p_vout )
         NewPictureVec( p_vout, p_vout->p_picture, MAX_DIRECTBUFFERS );
     }
 
-    if( p_vout->p_sys->b_using_overlay )
-    {
-        p_vout->pf_display = OverlayDisplay;
-    }
-
     /* Change the window title bar text */
     PostMessage( p_vout->p_sys->hwnd, WM_VLC_CHANGE_TEXT, 0, 0 );
 
@@ -829,15 +824,6 @@ static void Display( vout_thread_t *p_vout, picture_t *p_pic )
             msg_Warn( p_vout, "could not blit surface (error %li)", dxresult );
             return;
         }
-        else
-        {
-            /* if set, remove the black brush to avoid flickering in repaint operations */
-            if( 0UL != GetClassLong( p_vout->p_sys->hvideownd, GCL_HBRBACKGROUND) )
-            {
-                SetClassLong(p_vout->p_sys->hvideownd, GCL_HBRBACKGROUND, (ULONG)0UL);
-            }
-        }
-
     }
     else /* using overlay */
     {
@@ -870,10 +856,11 @@ static void Display( vout_thread_t *p_vout, picture_t *p_pic )
 
 /*
 ** this function is only used once when the first picture is received
-** The overlay colorkey replaces black as the background color on the
-** video window; this will cause the overlay surface to be displayed
+** this function will show the video window once video is ready for
+** display.
 */
-static void OverlayDisplay( vout_thread_t *p_vout, picture_t *p_pic )
+
+static void FirstDisplay( vout_thread_t *p_vout, picture_t *p_pic )
 {
     /* get initial picture rendered on overlay surface */
     Display(p_vout, p_pic);
@@ -881,10 +868,23 @@ static void OverlayDisplay( vout_thread_t *p_vout, picture_t *p_pic )
     IDirectDraw_WaitForVerticalBlank(p_vout->p_sys->p_ddobject,
             DDWAITVB_BLOCKBEGIN, NULL);
 
-    /* set the colorkey as the backgound brush for the video window */
-    SetClassLong( p_vout->p_sys->hvideownd, GCL_HBRBACKGROUND,
-                  (LONG)CreateSolidBrush( p_vout->p_sys->i_rgb_colorkey ) );
-    InvalidateRect( p_vout->p_sys->hvideownd, NULL, TRUE );
+    if( p_vout->p_sys->b_using_overlay )
+    {
+        /* set the colorkey as the backgound brush for the video window */
+        SetClassLong( p_vout->p_sys->hvideownd, GCL_HBRBACKGROUND,
+                      (LONG)CreateSolidBrush( p_vout->p_sys->i_rgb_colorkey ) );
+    }
+    /*
+    ** Video window is initially hidden, show it now since we got a 
+    ** picture to show.
+    */
+    SetWindowPos( p_vout->p_sys->hvideownd, NULL, 0, 0, 0, 0, 
+        SWP_ASYNCWINDOWPOS|
+        SWP_FRAMECHANGED|
+        SWP_SHOWWINDOW|
+        SWP_NOMOVE|
+        SWP_NOSIZE|
+        SWP_NOZORDER );
 
     /* use and restores proper display function for further pictures */
     p_vout->pf_display = Display;
@@ -1135,12 +1135,6 @@ static int DirectXCreateDisplay( vout_thread_t *p_vout )
     p_vout->p_sys->i_rgb_colorkey =
         DirectXFindColorkey( p_vout, &p_vout->p_sys->i_colorkey );
 
-    /* use black brush as the video background color,
-       if overlay video is used, this will be replaced by the
-       colorkey when the first picture is received */
-    SetClassLong( p_vout->p_sys->hvideownd, GCL_HBRBACKGROUND,
-                  (LONG)GetStockObject( BLACK_BRUSH ) );
-    InvalidateRect( p_vout->p_sys->hvideownd, NULL, TRUE );
     E_(DirectXUpdateRects)( p_vout, VLC_TRUE );
 
     return VLC_SUCCESS;

modules/video_output/directx/events.c

@@ -440,6 +440,7 @@ static int DirectXCreateWindow( vout_thread_t *p_vout )
 
     /* Register the video sub-window class */
     wc.lpszClassName = _T("VLC DirectX video"); wc.hIcon = 0;
+    wc.hbrBackground = NULL; /* no background color */
     if( !RegisterClass(&wc) )
     {
         WNDCLASS wndclass;
@@ -542,7 +543,7 @@ static int DirectXCreateWindow( vout_thread_t *p_vout )
      * without having them shown outside of the video area. */
     p_vout->p_sys->hvideownd =
     CreateWindow( _T("VLC DirectX video"), _T(""),   /* window class */
-        WS_CHILD | WS_VISIBLE,                   /* window style */
+        WS_CHILD,                   /* window style, not visible initially */
         0, 0,
         p_vout->render.i_width,         /* default width */
         p_vout->render.i_height,        /* default height */
@@ -551,9 +552,9 @@ static int DirectXCreateWindow( vout_thread_t *p_vout )
         (LPVOID)p_vout );            /* send p_vout to WM_CREATE */
 
     if( !p_vout->p_sys->hvideownd )
-    msg_Warn( p_vout, "can't create video sub-window" );
+        msg_Warn( p_vout, "can't create video sub-window" );
     else
-    msg_Dbg( p_vout, "created video sub-window" );
+        msg_Dbg( p_vout, "created video sub-window" );
 
     /* Now display the window */
     ShowWindow( p_vout->p_sys->hwnd, SW_SHOW );
@@ -793,17 +794,38 @@ static long FAR PASCAL DirectXEventProc( HWND hwnd, UINT message,
     {
         switch( message )
         {
+#ifdef MODULE_NAME_IS_vout_directx
         case WM_ERASEBKGND:
-            // erase the background only if a brush is set
-            return ( 0UL == GetClassLong( hwnd, GCL_HBRBACKGROUND) ) ?
+            /* For overlay, we need to erase background */
+            return !p_vout->p_sys->b_using_overlay ?
                 1 : DefWindowProc(hwnd, message, wParam, lParam);
         case WM_PAINT:
-            if( 0UL == GetClassLong( hwnd, GCL_HBRBACKGROUND) )
+        /*
+        ** For overlay, DefWindowProc() will erase dirty regions
+        ** with colorkey.
+        ** For non-overlay, vout will paint the whole window at 
+        ** regular interval, therefore dirty regions can be ignored
+        ** to minimize repaint.
+        */
+            if( !p_vout->p_sys->b_using_overlay )
             {
-                // vout paints the whole area, no need to repaint it
                 ValidateRect(hwnd, NULL);
             }
+            // fall through to default
+#else
+        /*
+        ** For OpenGL and Direct3D, vout will update the whole
+        ** window at regular interval, therefore dirty region
+        ** can be ignored to minimize repaint.
+        */
+        case WM_ERASEBKGND:
+            /* nothing to erase */
+            return 1;
+        case WM_PAINT:
+            /* nothing to repaint */
+            ValidateRect(hwnd, NULL);
             // fall through
+#endif 
         default:
             return DefWindowProc(hwnd, message, wParam, lParam);
         }

modules/video_output/directx/glwin32.c

@@ -54,6 +54,7 @@ static int  Init      ( vout_thread_t * );
 static void End       ( vout_thread_t * );
 static int  Manage    ( vout_thread_t * );
 static void GLSwapBuffers( vout_thread_t * );
+static void FirstSwap( vout_thread_t * );
 
 /*****************************************************************************
  * Module descriptor
@@ -102,7 +103,7 @@ static int OpenVideo( vlc_object_t *p_this )
     p_vout->pf_init = Init;
     p_vout->pf_end = End;
     p_vout->pf_manage = Manage;
-    p_vout->pf_swap = GLSwapBuffers;
+    p_vout->pf_swap = FirstSwap;
 
     p_vout->p_sys->hwnd = p_vout->p_sys->hvideownd = NULL;
     p_vout->p_sys->hparent = p_vout->p_sys->hfswnd = NULL;
@@ -481,3 +482,29 @@ static void GLSwapBuffers( vout_thread_t *p_vout )
 {
     SwapBuffers( p_vout->p_sys->hGLDC );
 }
+
+/*
+** this function is only used once when the first picture is received
+** this function will show the video window once a picture is ready
+*/
+
+static void FirstSwap( vout_thread_t *p_vout )
+{
+    /* get initial picture buffer swapped to front buffer */
+    GLSwapBuffers( p_vout );
+
+    /*
+    ** Video window is initially hidden, show it now since we got a 
+    ** picture to show.
+    */
+    SetWindowPos( p_vout->p_sys->hvideownd, NULL, 0, 0, 0, 0, 
+        SWP_ASYNCWINDOWPOS|
+        SWP_FRAMECHANGED|
+        SWP_SHOWWINDOW|
+        SWP_NOMOVE|
+        SWP_NOSIZE|
+        SWP_NOZORDER );
+
+    /* use and restores proper swap function for further pictures */
+    p_vout->pf_swap = GLSwapBuffers;
+}