Improved and fixed subtitles blending in direct3D vout.

It checks if the blending will succeed with the formats we use.
It fixes the order of the RGB components (close #4919).

modules/video_output/msw/common.h

@@ -182,6 +182,7 @@ struct vout_display_sys_t
     // scene objects
     LPDIRECT3DTEXTURE9      d3dtex;
     LPDIRECT3DVERTEXBUFFER9 d3dvtc;
+    D3DFORMAT               d3dregion_format;
     int                     d3dregion_count;
     struct d3d_region_t     *d3dregion;
 

modules/video_output/msw/direct3d.c

@@ -185,6 +185,7 @@ static int Open(vlc_object_t *object)
     info.has_pictures_invalid = true;
     info.has_event_thread = true;
     if (var_InheritBool(vd, "direct3d-hw-blending") &&
+        sys->d3dregion_format != D3DFMT_UNKNOWN &&
         (sys->d3dcaps.SrcBlendCaps  & D3DPBLENDCAPS_SRCALPHA) &&
         (sys->d3dcaps.DestBlendCaps & D3DPBLENDCAPS_INVSRCALPHA) &&
         (sys->d3dcaps.TextureCaps   & D3DPTEXTURECAPS_ALPHA) &&
@@ -697,6 +698,8 @@ static void Direct3DDestroyScene(vout_display_t *vd);
  */
 static int Direct3DCreateResources(vout_display_t *vd, video_format_t *fmt)
 {
+    vout_display_sys_t *sys = vd->sys;
+
     if (Direct3DCreatePool(vd, fmt)) {
         msg_Err(vd, "Direct3D picture pool initialization failed");
         return VLC_EGENERIC;
@@ -705,6 +708,20 @@ static int Direct3DCreateResources(vout_display_t *vd, video_format_t *fmt)
         msg_Err(vd, "Direct3D scene initialization failed !");
         return VLC_EGENERIC;
     }
+    sys->d3dregion_format = D3DFMT_UNKNOWN;
+    for (int i = 0; i < 2; i++) {
+        D3DFORMAT fmt = i == 0 ? D3DFMT_A8B8G8R8 : D3DFMT_A8R8G8B8;
+        if (SUCCEEDED(IDirect3D9_CheckDeviceFormat(sys->d3dobj,
+                                                   D3DADAPTER_DEFAULT,
+                                                   D3DDEVTYPE_HAL,
+                                                   sys->d3dpp.BackBufferFormat,
+                                                   D3DUSAGE_DYNAMIC,
+                                                   D3DRTYPE_TEXTURE,
+                                                   fmt))) {
+            sys->d3dregion_format = fmt;
+            break;
+        }
+    }
     return VLC_SUCCESS;
 }
 /**
@@ -1201,7 +1218,7 @@ static void Direct3DImportSubpicture(vout_display_t *vd,
         for (int j = 0; j < sys->d3dregion_count; j++) {
             d3d_region_t *cache = &sys->d3dregion[j];
             if (cache->texture &&
-                cache->format == D3DFMT_A8R8G8B8 &&
+                cache->format == sys->d3dregion_format &&
                 cache->width  == r->fmt.i_visible_width &&
                 cache->height == r->fmt.i_visible_height) {
                 msg_Dbg(vd, "Reusing %dx%d texture for OSD",
@@ -1211,7 +1228,7 @@ static void Direct3DImportSubpicture(vout_display_t *vd,
             }
         }
         if (!d3dr->texture) {
-            d3dr->format = D3DFMT_A8R8G8B8;
+            d3dr->format = sys->d3dregion_format;
             d3dr->width  = r->fmt.i_visible_width;
             d3dr->height = r->fmt.i_visible_height;
             hr = IDirect3DDevice9_CreateTexture(sys->d3ddev,
@@ -1237,11 +1254,21 @@ static void Direct3DImportSubpicture(vout_display_t *vd,
         if (SUCCEEDED(hr)) {
             uint8_t *dst_data  = lock.pBits;
             int      dst_pitch = lock.Pitch;
+            uint8_t *src_data  = r->p_picture->p->p_pixels;
+            int      src_pitch = r->p_picture->p->i_pitch;
             for (unsigned y = 0; y < r->fmt.i_visible_height; y++) {
                 int copy_pitch = __MIN(dst_pitch, r->p_picture->p->i_visible_pitch);
-                memcpy(&dst_data[y * dst_pitch],
-                       &r->p_picture->p->p_pixels[y * r->p_picture->p->i_pitch],
-                       copy_pitch);
+                if (d3dr->format == D3DFMT_A8B8G8R8) {
+                    memcpy(&dst_data[y * dst_pitch], &src_data[y * src_pitch],
+                           copy_pitch);
+                } else {
+                    for (int x = 0; x < copy_pitch; x += 4) {
+                        dst_data[y * dst_pitch + x + 0] = src_data[y * src_pitch + x + 2];
+                        dst_data[y * dst_pitch + x + 1] = src_data[y * src_pitch + x + 1];
+                        dst_data[y * dst_pitch + x + 2] = src_data[y * src_pitch + x + 0];
+                        dst_data[y * dst_pitch + x + 3] = src_data[y * src_pitch + x + 3];
+                    }
+                }
             }
             hr = IDirect3DTexture9_UnlockRect(d3dr->texture, 0);
             if (FAILED(hr))