GLX: use separate XCB and Xlib connections (close #5698)

Some GLX drivers, such as newer versions of the Intel drivers,
depend on X11 events coming from Xlib. So VLC cannot steal them from
the underlying XCB connection.

I have no time and will to duplicate the XCB code found in the GLX
plugin for windowing and events handling using Xlib. So lets make a
separate X connection with XCB. This is inefficient, but it works.

On the plus side, this patch removes the dependency on libx11-xcb.
So it is now possible to compile and use GLX with an Xtrans-based
Xlib (though libxcb must be present).

(cherry picked from commit 63a73525)
(cherry picked from commit f886dc14)

configure.ac

@@ -3071,11 +3071,7 @@ AS_IF([test "${enable_xcb}" != "no"], [
     AS_IF([test "${have_gl}" != "yes"], [
       AC_MSG_ERROR([${GL_PKG_ERRORS}. Pass --disable-glx if you do not need OpenGL X11 support.])
     ])
-    PKG_CHECK_MODULES(XLIB_XCB, [x11-xcb], [
-      VLC_ADD_PLUGIN([xcb_glx])
-    ], [
-      AC_MSG_ERROR([${XLIB_XCB_PKG_ERRORS}. Pass --disable-glx if you do not need OpenGL X11 support.])
-    ])
+    VLC_ADD_PLUGIN([xcb_glx])
   ])
 ])
 AM_CONDITIONAL([HAVE_XCB], [test "${have_xcb}" = "yes"])

modules/video_output/Modules.am

@@ -70,9 +70,9 @@ libxcb_glx_plugin_la_SOURCES = \
 	opengl.c \
 	xcb/events.c
 libxcb_glx_plugin_la_CFLAGS = $(AM_CFLAGS) \
-	$(XLIB_XCB_CFLAGS) $(GL_CFLAGS)
+	$(XCB_CFLAGS) $(GL_CFLAGS)
 libxcb_glx_plugin_la_LIBADD = $(AM_LIBADD) \
-	$(XLIB_XCB_LIBS) $(GL_LIBS)
+	$(XCB_LIBS) $(GL_LIBS) $(X_LIBS) $(X_PRE_LIBS) -lX11
 libxcb_glx_plugin_la_DEPENDENCIES =
 
 libxcb_window_plugin_la_SOURCES = xcb/window.c xcb/keys.c xcb/keysym.h xcb/xcb_keysym.h

modules/video_output/xcb/glx.c

@@ -29,7 +29,6 @@
 #include <assert.h>
 
 #include <xcb/xcb.h>
-#include <X11/Xlib-xcb.h>
 #include <GL/glx.h>
 #include <GL/glxext.h>
 
@@ -227,23 +226,29 @@ static int Open (vlc_object_t *obj)
     }
 
     /* Connect to X server */
+    xcb_connection_t *conn = xcb_connect (sys->embed->display.x11, NULL);
+    if (unlikely(xcb_connection_has_error (conn)))
+    {
+        vout_display_DeleteWindow (vd, sys->embed);
+        free (sys);
+        return VLC_EGENERIC;
+    }
+
     Display *dpy = XOpenDisplay (sys->embed->display.x11);
     if (dpy == NULL)
     {
+        xcb_disconnect (conn);
         vout_display_DeleteWindow (vd, sys->embed);
         free (sys);
         return VLC_EGENERIC;
     }
     sys->display = dpy;
+    sys->conn = conn;
     sys->ctx = NULL;
-    XSetEventQueueOwner (dpy, XCBOwnsEventQueue);
 
     if (!CheckGLX (vd, dpy, &sys->v1_3))
         goto error;
 
-    xcb_connection_t *conn = XGetXCBConnection (dpy);
-    assert (conn != NULL);
-    sys->conn = conn;
     RegisterMouseEvents (obj, conn, sys->embed->handle.xid);
 
     /* Find window parameters */
@@ -450,13 +455,14 @@ static void Close (vlc_object_t *obj)
         if (sys->v1_3)
             glXDestroyWindow (dpy, sys->glwin);
     }
+    XCloseDisplay (dpy);
 
     /* show the default cursor */
     xcb_change_window_attributes (sys->conn, sys->embed->handle.xid,
                                XCB_CW_CURSOR, &(uint32_t) { XCB_CURSOR_NONE });
     xcb_flush (sys->conn);
+    xcb_disconnect (sys->conn);
 
-    XCloseDisplay (dpy);
     vout_display_DeleteWindow (vd, sys->embed);
     free (sys);
 }