VAAPI XCB: Move XOpenDisplay/XCloseDisplay to libva connection struct.

The avcodec module and vaapi vout module share the same VA API connection.
This allowed for a bug when XCloseDisplay() is called first in one module,
when the other still uses the VA API connection. Ommitting XCloseDisplay()
in the module that quits first allows for a resource leak. Therefor the best
solution is to also share the X11 Display connection for VA API.

Note: the VAAPI XCB vout uses another X11/XCB connection for managing its
windows and their events.

This reverts commit 74a2179e.

modules/codec/avcodec/va.c

@@ -173,6 +173,7 @@ static void vlc_va_destroy_surfaces( vlc_va_conn_t *conn )
 
 /* Global VAAPI connection state */
 static vlc_va_conn_t vlc_va_conn = {
+    .x11 = NULL,
     .p_display = 0,
     /* libva version */
     .i_version_major = 0,
@@ -196,7 +197,7 @@ static vlc_va_conn_t *vlc_va_get_conn( void )
     return (vlc_va_conn_t *) &vlc_va_conn;
 }
 
-vlc_va_conn_t *vlc_va_Initialize( Display *display )
+vlc_va_conn_t *vlc_va_Initialize( const char *display )
 {
     vlc_mutex_lock( &vlc_va_conn_lock );
 
@@ -210,8 +211,13 @@ vlc_va_conn_t *vlc_va_Initialize( Display *display )
         return conn;
     }
 
+    /* X11 display */
+    conn->x11 = XOpenDisplay(display);
+    if( !conn->x11 )
+        goto error;
+
     /* Create a VA display */
-    conn->p_display = vaGetDisplay(display);
+    conn->p_display = vaGetDisplay(conn->x11);
     if( !conn->p_display )
         goto error;
 
@@ -223,6 +229,8 @@ vlc_va_conn_t *vlc_va_Initialize( Display *display )
     return conn;
 
 error:
+    if( conn->x11 )
+        XCloseDisplay( conn->x11 );
     vlc_mutex_unlock( &vlc_va_conn_lock );
     return NULL;
 }
@@ -241,8 +249,10 @@ void vlc_va_Terminate( vlc_va_conn_t *conn )
 
         assert( conn->p_display );
         vaTerminate( conn->p_display );
+        XCloseDisplay( conn->x11 );
 
         /* Reset values */
+        conn->x11 = NULL;
         conn->p_display = 0;
         conn->i_version_major = conn->i_version_minor = 0;
         conn->i_ref_count = 0;

modules/codec/avcodec/vaapi.c

@@ -55,7 +55,6 @@ typedef struct
 
     /* */
     vlc_va_conn_t *conn;
-    Display       *display;
 
     VAConfigID    i_config_id;
     VAContextID   i_context_id;
@@ -134,11 +133,7 @@ static int Open( vlc_va_vaapi_t *p_va, int i_codec_id, int i_count )
     p_va->image.image_id = VA_INVALID_ID;
 
     /* Create a VA display */
-    p_va->display = XOpenDisplay(NULL);
-    if (!p_va->display)
-        goto error;
-
-    p_va->conn = vlc_va_Initialize(p_va->display);
+    p_va->conn = vlc_va_Initialize(NULL);
     if (!p_va->conn)
         goto error;
 
@@ -579,10 +574,6 @@ static void Close( vlc_va_vaapi_t *p_va )
 
         vlc_va_Terminate( p_va->conn );
     }
-#if 0 /* FIXME: workaround for ATI and AMD graphics GPU */
-    if( p_va->display )
-        XCloseDisplay( p_va->display );
-#endif
 }
 
 static void Delete( vlc_va_t *p_external )

modules/codec/avcodec/vaapi.h

@@ -31,6 +31,7 @@
 typedef struct vlc_va_conn_t vlc_va_conn_t;
 struct vlc_va_conn_t
 {
+    Display      *x11; /* x11 display connection */
     VADisplay     p_display;
     int           i_version_major;
     int           i_version_minor;
@@ -51,7 +52,7 @@ struct vlc_va_conn_t
 };
 
 /* Initialize shared connection to libva */
-vlc_va_conn_t *vlc_va_Initialize(Display *display);
+vlc_va_conn_t *vlc_va_Initialize(const char *display);
 
 /* Deinitialize shared connection to libva */
 void vlc_va_Terminate(vlc_va_conn_t *conn);

modules/codec/avcodec/vaapi_x11.c

@@ -93,7 +93,6 @@ struct vout_display_sys_t
     vlc_mutex_t   cache_lock;
     vlc_array_t   cache;  /* array of subpicture_subpicture_cache_t */
 
-    Display      *x11display; /* x11 display connection */
     xcb_cursor_t cursor; /* blank cursor */
     xcb_window_t window; /* drawable X window */
 
@@ -229,25 +228,18 @@ int OpenVaapiX11(vlc_object_t *obj)
     if (unlikely(sys->embed == NULL))
         goto error;
 
+    /* Create a VA display */
+    sys->vaconn = vlc_va_Initialize(sys->embed->display.x11);
+    if (!sys->vaconn)
+        goto error;
+
     /* Connect to X server */
     xcb_connection_t *conn = xcb_connect(sys->embed->display.x11, NULL);
     if (unlikely(xcb_connection_has_error (conn)))
         goto error;
 
-    /* X11 display */
-    sys->x11display = XOpenDisplay(sys->embed->display.x11);
-    if (!sys->x11display)
-    {
-        xcb_disconnect(conn);
-        goto error;
-    }
     sys->conn = conn;
 
-    /* Create a VA display */
-    sys->vaconn = vlc_va_Initialize(sys->x11display);
-    if (!sys->vaconn)
-        goto error;
-
     vlc_fourcc_t i_chroma;
     int32_t i_bits_per_pixel;
     if (FindVAFourCC(sys->vaconn, &sys->img_fmt, &i_chroma, &i_bits_per_pixel) != VLC_SUCCESS)
@@ -256,7 +248,7 @@ int OpenVaapiX11(vlc_object_t *obj)
     vd->fmt.i_chroma = i_chroma;
     vd->fmt.i_bits_per_pixel = i_bits_per_pixel;
 
-    XSetEventQueueOwner(sys->x11display, XCBOwnsEventQueue);
+    XSetEventQueueOwner(sys->vaconn->x11, XCBOwnsEventQueue);
 
     RegisterMouseEvents(obj, conn, sys->embed->handle.xid);
 
@@ -371,9 +363,6 @@ void CloseVaapiX11(vlc_object_t *obj)
         xcb_disconnect(sys->conn);
     }
 
-    if (sys->x11display)
-        XCloseDisplay(sys->x11display);
-
     if (sys->embed)
         vout_display_DeleteWindow (vd, sys->embed);