Lua: remap file descriptors in Lua and GC at exit (fixes #8898)

modules/lua/intf.c

@@ -199,7 +199,6 @@ static const luaL_Reg p_reg[] = { { NULL, NULL } };
 static int Start_LuaIntf( vlc_object_t *p_this, const char *name )
 {
     intf_thread_t *p_intf = (intf_thread_t*)p_this;
-    intf_sys_t *p_sys;
     lua_State *L;
 
     config_ChainParse( p_intf, "lua-", ppsz_intf_options, p_intf->p_cfg );
@@ -215,14 +214,17 @@ static int Start_LuaIntf( vlc_object_t *p_this, const char *name )
         /* Cleaned up by vlc_object_release() */
         p_intf->psz_header = strdup( name );
 
-    p_intf->p_sys = (intf_sys_t*)malloc( sizeof(intf_sys_t) );
-    if( !p_intf->p_sys )
+    intf_sys_t *p_sys = malloc( sizeof(*p_sys) );
+    if( unlikely(p_sys == NULL) )
     {
         free( p_intf->psz_header );
         p_intf->psz_header = NULL;
         return VLC_ENOMEM;
     }
-    p_sys = p_intf->p_sys;
+    p_intf->p_sys = p_sys;
+
+    vlclua_fd_init( p_sys );
+
     p_sys->psz_filename = vlclua_find_file( "intf", name );
     if( !p_sys->psz_filename )
     {
@@ -399,6 +401,7 @@ void Close_LuaIntf( vlc_object_t *p_this )
 
     lua_close( p_sys->L );
     close( p_sys->fd[0] );
+    vlclua_fd_destroy( p_sys );
     free( p_sys->psz_filename );
     free( p_sys );
 }

modules/lua/libs/net.c

@@ -28,6 +28,7 @@
 # include "config.h"
 #endif
 
+#include <assert.h>
 #include <errno.h>
 #ifdef _WIN32
 #include <io.h>
@@ -46,6 +47,114 @@
 #include "../vlc.h"
 #include "../libs.h"
 
+void vlclua_fd_init( intf_sys_t *sys )
+{
+    for( unsigned i = 0; i < (sizeof(sys->fds)/sizeof(sys->fds[0])); i++ )
+        sys->fds[i] = -1;
+}
+
+/** Releases all (leaked) VLC Lua file descriptors. */
+void vlclua_fd_destroy( intf_sys_t *sys )
+{
+    for( unsigned i = 0; i < (sizeof(sys->fds)/sizeof(sys->fds[0])); i++ )
+        if( sys->fds[i] != -1 )
+            net_Close( sys->fds[i] );
+}
+
+
+/** Maps an OS file descriptor to a VLC Lua file descriptor */
+static int vlclua_fd_map( lua_State *L, int fd )
+{
+    intf_thread_t *intf = (intf_thread_t *)vlclua_get_this( L );
+    intf_sys_t *sys = intf->p_sys;
+
+    if( (unsigned)fd < 3u )
+        return -1;
+
+#ifndef NDEBUG
+    for( unsigned i = 0; i < (sizeof(sys->fds)/sizeof(sys->fds[0])); i++ )
+        assert( sys->fds[i] != fd );
+#endif
+    for( unsigned i = 0; i < (sizeof(sys->fds)/sizeof(sys->fds[0])); i++ )
+        if( sys->fds[i] == -1 )
+        {
+            sys->fds[i] = fd;
+            return 3 + i;
+        }
+
+    return -1;
+}
+
+static int vlclua_fd_map_safe( lua_State *L, int fd )
+{
+    int luafd = vlclua_fd_map( L, fd );
+    if( luafd == -1 )
+        net_Close( fd );
+    return luafd;
+}
+
+/** Gets the OS file descriptor mapped to a VLC Lua file descriptor */
+static int vlclua_fd_get( lua_State *L, unsigned idx )
+{
+    intf_thread_t *intf = (intf_thread_t *)vlclua_get_this( L );
+    intf_sys_t *sys = intf->p_sys;
+
+    if( idx < 3u )
+        return idx;
+    idx -= 3;
+    if( sizeof(sys->fds[0]) * idx < sizeof(sys->fds) )
+        return sys->fds[idx];
+    return -1;
+}
+
+/** Gets the VLC Lua file descriptor mapped from an OS file descriptor */
+static int vlclua_fd_get_lua( lua_State *L, int fd )
+{
+    intf_thread_t *intf = (intf_thread_t *)vlclua_get_this( L );
+    intf_sys_t *sys = intf->p_sys;
+
+    if( (unsigned)fd < 3u )
+        return fd;
+    for( unsigned i = 0; i < (sizeof(sys->fds)/sizeof(sys->fds[0])); i++ )
+        if( sys->fds[i] == fd )
+            return 3 + i;
+    return -1;
+}
+
+/** Unmaps an OS file descriptor from VLC Lua */
+static void vlclua_fd_unmap( lua_State *L, unsigned idx )
+{
+    intf_thread_t *intf = (intf_thread_t *)vlclua_get_this( L );
+    intf_sys_t *sys = intf->p_sys;
+    int fd = -1;
+
+    if( idx < 3u )
+        return; /* Never close stdin/stdout/stderr. */
+
+    idx -= 3;
+    if( idx < (sizeof(sys->fds)/sizeof(sys->fds[0])) )
+    {
+        fd = sys->fds[idx];
+        sys->fds[idx] = -1;
+    }
+
+    if( fd == -1 )
+        return;
+#ifndef NDEBUG
+    for( unsigned i = 0; i < (sizeof(sys->fds)/sizeof(sys->fds[0])); i++ )
+        assert( sys->fds[i] != fd );
+#endif
+}
+
+static void vlclua_fd_unmap_safe( lua_State *L, unsigned idx )
+{
+    int fd = vlclua_fd_get( L, idx );
+
+    vlclua_fd_unmap( L, idx );
+    if( fd != -1 )
+        net_Close( fd );
+}
+
 /*****************************************************************************
  *
  *****************************************************************************/
@@ -100,6 +209,16 @@ static int vlclua_net_listen_tcp( lua_State *L )
     if( pi_fd == NULL )
         return luaL_error( L, "Cannot listen on %s:%d", psz_host, i_port );
 
+    for( unsigned i = 0; pi_fd[i] != -1; i++ )
+        if( vlclua_fd_map( L, pi_fd[i] ) == -1 )
+        {
+            while( i > 0 )
+                vlclua_fd_unmap( L, vlclua_fd_get_lua( L, pi_fd[--i] ) );
+
+            net_ListenClose( pi_fd );
+            return luaL_error( L, "Cannot listen on %s:%d", psz_host, i_port );
+        }
+
     int **ppi_fd = lua_newuserdata( L, sizeof( int * ) );
     *ppi_fd = pi_fd;
 
@@ -119,7 +238,12 @@ static int vlclua_net_listen_tcp( lua_State *L )
 static int vlclua_net_listen_close( lua_State *L )
 {
     int **ppi_fd = (int**)luaL_checkudata( L, 1, "net_listen" );
-    net_ListenClose( *ppi_fd );
+    int *pi_fd = *ppi_fd;
+
+    for( unsigned i = 0; pi_fd[i] != -1; i++ )
+        vlclua_fd_unmap( L, vlclua_fd_get_lua( L, pi_fd[i] ) );
+
+    net_ListenClose( pi_fd );
     return 0;
 }
 
@@ -130,7 +254,7 @@ static int vlclua_net_fds( lua_State *L )
 
     int i_count = 0;
     while( pi_fd[i_count] != -1 )
-        lua_pushinteger( L, pi_fd[i_count++] );
+        lua_pushinteger( L, vlclua_fd_get_lua( L, pi_fd[i_count++] ) );
 
     return i_count;
 }
@@ -141,7 +265,7 @@ static int vlclua_net_accept( lua_State *L )
     int **ppi_fd = (int**)luaL_checkudata( L, 1, "net_listen" );
     int i_fd = net_Accept( p_this, *ppi_fd );
 
-    lua_pushinteger( L, i_fd );
+    lua_pushinteger( L, vlclua_fd_map_safe( L, i_fd ) );
     return 1;
 }
 
@@ -154,14 +278,14 @@ static int vlclua_net_connect_tcp( lua_State *L )
     const char *psz_host = luaL_checkstring( L, 1 );
     int i_port = luaL_checkint( L, 2 );
     int i_fd = net_Connect( p_this, psz_host, i_port, SOCK_STREAM, IPPROTO_TCP );
-    lua_pushinteger( L, i_fd );
+    lua_pushinteger( L, vlclua_fd_map_safe( L, i_fd ) );
     return 1;
 }
 
 static int vlclua_net_close( lua_State *L )
 {
     int i_fd = luaL_checkint( L, 1 );
-    net_Close( i_fd );
+    vlclua_fd_unmap_safe( L, i_fd );
     return 0;
 }
 
@@ -171,7 +295,7 @@ static int vlclua_net_send( lua_State *L )
     size_t i_len;
     const char *psz_buffer = luaL_checklstring( L, 2, &i_len );
     i_len = luaL_optint( L, 3, i_len );
-    i_len = send( i_fd, psz_buffer, i_len, 0 );
+    i_len = send( vlclua_fd_get( L, i_fd ), psz_buffer, i_len, 0 );
     lua_pushinteger( L, i_len );
     return 1;
 }
@@ -181,7 +305,7 @@ static int vlclua_net_recv( lua_State *L )
     int i_fd = luaL_checkint( L, 1 );
     size_t i_len = luaL_optint( L, 2, 1 );
     char psz_buffer[i_len];
-    ssize_t i_ret = recv( i_fd, psz_buffer, i_len, 0 );
+    ssize_t i_ret = recv( vlclua_fd_get( L, i_fd ), psz_buffer, i_len, 0 );
     if( i_ret > 0 )
         lua_pushlstring( L, psz_buffer, i_ret );
     else
@@ -210,6 +334,7 @@ static int vlclua_net_poll( lua_State *L )
     }
 
     struct pollfd *p_fds = xmalloc( i_fds * sizeof( *p_fds ) );
+    int *luafds = xmalloc( i_fds * sizeof( *luafds ) );
 
     lua_pushnil( L );
     int i = 1;
@@ -217,7 +342,8 @@ static int vlclua_net_poll( lua_State *L )
     p_fds[0].events = POLLIN;
     while( lua_next( L, 1 ) )
     {
-        p_fds[i].fd = luaL_checkinteger( L, -2 );
+        luafds[i] = luaL_checkinteger( L, -2 );
+        p_fds[i].fd = vlclua_fd_get( L, luafds[i] );
         p_fds[i].events = luaL_checkinteger( L, -1 );
         lua_pop( L, 1 );
         i++;
@@ -230,7 +356,7 @@ static int vlclua_net_poll( lua_State *L )
 
     for( i = 1; i < i_fds; i++ )
     {
-        lua_pushinteger( L, p_fds[i].fd );
+        lua_pushinteger( L, luafds[i] );
         lua_pushinteger( L, p_fds[i].revents );
         lua_settable( L, 1 );
     }
@@ -261,7 +387,7 @@ static int vlclua_fd_write( lua_State *L )
     ssize_t i_ret;
     const char *psz_buffer = luaL_checklstring( L, 2, &i_len );
     i_len = luaL_optint( L, 3, i_len );
-    i_ret = write( i_fd, psz_buffer, i_len );
+    i_ret = write( vlclua_fd_get( L, i_fd ), psz_buffer, i_len );
     lua_pushinteger( L, i_ret );
     return 1;
 }
@@ -271,7 +397,7 @@ static int vlclua_fd_read( lua_State *L )
     int i_fd = luaL_checkint( L, 1 );
     size_t i_len = luaL_optint( L, 2, 1 );
     char psz_buffer[i_len];
-    ssize_t i_ret = read( i_fd, psz_buffer, i_len );
+    ssize_t i_ret = read( vlclua_fd_get( L, i_fd ), psz_buffer, i_len );
     if( i_ret > 0 )
         lua_pushlstring( L, psz_buffer, i_ret );
     else

modules/lua/vlc.h

@@ -161,9 +161,13 @@ struct intf_sys_t
 #ifndef _WIN32
     int fd[2];
 #endif
+    int fds[64];
 
     vlc_thread_t thread;
 };
 
+void vlclua_fd_init( struct intf_sys_t * );
+void vlclua_fd_destroy( struct intf_sys_t * );
+
 #endif /* VLC_LUA_H */