Whenever using select() companion macro FD_SET(), one must ensure that

the specified file descriptor does not exceeds the fd_set limit (FD_SETSIZE).
Registering a file descriptor above this limit is not defined (read: this is
a buffer overflow).

This is not normally exploitable on Linux because FD_SETSIZE equals the
default limit for file descriptors per processes, but should work on BSDs.
Cygwin is also very prone to this problem (but right, we don't use it). As
for Windows, it will overflow if you try to select() more than 64 sockets at
a time (regardless of the total numbers of socket), but for the sake of
simplicity and portability, this patch actually prevent opening more than
64 sockets at all.

There are still quite a bunch of plugins that needs fixing.

modules/access/file.c

@@ -623,8 +623,20 @@ static int _OpenFile( access_t * p_access, const char * psz_name )
         return VLC_EGENERIC;
     }
 
+    // FIXME: support non-ANSI filenames on Win32
     p_sys->fd = open( psz_localname, O_NONBLOCK /*| O_LARGEFILE*/ );
     LocaleFree( psz_localname );
+
+#ifndef WIN32
+    if( p_sys->fd >= FD_SETSIZE )
+    {
+        // Avoid overflowing fd_set
+        close( p_sys->fd );
+        p_sys->fd = -1;
+        errno = EMFILE;
+    }
+#endif
+
     if ( p_sys->fd == -1 )
     {
         msg_Err( p_access, "cannot open file %s (%s)", psz_name,

src/network/httpd.c

@@ -2435,7 +2435,14 @@ static void httpd_HostThread( httpd_host_t *host )
                 socklen_t i_sock_size = sizeof( struct sockaddr_storage );
                 struct  sockaddr_storage sock;
 
+                // FIXME: use net_Accept()
                 fd = accept( fd, (struct sockaddr *)&sock, &i_sock_size );
+                if( fd >= FD_SETSIZE )
+                {
+                    net_Close( fd );
+                    fd = -1;
+                }
+
                 if( fd >= 0 )
                 {
                     int i_state = 0;

src/network/io.c

@@ -69,6 +69,13 @@ int net_Socket( vlc_object_t *p_this, int i_family, int i_socktype,
         return -1;
     }
 
+    if( fd >= FD_SETSIZE )
+    {
+        msg_Err( p_this, "cannot create socket (too many already in use)" );
+        net_Close( fd );
+        return -1;
+    }
+
     /* Set to non-blocking */
 #if defined( WIN32 ) || defined( UNDER_CE )
     {

src/network/tcp.c

@@ -416,6 +416,11 @@ int __net_Accept( vlc_object_t *p_this, int *pi_fd, mtime_t i_wait )
             if( i_val < 0 )
                 msg_Err( p_this, "accept failed (%s)",
                          net_strerror( net_errno ) );
+            else if( i_val >= FD_SETSIZE )
+            {
+                net_Close( i_val ); /* avoid future overflows in FD_SET */
+                msg_Err( p_this, "accept failed (too many sockets opened)" );
+            }
             else
             {
                 /*