dvdnav: clean up, fix error message and probing encoding (fix #3816)

Paths for error message must be UTF-8; they were locale.
Paths for probing must be UTF-8 also, as vlc_fopen() does the
conversion to locale internally.
(cherry picked from commit f2513588)

Conflicts:

	modules/access/dvdnav.c

modules/access/dvdnav.c

@@ -187,7 +187,7 @@ static int Open( vlc_object_t *p_this )
     demux_sys_t *p_sys;
     dvdnav_t    *p_dvdnav;
     int         i_angle;
-    char        *psz_name;
+    char        *psz_file;
     char        *psz_code;
 
     if( !p_demux->psz_path || !*p_demux->psz_path )
@@ -196,38 +196,44 @@ static int Open( vlc_object_t *p_this )
         if( !p_demux->psz_access || !*p_demux->psz_access )
             return VLC_EGENERIC;
 
-        psz_name = var_CreateGetString( p_this, "dvd" );
-        if( !psz_name )
-        {
-            psz_name = strdup("");
-        }
+        psz_file = var_InheritString( p_this, "dvd" );
     }
     else
-        psz_name = ToLocaleDup( p_demux->psz_path );
+        psz_file = strdup( p_demux->psz_path );
 
 #ifdef WIN32
-    /* Remove trailing backslash, otherwise dvdnav_open will fail */
-    if( *psz_name && *(psz_name + strlen(psz_name) - 1) == '\\' )
+    if( psz_file != NULL )
     {
-        *(psz_name + strlen(psz_name) - 1) = '\0';
+        /* Remove trailing backslash, otherwise dvdnav_open will fail */
+        size_t flen = strlen( psz_file );
+        if( flen > 0 && psz_file[flen - 1] == '\\' )
+            psz_file[flen - 1] = '\0';
     }
+    else
+        psz_file = strdup("");
 #endif
+    if( unlikely(psz_file == NULL) )
+        return VLC_EGENERIC;
 
     /* Try some simple probing to avoid going through dvdnav_open too often */
-    if( ProbeDVD( p_demux, psz_name ) != VLC_SUCCESS )
+    if( ProbeDVD( p_demux, psz_file ) != VLC_SUCCESS )
     {
-        free( psz_name );
+        free( psz_file );
         return VLC_EGENERIC;
     }
 
     /* Open dvdnav */
-    if( dvdnav_open( &p_dvdnav, psz_name ) != DVDNAV_STATUS_OK )
+    const char *psz_path = ToLocale( psz_file );
+    if( dvdnav_open( &p_dvdnav, psz_path ) != DVDNAV_STATUS_OK )
+        p_dvdnav = NULL;
+    LocaleFree( psz_path );
+    if( p_dvdnav == NULL )
     {
-        msg_Warn( p_demux, "cannot open dvdnav" );
-        free( psz_name );
+        msg_Warn( p_demux, "cannot open DVD (%s)", psz_file);
+        free( psz_file );
         return VLC_EGENERIC;
     }
-    free( psz_name );
+    free( psz_file );
 
     /* Fill p_demux field */
     DEMUX_INIT_COMMON(); p_sys = p_demux->p_sys;