macosx: better check if we can open the new vout within the main window

fixes #9919

modules/gui/macosx/VLCVoutWindowController.h

@@ -38,6 +38,8 @@
     // save the status level if at least one video window is on status level
     NSUInteger i_statusLevelWindowCounter;
     NSInteger i_currentWindowLevel;
+
+    BOOL b_mainwindow_has_video;
 }
 
 @property (readonly, nonatomic) NSInteger currentWindowLevel;

modules/gui/macosx/VLCVoutWindowController.m

@@ -129,10 +129,11 @@
         [[VLCMainWindow sharedInstance] setNonembedded:YES];
         b_nonembedded = YES;
     } else {
-        if ((var_InheritBool(VLCIntf, "embedded-video") && !b_multiple_vout_windows)) {
+        if ((var_InheritBool(VLCIntf, "embedded-video") && !b_mainwindow_has_video)) {
             // setup embedded video
             o_vout_view = [[[VLCMainWindow sharedInstance] videoView] retain];
             o_new_video_window = [[VLCMainWindow sharedInstance] retain];
+            b_mainwindow_has_video = YES;
             b_nonembedded = NO;
         } else {
             // setup detached window with controls
@@ -243,6 +244,9 @@
         return;
     }
 
+    if ([o_window class] == [VLCMainWindow class])
+        b_mainwindow_has_video = NO;
+
     if ([o_window fullscreen] && ![[VLCMainWindow sharedInstance] nativeFullscreenMode])
         [o_window leaveFullscreen];