macosx: don't let the video window get larger than the screen it is on (close #5875)

(cherry picked from commit 93b21142)

modules/video_output/macosx.m

@@ -356,6 +356,7 @@ static int Control (vout_display_t *vd, int query, va_list ap)
             NSRect windowFrame = [o_window frame];
             NSRect glViewFrame = [sys->glView frame];
             NSSize windowMinSize = [o_window minSize];
+            NSSize windowMaxSize = [o_window constrainFrameRect: windowFrame toScreen: [o_window screen]].size;
 
             topleftbase.x = 0;
             topleftbase.y = windowFrame.size.height;
@@ -370,6 +371,12 @@ static int Control (vout_display_t *vd, int query, va_list ap)
             if (i_height < windowMinSize.height)
                 i_height = windowMinSize.height;
 
+            /* don't make the window than its screen */
+            if (i_width > windowMaxSize.width)
+                i_width = windowMaxSize.width;
+            if (i_height > windowMaxSize.height)
+                i_height = windowMaxSize.height;
+
             if( i_height != glViewFrame.size.height || i_width != glViewFrame.size.width )
             {
                 new_frame.size.width = windowFrame.size.width - glViewFrame.size.width + i_width;