macosx: reset window size to the one before video was started

close #6019

modules/gui/macosx/MainWindow.h

@@ -161,6 +161,8 @@
 
     VLCWindow *o_extra_video_window;
     id o_current_video_window;
+
+    NSRect frameBeforePlayback;
 }
 + (VLCMainWindow *)sharedInstance;
 @property (readonly) BOOL fullscreen;

modules/gui/macosx/MainWindow.m

@@ -1887,9 +1887,13 @@ static VLCMainWindow *_o_sharedInstance = nil;
     BOOL b_videoPlayback = [[VLCMain sharedInstance] activeVideoPlayback];
 
     if (b_videoPlayback) {
+        frameBeforePlayback = [self frame];
+
         // look for 'start at fullscreen'
         [[VLCMain sharedInstance] fullscreenChanged];
     } else {
+        [[self animator] setFrame:frameBeforePlayback display:YES];
+
         [self makeFirstResponder: nil];
         [o_detached_video_window orderOut: nil];
         if (o_extra_video_window)