avformat/mux.c: Check malloc return value.

(cherry picked from commit 8fe6c20c7a040796758a684c54220a7a365b84b0)
Signed-off-by: Jean-Baptiste Kempf <jb@videolan.org>

modules/demux/avformat/mux.c

@@ -127,6 +127,8 @@ int OpenMux( vlc_object_t *p_this )
     p_mux->pf_delstream = DelStream;
     p_mux->pf_mux       = Mux;
     p_mux->p_sys = p_sys = malloc( sizeof( sout_mux_sys_t ) );
+    if( !p_sys )
+        return VLC_ENOMEM;
 
     p_sys->oc = avformat_alloc_context();
     p_sys->oc->oformat = file_oformat;