vout: return an error if render filter chain cannot be created

src/video_output/display.c

@@ -410,7 +410,7 @@ struct vout_display_owner_sys_t {
     } event;
 };
 
-static void VoutDisplayCreateRender(vout_display_t *vd)
+static int VoutDisplayCreateRender(vout_display_t *vd)
 {
     vout_display_owner_sys_t *osys = vd->owner.sys;
 
@@ -433,7 +433,7 @@ static void VoutDisplayCreateRender(vout_display_t *vd)
 
     const bool convert = memcmp(&v_src, &v_dst_cmp, sizeof(v_src)) != 0;
     if (!convert)
-        return;
+        return 0;
 
     msg_Dbg(vd, "A filter to adapt decoder to display is needed");
 
@@ -466,8 +466,12 @@ static void VoutDisplayCreateRender(vout_display_t *vd)
             break;
     }
     es_format_Clean(&src);
-    if (!filter)
+
+    if (filter == NULL) {
         msg_Err(vd, "Failed to adapt decoder format to display");
+        return -1;
+    }
+    return 0;
 }
 
 static void VoutDisplayDestroyRender(vout_display_t *vd)
@@ -478,11 +482,12 @@ static void VoutDisplayDestroyRender(vout_display_t *vd)
         filter_chain_Delete(osys->filters);
 }
 
-static void VoutDisplayResetRender(vout_display_t *vd)
+static int VoutDisplayResetRender(vout_display_t *vd)
 {
     VoutDisplayDestroyRender(vd);
-    VoutDisplayCreateRender(vd);
+    return VoutDisplayCreateRender(vd);
 }
+
 static void VoutDisplayEventMouse(vout_display_t *vd, int event, va_list args)
 {
     vout_display_owner_sys_t *osys = vd->owner.sys;
@@ -1266,7 +1271,10 @@ static vout_display_t *DisplayNew(vout_thread_t *vout,
     if (!p_display)
         goto error;
 
-    VoutDisplayCreateRender(p_display);
+    if (VoutDisplayCreateRender(p_display)) {
+        vout_display_Delete(p_display);
+        goto error;
+    }
 
     /* Setup delayed request */
     if (osys->sar.num != source->i_sar_num ||