mmal/codec: Flush the decoder when running out of buffers

Normally the decoder should not run out of input buffers as it is fed with
packetized data and for each input block a picture shall be generated. In rare
situations the decoder seems to fail internally and does stop generating
pictures. In this case the input buffers are not released either. If this
happens the whole pipeline stalls and vlc buffers input data until system runs
out of memory. To avoid this error case force the decoder to flush its port
if it runs out of input buffers. This causes a short image distortion but the
decoder won't block anymore.
Signed-off-by: Julian Scheel <julian@jusst.de>
Signed-off-by: Rémi Denis-Courmont <remi@remlab.net>

modules/hw/mmal/codec.c

@@ -481,6 +481,39 @@ static void fill_output_port(decoder_t *dec)
             break;
 }
 
+static int flush_decoder(decoder_t *dec)
+{
+    decoder_sys_t *sys = dec->p_sys;
+    MMAL_STATUS_T status;
+    int ret = 0;
+
+    msg_Dbg(dec, "Flushing decoder ports...");
+    mmal_port_disable(sys->output);
+    mmal_port_disable(sys->input);
+    mmal_port_flush(sys->output);
+    mmal_port_flush(sys->input);
+    msg_Dbg(dec, "Ports flushed, returning to normal operation");
+    status = mmal_port_enable(sys->input, input_port_cb);
+    if (status != MMAL_SUCCESS) {
+        msg_Err(dec, "Failed to enable input port %s (status=%"PRIx32" %s)",
+                sys->input->name, status, mmal_status_to_string(status));
+        ret = VLC_EGENERIC;
+        goto out;
+    }
+    msg_Dbg(dec, "Input enabled");
+    status = mmal_port_enable(sys->output, output_port_cb);
+    if (status != MMAL_SUCCESS) {
+        msg_Err(dec, "Failed to enable output port %s (status=%"PRIx32" %s)",
+                sys->output->name, status, mmal_status_to_string(status));
+        ret = VLC_EGENERIC;
+        goto out;
+    }
+    msg_Dbg(dec, "Output enabled");
+
+out:
+    return ret;
+}
+
 static picture_t *decode(decoder_t *dec, block_t **pblock)
 {
     decoder_sys_t *sys = dec->p_sys;
@@ -546,6 +579,14 @@ static picture_t *decode(decoder_t *dec, block_t **pblock)
         buffer = mmal_queue_timedwait(sys->input_pool->queue, 2);
         if (!buffer) {
             msg_Err(dec, "Failed to retrieve buffer header for input data");
+
+            /* Flush the decoder output port to make sure we don't get stuck
+             * forever. This shall normally not happen but may happen if the
+             * decoder goes mad. */
+            if (flush_decoder(dec)) {
+                msg_Err(dec, "Failed to flush decoder, something seems really broken...");
+                dec->b_error = true;
+            }
             break;
         }
         mmal_buffer_header_reset(buffer);