omxil: Don't trust a qcom decoder claiming to output 420 planar

Also move the color format definition to the header, making
it available to omxil.c.

In practice on Nexus One with stock firmware, it is NV21
even though the decoder says it's planar. (On CyanogenMod with
OMX decoder built from source, the decoder returns
OMX_QCOM_COLOR_FormatYVU420SemiPlanar as pixel format, though.)
Signed-off-by: Jean-Baptiste Kempf <jb@videolan.org>

modules/codec/omxil/omxil.c

@@ -391,6 +391,7 @@ static OMX_ERRORTYPE SetPortDefinition(decoder_t *p_dec, OmxPort *p_port,
 static OMX_ERRORTYPE GetPortDefinition(decoder_t *p_dec, OmxPort *p_port,
                                        es_format_t *p_fmt)
 {
+    decoder_sys_t *p_sys = p_dec->p_sys;
     OMX_PARAM_PORTDEFINITIONTYPE *def = &p_port->definition;
     OMX_ERRORTYPE omx_error;
 
@@ -409,6 +410,14 @@ static OMX_ERRORTYPE GetPortDefinition(decoder_t *p_dec, OmxPort *p_port,
         p_fmt->video.i_frame_rate = p_dec->fmt_in.video.i_frame_rate;
         p_fmt->video.i_frame_rate_base = p_dec->fmt_in.video.i_frame_rate_base;
 
+        /* Hack: Nexus One (stock firmware with binary OMX driver blob)
+         * claims to output 420Planar even though it in in practice is
+         * NV21. */
+        if(def->format.video.eColorFormat == OMX_COLOR_FormatYUV420Planar &&
+           !strncmp(p_sys->psz_component, "OMX.qcom.video.decoder",
+                    strlen("OMX.qcom.video.decoder")))
+            def->format.video.eColorFormat = OMX_QCOM_COLOR_FormatYVU420SemiPlanar;
+
         if(!GetVlcVideoFormat( def->format.video.eCompressionFormat,
                                &p_fmt->i_codec, 0 ) )
         {

modules/codec/omxil/omxil_utils.h

@@ -180,3 +180,8 @@ OMX_ERRORTYPE GetAudioParameters(OMX_HANDLETYPE handle,
     uint8_t *pi_channels, unsigned int *pi_samplerate,
     unsigned int *pi_bitrate, unsigned int *pi_bps, unsigned int *pi_blocksize);
 unsigned int GetAudioParamSize(OMX_INDEXTYPE index);
+
+/*****************************************************************************
+ * Vendor specific color formats
+ *****************************************************************************/
+#define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00

modules/codec/omxil/utils.c

@@ -36,8 +36,6 @@
 
 #include "omxil.h"
 
-#define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
-
 /*****************************************************************************
  * Events utility functions
  *****************************************************************************/