* Backport of the latest endianness fixes

include/vlc_video.h

@@ -139,9 +139,9 @@ struct picture_heap_t
     vlc_bool_t      b_allow_modify_pics;
 
     /* Stuff used for truecolor RGB planes */
-    int i_rmask, i_rrshift, i_lrshift;
-    int i_gmask, i_rgshift, i_lgshift;
-    int i_bmask, i_rbshift, i_lbshift;
+    uint32_t i_rmask; int i_rrshift, i_lrshift;
+    uint32_t i_gmask; int i_rgshift, i_lgshift;
+    uint32_t i_bmask; int i_rbshift, i_lbshift;
 
     /** Stuff used for palettized RGB planes */
     void (* pf_setpalette) ( vout_thread_t *, uint16_t *, uint16_t *, uint16_t * );

modules/video_output/opengl.c

@@ -280,22 +280,40 @@ static int Init( vout_thread_t *p_vout )
 #elif VLCGL_FORMAT == GL_RGB
 #   if VLCGL_TYPE == GL_UNSIGNED_BYTE
     p_vout->output.i_chroma = VLC_FOURCC('R','V','2','4');
+#       if defined( WORDS_BIGENDIAN )
+    p_vout->output.i_rmask = 0x00ff0000;
+    p_vout->output.i_gmask = 0x0000ff00;
+    p_vout->output.i_bmask = 0x000000ff;
+#       else
     p_vout->output.i_rmask = 0x000000ff;
     p_vout->output.i_gmask = 0x0000ff00;
     p_vout->output.i_bmask = 0x00ff0000;
+#       endif
     i_pixel_pitch = 3;
 #   else
     p_vout->output.i_chroma = VLC_FOURCC('R','V','1','6');
+#       if defined( WORDS_BIGENDIAN )
+    p_vout->output.i_rmask = 0x001f;
+    p_vout->output.i_gmask = 0x07e0;
+    p_vout->output.i_bmask = 0xf800;
+#       else
     p_vout->output.i_rmask = 0xf800;
     p_vout->output.i_gmask = 0x07e0;
     p_vout->output.i_bmask = 0x001f;
+#       endif
     i_pixel_pitch = 2;
 #   endif
 #else
     p_vout->output.i_chroma = VLC_FOURCC('R','V','3','2');
+#       if defined( WORDS_BIGENDIAN )
+    p_vout->output.i_rmask = 0xff000000;
+    p_vout->output.i_gmask = 0x00ff0000;
+    p_vout->output.i_bmask = 0x0000ff00;
+#       else
     p_vout->output.i_rmask = 0x000000ff;
     p_vout->output.i_gmask = 0x0000ff00;
     p_vout->output.i_bmask = 0x00ff0000;
+#       endif
     i_pixel_pitch = 4;
 #endif
 

src/video_output/video_output.c

@@ -1216,12 +1216,15 @@ static void MaskToShift( int *pi_left, int *pi_right, uint32_t i_mask )
     }
 
     /* Get bits */
-    i_low =  i_mask & (- (int32_t)i_mask);          /* lower bit of the mask */
-    i_high = i_mask + i_low;                       /* higher bit of the mask */
+    i_low = i_high = i_mask;
 
-    /* Transform bits into an index */
+    i_low &= - (int32_t)i_low;          /* lower bit of the mask */
+    i_high += i_low;                    /* higher bit of the mask */
+
+    /* Transform bits into an index. Also deal with i_high overflow, which
+     * is faster than changing the BinaryLog code to handle 64 bit integers. */
     i_low =  BinaryLog (i_low);
-    i_high = BinaryLog (i_high);
+    i_high = i_high ? BinaryLog (i_high) : 32;
 
     /* Update pointers and return */
     *pi_left =   i_low;