Don't try to shift more bits than the variable length

Shifting by more than (or equal to) the variable length is undefined in C.

Don't try to shift more bits than the variable length
Shifting by more than (or equal to) the variable length is undefined in C.
24021f19 · Martin Storsjo · 839ae290 · 24021f19
Commit 24021f19 authored Aug 10, 2012 by Martin Storsjo
Hide whitespace changes
Inline Side-by-side

Showing with 8 additions and 2 deletions

libAACenc/src/quantize.cpp libAACenc/src/quantize.cpp +8 -2

No files found.
--- a/libAACenc/src/quantize.cpp
+++ b/libAACenc/src/quantize.cpp
@@ -127,7 +127,10 @@ static void FDKaacEnc_quantizeLines(INT      gain,
      accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]);
      totalShift = (16-4)-(3*(totalShift>>2));
      FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */
-      accu>>=totalShift;
+      if (totalShift < 32)
+          accu>>=totalShift;
+      else
+          accu = 0;
      quaSpectrum[line] = (SHORT)(-((LONG)(k + accu) >> (DFRACT_BITS-1-16)));
    }
    else if(accu > FL2FXCONST_DBL(0.0f))
@@ -140,7 +143,10 @@ static void FDKaacEnc_quantizeLines(INT      gain,
      accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]);
      totalShift = (16-4)-(3*(totalShift>>2));
      FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */
-      accu>>=totalShift;
+      if (totalShift < 32)
+          accu>>=totalShift;
+      else
+          accu = 0;
      quaSpectrum[line] = (SHORT)((LONG)(k + accu) >> (DFRACT_BITS-1-16));
    }
    else