Commit 65745694 authored by Martin Storsjo's avatar Martin Storsjo

Don't try to shift more bits than the variable length

Shifting by more than (or equal to) the variable length is
undefined in C.

This caused the quantized spectrum values to go out of range,
triggering asserts later.

Change-Id: If81b6c8caa7b9c75941ad9d280b686d2069c968c
parent d2854e2a
......@@ -127,7 +127,10 @@ static void FDKaacEnc_quantizeLines(INT gain,
accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]);
totalShift = (16-4)-(3*(totalShift>>2));
FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */
accu>>=totalShift;
if (totalShift < 32)
accu>>=totalShift;
else
accu = 0;
quaSpectrum[line] = (SHORT)(-((LONG)(k + accu) >> (DFRACT_BITS-1-16)));
}
else if(accu > FL2FXCONST_DBL(0.0f))
......@@ -140,7 +143,10 @@ static void FDKaacEnc_quantizeLines(INT gain,
accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]);
totalShift = (16-4)-(3*(totalShift>>2));
FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */
accu>>=totalShift;
if (totalShift < 32)
accu>>=totalShift;
else
accu = 0;
quaSpectrum[line] = (SHORT)((LONG)(k + accu) >> (DFRACT_BITS-1-16));
}
else
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment