Skip to content

F
ffmpeg-mt
Commit 9e2accac authored by astrange's avatar astrange
Browse files
Options

H264: Fix out of bounds reads in SSSE3 MC 

Reading above src[-2] isn't safe, so move loads and palignr ahead
3 pixels to load starting at the first pixel actually used.

Fixes issue941.


git-svn-id: file:///var/local/repositories/ffmpeg/trunk@18999 9553f0bf-9b14-0410-a0b8-cfaf0461ba5b
parent 85c84d06
Show whitespace changes
Inline Side-by-side
Showing with 38 additions and 38 deletions
libavcodec/x86/h264dsp_mmx.c
View file @ 9e2accac
...@@ -1497,8 +1497,8 @@ static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, ...@@ -1497,8 +1497,8 @@ static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst,
"movdqa %6, %%xmm14 \n\t"\ "movdqa %6, %%xmm14 \n\t"\
"movdqa %7, %%xmm13 \n\t"\ "movdqa %7, %%xmm13 \n\t"\
"1: \n\t"\ "1: \n\t"\
"lddqu 3(%0), %%xmm1 \n\t"\ "lddqu 6(%0), %%xmm1 \n\t"\
"lddqu -5(%0), %%xmm7 \n\t"\ "lddqu -2(%0), %%xmm7 \n\t"\
"movdqa %%xmm1, %%xmm0 \n\t"\ "movdqa %%xmm1, %%xmm0 \n\t"\
"punpckhbw %%xmm15, %%xmm1 \n\t"\ "punpckhbw %%xmm15, %%xmm1 \n\t"\
"punpcklbw %%xmm15, %%xmm0 \n\t"\ "punpcklbw %%xmm15, %%xmm0 \n\t"\
...@@ -1509,20 +1509,20 @@ static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, ...@@ -1509,20 +1509,20 @@ static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst,
"movdqa %%xmm0, %%xmm8 \n\t"\ "movdqa %%xmm0, %%xmm8 \n\t"\
"movdqa %%xmm1, %%xmm4 \n\t"\ "movdqa %%xmm1, %%xmm4 \n\t"\
"movdqa %%xmm0, %%xmm9 \n\t"\ "movdqa %%xmm0, %%xmm9 \n\t"\
"movdqa %%xmm1, %%xmm5 \n\t"\ "movdqa %%xmm0, %%xmm12 \n\t"\
"movdqa %%xmm0, %%xmm10 \n\t"\ "movdqa %%xmm1, %%xmm11 \n\t"\
"palignr $6, %%xmm0, %%xmm5 \n\t"\ "palignr $10,%%xmm0, %%xmm11\n\t"\
"palignr $6, %%xmm7, %%xmm10\n\t"\ "palignr $10,%%xmm7, %%xmm12\n\t"\
"palignr $8, %%xmm0, %%xmm4 \n\t"\ "palignr $2, %%xmm0, %%xmm4 \n\t"\
"palignr $8, %%xmm7, %%xmm9 \n\t"\ "palignr $2, %%xmm7, %%xmm9 \n\t"\
"palignr $10,%%xmm0, %%xmm3 \n\t"\ "palignr $4, %%xmm0, %%xmm3 \n\t"\
"palignr $10,%%xmm7, %%xmm8 \n\t"\ "palignr $4, %%xmm7, %%xmm8 \n\t"\
"paddw %%xmm1, %%xmm5 \n\t"\ "palignr $6, %%xmm0, %%xmm2 \n\t"\
"paddw %%xmm0, %%xmm10 \n\t"\ "palignr $6, %%xmm7, %%xmm6 \n\t"\
"palignr $12,%%xmm0, %%xmm2 \n\t"\ "paddw %%xmm0 ,%%xmm11 \n\t"\
"palignr $12,%%xmm7, %%xmm6 \n\t"\ "palignr $8, %%xmm0, %%xmm1 \n\t"\
"palignr $14,%%xmm0, %%xmm1 \n\t"\ "palignr $8, %%xmm7, %%xmm0 \n\t"\
"palignr $14,%%xmm7, %%xmm0 \n\t"\ "paddw %%xmm12,%%xmm7 \n\t"\
"paddw %%xmm3, %%xmm2 \n\t"\ "paddw %%xmm3, %%xmm2 \n\t"\
"paddw %%xmm8, %%xmm6 \n\t"\ "paddw %%xmm8, %%xmm6 \n\t"\
"paddw %%xmm4, %%xmm1 \n\t"\ "paddw %%xmm4, %%xmm1 \n\t"\
...@@ -1531,13 +1531,13 @@ static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, ...@@ -1531,13 +1531,13 @@ static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst,
"psllw $2, %%xmm6 \n\t"\ "psllw $2, %%xmm6 \n\t"\
"psubw %%xmm1, %%xmm2 \n\t"\ "psubw %%xmm1, %%xmm2 \n\t"\
"psubw %%xmm0, %%xmm6 \n\t"\ "psubw %%xmm0, %%xmm6 \n\t"\
"paddw %%xmm13,%%xmm5 \n\t"\ "paddw %%xmm13,%%xmm11 \n\t"\
"paddw %%xmm13,%%xmm10 \n\t"\ "paddw %%xmm13,%%xmm7 \n\t"\
"pmullw %%xmm14,%%xmm2 \n\t"\ "pmullw %%xmm14,%%xmm2 \n\t"\
"pmullw %%xmm14,%%xmm6 \n\t"\ "pmullw %%xmm14,%%xmm6 \n\t"\
"lddqu (%2), %%xmm3 \n\t"\ "lddqu (%2), %%xmm3 \n\t"\
"paddw %%xmm5, %%xmm2 \n\t"\ "paddw %%xmm11,%%xmm2 \n\t"\
"paddw %%xmm10,%%xmm6 \n\t"\ "paddw %%xmm7, %%xmm6 \n\t"\
"psraw $5, %%xmm2 \n\t"\ "psraw $5, %%xmm2 \n\t"\
"psraw $5, %%xmm6 \n\t"\ "psraw $5, %%xmm6 \n\t"\
"packuswb %%xmm2,%%xmm6 \n\t"\ "packuswb %%xmm2,%%xmm6 \n\t"\
...@@ -1577,7 +1577,7 @@ static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst, ...@@ -1577,7 +1577,7 @@ static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst,
);\ );\
do{\ do{\
__asm__ volatile(\ __asm__ volatile(\
"lddqu -5(%0), %%xmm1 \n\t"\ "lddqu -2(%0), %%xmm1 \n\t"\
"movdqa %%xmm1, %%xmm0 \n\t"\ "movdqa %%xmm1, %%xmm0 \n\t"\
"punpckhbw %%xmm7, %%xmm1 \n\t"\ "punpckhbw %%xmm7, %%xmm1 \n\t"\
"punpcklbw %%xmm7, %%xmm0 \n\t"\ "punpcklbw %%xmm7, %%xmm0 \n\t"\
...@@ -1585,20 +1585,20 @@ static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst, ...@@ -1585,20 +1585,20 @@ static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst,
"movdqa %%xmm1, %%xmm3 \n\t"\ "movdqa %%xmm1, %%xmm3 \n\t"\
"movdqa %%xmm1, %%xmm4 \n\t"\ "movdqa %%xmm1, %%xmm4 \n\t"\
"movdqa %%xmm1, %%xmm5 \n\t"\ "movdqa %%xmm1, %%xmm5 \n\t"\
"palignr $6, %%xmm0, %%xmm5 \n\t"\ "palignr $2, %%xmm0, %%xmm4 \n\t"\
"palignr $8, %%xmm0, %%xmm4 \n\t"\ "palignr $4, %%xmm0, %%xmm3 \n\t"\
"palignr $10,%%xmm0, %%xmm3 \n\t"\ "palignr $6, %%xmm0, %%xmm2 \n\t"\
"paddw %%xmm1, %%xmm5 \n\t"\ "palignr $8, %%xmm0, %%xmm1 \n\t"\
"palignr $12,%%xmm0, %%xmm2 \n\t"\ "palignr $10,%%xmm0, %%xmm5 \n\t"\
"palignr $14,%%xmm0, %%xmm1 \n\t"\ "paddw %%xmm5, %%xmm0 \n\t"\
"paddw %%xmm3, %%xmm2 \n\t"\ "paddw %%xmm3, %%xmm2 \n\t"\
"paddw %%xmm4, %%xmm1 \n\t"\ "paddw %%xmm4, %%xmm1 \n\t"\
"psllw $2, %%xmm2 \n\t"\ "psllw $2, %%xmm2 \n\t"\
"movq (%2), %%xmm3 \n\t"\ "movq (%2), %%xmm3 \n\t"\
"psubw %%xmm1, %%xmm2 \n\t"\ "psubw %%xmm1, %%xmm2 \n\t"\
"paddw %5, %%xmm5 \n\t"\ "paddw %5, %%xmm0 \n\t"\
"pmullw %%xmm6, %%xmm2 \n\t"\ "pmullw %%xmm6, %%xmm2 \n\t"\
"paddw %%xmm5, %%xmm2 \n\t"\ "paddw %%xmm0, %%xmm2 \n\t"\
"psraw $5, %%xmm2 \n\t"\ "psraw $5, %%xmm2 \n\t"\
"packuswb %%xmm2, %%xmm2 \n\t"\ "packuswb %%xmm2, %%xmm2 \n\t"\
"pavgb %%xmm3, %%xmm2 \n\t"\ "pavgb %%xmm3, %%xmm2 \n\t"\
...@@ -1621,7 +1621,7 @@ static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uin ...@@ -1621,7 +1621,7 @@ static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uin
"pxor %%xmm7, %%xmm7 \n\t"\ "pxor %%xmm7, %%xmm7 \n\t"\
"movdqa %5, %%xmm6 \n\t"\ "movdqa %5, %%xmm6 \n\t"\
"1: \n\t"\ "1: \n\t"\
"lddqu -5(%0), %%xmm1 \n\t"\ "lddqu -2(%0), %%xmm1 \n\t"\
"movdqa %%xmm1, %%xmm0 \n\t"\ "movdqa %%xmm1, %%xmm0 \n\t"\
"punpckhbw %%xmm7, %%xmm1 \n\t"\ "punpckhbw %%xmm7, %%xmm1 \n\t"\
"punpcklbw %%xmm7, %%xmm0 \n\t"\ "punpcklbw %%xmm7, %%xmm0 \n\t"\
...@@ -1629,19 +1629,19 @@ static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uin ...@@ -1629,19 +1629,19 @@ static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uin
"movdqa %%xmm1, %%xmm3 \n\t"\ "movdqa %%xmm1, %%xmm3 \n\t"\
"movdqa %%xmm1, %%xmm4 \n\t"\ "movdqa %%xmm1, %%xmm4 \n\t"\
"movdqa %%xmm1, %%xmm5 \n\t"\ "movdqa %%xmm1, %%xmm5 \n\t"\
"palignr $6, %%xmm0, %%xmm5 \n\t"\ "palignr $2, %%xmm0, %%xmm4 \n\t"\
"palignr $8, %%xmm0, %%xmm4 \n\t"\ "palignr $4, %%xmm0, %%xmm3 \n\t"\
"palignr $10,%%xmm0, %%xmm3 \n\t"\ "palignr $6, %%xmm0, %%xmm2 \n\t"\
"paddw %%xmm1, %%xmm5 \n\t"\ "palignr $8, %%xmm0, %%xmm1 \n\t"\
"palignr $12,%%xmm0, %%xmm2 \n\t"\ "palignr $10,%%xmm0, %%xmm5 \n\t"\
"palignr $14,%%xmm0, %%xmm1 \n\t"\ "paddw %%xmm5, %%xmm0 \n\t"\
"paddw %%xmm3, %%xmm2 \n\t"\ "paddw %%xmm3, %%xmm2 \n\t"\
"paddw %%xmm4, %%xmm1 \n\t"\ "paddw %%xmm4, %%xmm1 \n\t"\
"psllw $2, %%xmm2 \n\t"\ "psllw $2, %%xmm2 \n\t"\
"psubw %%xmm1, %%xmm2 \n\t"\ "psubw %%xmm1, %%xmm2 \n\t"\
"paddw %6, %%xmm5 \n\t"\ "paddw %6, %%xmm0 \n\t"\
"pmullw %%xmm6, %%xmm2 \n\t"\ "pmullw %%xmm6, %%xmm2 \n\t"\
"paddw %%xmm5, %%xmm2 \n\t"\ "paddw %%xmm0, %%xmm2 \n\t"\
"psraw $5, %%xmm2 \n\t"\ "psraw $5, %%xmm2 \n\t"\
"packuswb %%xmm2, %%xmm2 \n\t"\ "packuswb %%xmm2, %%xmm2 \n\t"\
OP(%%xmm2, (%1), %%xmm4, q)\ OP(%%xmm2, (%1), %%xmm4, q)\
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment