Skip to content

F
ffmpeg-mt
Commit df939815 authored by lorenm's avatar lorenm
Browse files
Options

40% faster mmx2 sad 


git-svn-id: file:///var/local/repositories/ffmpeg/trunk@8991 9553f0bf-9b14-0410-a0b8-cfaf0461ba5b
parent eb5d5e2a
Hide whitespace changes
Inline Side-by-side
Showing with 121 additions and 90 deletions
libavcodec/i386/motion_est_mmx.c
View file @ df939815
...@@ -70,74 +70,98 @@ static inline void sad8_1_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h) ...@@ -70,74 +70,98 @@ static inline void sad8_1_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h)
static inline void sad8_1_mmx2(uint8_t *blk1, uint8_t *blk2, int stride, int h) static inline void sad8_1_mmx2(uint8_t *blk1, uint8_t *blk2, int stride, int h)
{ {
long len= -(stride*h);
asm volatile( asm volatile(
ASMALIGN(4) ASMALIGN(4)
"1: \n\t" "1: \n\t"
"movq (%1, %%"REG_a"), %%mm0 \n\t" "movq (%1), %%mm0 \n\t"
"psadbw (%2, %%"REG_a"), %%mm0 \n\t" "movq (%1, %3), %%mm1 \n\t"
"add %3, %%"REG_a" \n\t" "psadbw (%2), %%mm0 \n\t"
"movq (%1, %%"REG_a"), %%mm1 \n\t" "psadbw (%2, %3), %%mm1 \n\t"
"psadbw (%2, %%"REG_a"), %%mm1 \n\t"
"paddw %%mm1, %%mm0 \n\t"
"paddw %%mm0, %%mm6 \n\t" "paddw %%mm0, %%mm6 \n\t"
"add %3, %%"REG_a" \n\t" "paddw %%mm1, %%mm6 \n\t"
" js 1b \n\t" "lea (%1,%3,2), %1 \n\t"
: "+a" (len) "lea (%2,%3,2), %2 \n\t"
: "r" (blk1 - len), "r" (blk2 - len), "r" ((long)stride) "sub $2, %0 \n\t"
" jg 1b \n\t"
: "+r" (h), "+r" (blk1), "+r" (blk2)
: "r" ((long)stride)
); );
} }
static inline void sad8_2_mmx2(uint8_t *blk1a, uint8_t *blk1b, uint8_t *blk2, int stride, int h) static inline void sad8_x2a_mmx2(uint8_t *blk1, uint8_t *blk2, int stride, int h)
{ {
long len= -(stride*h);
asm volatile( asm volatile(
ASMALIGN(4) ASMALIGN(4)
"1: \n\t" "1: \n\t"
"movq (%1, %%"REG_a"), %%mm0 \n\t" "movq (%1), %%mm0 \n\t"
"pavgb (%2, %%"REG_a"), %%mm0 \n\t" "movq (%1, %3), %%mm1 \n\t"
"psadbw (%3, %%"REG_a"), %%mm0 \n\t" "pavgb 1(%1), %%mm0 \n\t"
"add %4, %%"REG_a" \n\t" "pavgb 1(%1, %3), %%mm1 \n\t"
"movq (%1, %%"REG_a"), %%mm1 \n\t" "psadbw (%2), %%mm0 \n\t"
"pavgb (%2, %%"REG_a"), %%mm1 \n\t" "psadbw (%2, %3), %%mm1 \n\t"
"psadbw (%3, %%"REG_a"), %%mm1 \n\t"
"paddw %%mm1, %%mm0 \n\t"
"paddw %%mm0, %%mm6 \n\t" "paddw %%mm0, %%mm6 \n\t"
"add %4, %%"REG_a" \n\t" "paddw %%mm1, %%mm6 \n\t"
" js 1b \n\t" "lea (%1,%3,2), %1 \n\t"
: "+a" (len) "lea (%2,%3,2), %2 \n\t"
: "r" (blk1a - len), "r" (blk1b -len), "r" (blk2 - len), "r" ((long)stride) "sub $2, %0 \n\t"
" jg 1b \n\t"
: "+r" (h), "+r" (blk1), "+r" (blk2)
: "r" ((long)stride)
);
}
static inline void sad8_y2a_mmx2(uint8_t *blk1, uint8_t *blk2, int stride, int h)
{
asm volatile(
"movq (%1), %%mm0 \n\t"
"add %3, %1 \n\t"
ASMALIGN(4)
"1: \n\t"
"movq (%1), %%mm1 \n\t"
"movq (%1, %3), %%mm2 \n\t"
"pavgb %%mm1, %%mm0 \n\t"
"pavgb %%mm2, %%mm1 \n\t"
"psadbw (%2), %%mm0 \n\t"
"psadbw (%2, %3), %%mm1 \n\t"
"paddw %%mm0, %%mm6 \n\t"
"paddw %%mm1, %%mm6 \n\t"
"movq %%mm2, %%mm0 \n\t"
"lea (%1,%3,2), %1 \n\t"
"lea (%2,%3,2), %2 \n\t"
"sub $2, %0 \n\t"
" jg 1b \n\t"
: "+r" (h), "+r" (blk1), "+r" (blk2)
: "r" ((long)stride)
); );
} }
static inline void sad8_4_mmx2(uint8_t *blk1, uint8_t *blk2, int stride, int h) static inline void sad8_4_mmx2(uint8_t *blk1, uint8_t *blk2, int stride, int h)
{ //FIXME reuse src {
long len= -(stride*h);
asm volatile( asm volatile(
"movq "MANGLE(bone)", %%mm5 \n\t" "movq "MANGLE(bone)", %%mm5 \n\t"
"movq (%1), %%mm0 \n\t"
"pavgb 1(%1), %%mm0 \n\t"
"add %3, %1 \n\t"
ASMALIGN(4) ASMALIGN(4)
"1: \n\t" "1: \n\t"
"movq (%1, %%"REG_a"), %%mm0 \n\t" "movq (%1), %%mm1 \n\t"
"movq 1(%1, %%"REG_a"), %%mm1 \n\t" "movq (%1,%3), %%mm2 \n\t"
"pavgb (%2, %%"REG_a"), %%mm0 \n\t" "pavgb 1(%1), %%mm1 \n\t"
"pavgb 1(%2, %%"REG_a"), %%mm1 \n\t" "pavgb 1(%1,%3), %%mm2 \n\t"
"psubusb %%mm5, %%mm1 \n\t" "psubusb %%mm5, %%mm1 \n\t"
"pavgb %%mm1, %%mm0 \n\t" "pavgb %%mm1, %%mm0 \n\t"
"psadbw (%3, %%"REG_a"), %%mm0 \n\t" "pavgb %%mm2, %%mm1 \n\t"
"add %4, %%"REG_a" \n\t" "psadbw (%2), %%mm0 \n\t"
"movq (%1, %%"REG_a"), %%mm2 \n\t" "psadbw (%2,%3), %%mm1 \n\t"
"movq 1(%1, %%"REG_a"), %%mm3 \n\t"
"pavgb (%2, %%"REG_a"), %%mm2 \n\t"
"pavgb 1(%2, %%"REG_a"), %%mm3 \n\t"
"psubusb %%mm5, %%mm3 \n\t"
"pavgb %%mm3, %%mm2 \n\t"
"psadbw (%3, %%"REG_a"), %%mm2 \n\t"
"paddw %%mm2, %%mm0 \n\t"
"paddw %%mm0, %%mm6 \n\t" "paddw %%mm0, %%mm6 \n\t"
"add %4, %%"REG_a" \n\t" "paddw %%mm1, %%mm6 \n\t"
" js 1b \n\t" "movq %%mm2, %%mm0 \n\t"
: "+a" (len) "lea (%1,%3,2), %1 \n\t"
: "r" (blk1 - len), "r" (blk1 - len + stride), "r" (blk2 - len), "r" ((long)stride) "lea (%2,%3,2), %2 \n\t"
"sub $2, %0 \n\t"
" jg 1b \n\t"
: "+r" (h), "+r" (blk1), "+r" (blk2)
: "r" ((long)stride)
); );
} }
...@@ -183,45 +207,48 @@ static inline void sad8_4_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h) ...@@ -183,45 +207,48 @@ static inline void sad8_4_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h)
{ {
long len= -(stride*h); long len= -(stride*h);
asm volatile( asm volatile(
ASMALIGN(4)
"1: \n\t"
"movq (%1, %%"REG_a"), %%mm0 \n\t" "movq (%1, %%"REG_a"), %%mm0 \n\t"
"movq (%2, %%"REG_a"), %%mm1 \n\t"
"movq %%mm0, %%mm4 \n\t"
"movq %%mm1, %%mm2 \n\t"
"punpcklbw %%mm7, %%mm0 \n\t"
"punpcklbw %%mm7, %%mm1 \n\t"
"punpckhbw %%mm7, %%mm4 \n\t"
"punpckhbw %%mm7, %%mm2 \n\t"
"paddw %%mm1, %%mm0 \n\t"
"paddw %%mm2, %%mm4 \n\t"
"movq 1(%1, %%"REG_a"), %%mm2 \n\t" "movq 1(%1, %%"REG_a"), %%mm2 \n\t"
"movq 1(%2, %%"REG_a"), %%mm3 \n\t" "movq %%mm0, %%mm1 \n\t"
"movq %%mm2, %%mm1 \n\t" "movq %%mm2, %%mm3 \n\t"
"punpcklbw %%mm7, %%mm2 \n\t" "punpcklbw %%mm7, %%mm0 \n\t"
"punpckhbw %%mm7, %%mm1 \n\t" "punpckhbw %%mm7, %%mm1 \n\t"
"paddw %%mm0, %%mm2 \n\t" "punpcklbw %%mm7, %%mm2 \n\t"
"paddw %%mm4, %%mm1 \n\t" "punpckhbw %%mm7, %%mm3 \n\t"
"movq %%mm3, %%mm4 \n\t" "paddw %%mm2, %%mm0 \n\t"
"punpcklbw %%mm7, %%mm3 \n\t" "paddw %%mm3, %%mm1 \n\t"
"punpckhbw %%mm7, %%mm4 \n\t" ASMALIGN(4)
"paddw %%mm3, %%mm2 \n\t" "1: \n\t"
"paddw %%mm4, %%mm1 \n\t" "movq (%2, %%"REG_a"), %%mm2 \n\t"
"movq (%3, %%"REG_a"), %%mm3 \n\t" "movq 1(%2, %%"REG_a"), %%mm4 \n\t"
"movq (%3, %%"REG_a"), %%mm4 \n\t" "movq %%mm2, %%mm3 \n\t"
"paddw %%mm5, %%mm2 \n\t" "movq %%mm4, %%mm5 \n\t"
"punpcklbw %%mm7, %%mm2 \n\t"
"punpckhbw %%mm7, %%mm3 \n\t"
"punpcklbw %%mm7, %%mm4 \n\t"
"punpckhbw %%mm7, %%mm5 \n\t"
"paddw %%mm4, %%mm2 \n\t"
"paddw %%mm5, %%mm3 \n\t"
"movq 16+"MANGLE(round_tab)", %%mm5 \n\t"
"paddw %%mm2, %%mm0 \n\t"
"paddw %%mm3, %%mm1 \n\t"
"paddw %%mm5, %%mm0 \n\t"
"paddw %%mm5, %%mm1 \n\t" "paddw %%mm5, %%mm1 \n\t"
"psrlw $2, %%mm2 \n\t" "movq (%3, %%"REG_a"), %%mm4 \n\t"
"movq (%3, %%"REG_a"), %%mm5 \n\t"
"psrlw $2, %%mm0 \n\t"
"psrlw $2, %%mm1 \n\t" "psrlw $2, %%mm1 \n\t"
"packuswb %%mm1, %%mm2 \n\t" "packuswb %%mm1, %%mm0 \n\t"
"psubusb %%mm2, %%mm3 \n\t" "psubusb %%mm0, %%mm4 \n\t"
"psubusb %%mm4, %%mm2 \n\t" "psubusb %%mm5, %%mm0 \n\t"
"por %%mm3, %%mm2 \n\t" "por %%mm4, %%mm0 \n\t"
"movq %%mm2, %%mm0 \n\t" "movq %%mm0, %%mm4 \n\t"
"punpcklbw %%mm7, %%mm0 \n\t" "punpcklbw %%mm7, %%mm0 \n\t"
"punpckhbw %%mm7, %%mm2 \n\t" "punpckhbw %%mm7, %%mm4 \n\t"
"paddw %%mm2, %%mm0 \n\t"
"paddw %%mm0, %%mm6 \n\t" "paddw %%mm0, %%mm6 \n\t"
"paddw %%mm4, %%mm6 \n\t"
"movq %%mm2, %%mm0 \n\t"
"movq %%mm3, %%mm1 \n\t"
"add %4, %%"REG_a" \n\t" "add %4, %%"REG_a" \n\t"
" js 1b \n\t" " js 1b \n\t"
: "+a" (len) : "+a" (len)
...@@ -255,6 +282,15 @@ static inline int sum_mmx2(void) ...@@ -255,6 +282,15 @@ static inline int sum_mmx2(void)
return ret; return ret;
} }
static inline void sad8_x2a_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h)
{
sad8_2_mmx(blk1, blk1+1, blk2, stride, h);
}
static inline void sad8_y2a_mmx(uint8_t *blk1, uint8_t *blk2, int stride, int h)
{
sad8_2_mmx(blk1, blk1+stride, blk2, stride, h);
}
#define PIX_SAD(suf)\ #define PIX_SAD(suf)\
static int sad8_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\ static int sad8_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, int h)\
...@@ -272,11 +308,10 @@ static int sad8_x2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, in ...@@ -272,11 +308,10 @@ static int sad8_x2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, in
assert(h==8);\ assert(h==8);\
asm volatile("pxor %%mm7, %%mm7 \n\t"\ asm volatile("pxor %%mm7, %%mm7 \n\t"\
"pxor %%mm6, %%mm6 \n\t"\ "pxor %%mm6, %%mm6 \n\t"\
"movq %0, %%mm5 \n\t"\
:: "m"(round_tab[1]) \ :: "m"(round_tab[1]) \
);\ );\
\ \
sad8_2_ ## suf(blk1, blk1+1, blk2, stride, 8);\ sad8_x2a_ ## suf(blk1, blk2, stride, 8);\
\ \
return sum_ ## suf();\ return sum_ ## suf();\
}\ }\
...@@ -290,7 +325,7 @@ static int sad8_y2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, in ...@@ -290,7 +325,7 @@ static int sad8_y2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, in
:: "m"(round_tab[1]) \ :: "m"(round_tab[1]) \
);\ );\
\ \
sad8_2_ ## suf(blk1, blk1+stride, blk2, stride, 8);\ sad8_y2a_ ## suf(blk1, blk2, stride, 8);\
\ \
return sum_ ## suf();\ return sum_ ## suf();\
}\ }\
...@@ -300,9 +335,7 @@ static int sad8_xy2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, i ...@@ -300,9 +335,7 @@ static int sad8_xy2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, i
assert(h==8);\ assert(h==8);\
asm volatile("pxor %%mm7, %%mm7 \n\t"\ asm volatile("pxor %%mm7, %%mm7 \n\t"\
"pxor %%mm6, %%mm6 \n\t"\ "pxor %%mm6, %%mm6 \n\t"\
"movq %0, %%mm5 \n\t"\ ::);\
:: "m"(round_tab[2]) \
);\
\ \
sad8_4_ ## suf(blk1, blk2, stride, 8);\ sad8_4_ ## suf(blk1, blk2, stride, 8);\
\ \
...@@ -327,8 +360,8 @@ static int sad16_x2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, i ...@@ -327,8 +360,8 @@ static int sad16_x2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, i
:: "m"(round_tab[1]) \ :: "m"(round_tab[1]) \
);\ );\
\ \
sad8_2_ ## suf(blk1 , blk1+1, blk2 , stride, h);\ sad8_x2a_ ## suf(blk1 , blk2 , stride, h);\
sad8_2_ ## suf(blk1+8, blk1+9, blk2+8, stride, h);\ sad8_x2a_ ## suf(blk1+8, blk2+8, stride, h);\
\ \
return sum_ ## suf();\ return sum_ ## suf();\
}\ }\
...@@ -340,8 +373,8 @@ static int sad16_y2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, i ...@@ -340,8 +373,8 @@ static int sad16_y2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, i
:: "m"(round_tab[1]) \ :: "m"(round_tab[1]) \
);\ );\
\ \
sad8_2_ ## suf(blk1 , blk1+stride, blk2 , stride, h);\ sad8_y2a_ ## suf(blk1 , blk2 , stride, h);\
sad8_2_ ## suf(blk1+8, blk1+stride+8,blk2+8, stride, h);\ sad8_y2a_ ## suf(blk1+8, blk2+8, stride, h);\
\ \
return sum_ ## suf();\ return sum_ ## suf();\
}\ }\
...@@ -349,9 +382,7 @@ static int sad16_xy2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride, ...@@ -349,9 +382,7 @@ static int sad16_xy2_ ## suf(void *v, uint8_t *blk2, uint8_t *blk1, int stride,
{\ {\
asm volatile("pxor %%mm7, %%mm7 \n\t"\ asm volatile("pxor %%mm7, %%mm7 \n\t"\
"pxor %%mm6, %%mm6 \n\t"\ "pxor %%mm6, %%mm6 \n\t"\
"movq %0, %%mm5 \n\t"\ ::);\
:: "m"(round_tab[2]) \
);\
\ \
sad8_4_ ## suf(blk1 , blk2 , stride, h);\ sad8_4_ ## suf(blk1 , blk2 , stride, h);\
sad8_4_ ## suf(blk1+8, blk2+8, stride, h);\ sad8_4_ ## suf(blk1+8, blk2+8, stride, h);\
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment