Skip to content

L
linux-davinci
Commit 7f0536f8 authored by Jeff Dike's avatar Jeff Dike Committed by Linus Torvalds
Browse files
Options

uml: speed up page table walking 

The previous page table walking code was horribly inefficient.  This patch
replaces it with code taken from elsewhere in the kernel.

Forking from bash is now ~5% faster and page faults are handled ~10% faster.
Signed-off-by: default avatarJeff Dike <jdike@linux.intel.com>
Cc: Paolo 'Blaisorblade' Giarrusso <blaisorblade@yahoo.it>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent f30c2c98
Show whitespace changes
Inline Side-by-side
Showing with 120 additions and 82 deletions
arch/um/kernel/tlb.c
View file @ 7f0536f8
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
#include "linux/mm.h" #include "linux/mm.h"
#include "asm/page.h" #include "asm/page.h"
#include "asm/pgalloc.h" #include "asm/pgalloc.h"
#include "asm/pgtable.h"
#include "asm/tlbflush.h" #include "asm/tlbflush.h"
#include "choose-mode.h" #include "choose-mode.h"
#include "mode_kern.h" #include "mode_kern.h"
...@@ -123,106 +124,143 @@ static int add_mprotect(unsigned long addr, unsigned long len, int r, int w, ...@@ -123,106 +124,143 @@ static int add_mprotect(unsigned long addr, unsigned long len, int r, int w,
#define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1)) #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
void fix_range_common(struct mm_struct *mm, unsigned long start_addr, static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
unsigned long end_addr, int force, unsigned long end, struct host_vm_op *ops,
int (*do_ops)(union mm_context *, struct host_vm_op *, int last_op, int *op_index, int force,
int, int, void **)) union mm_context *mmu, void **flush,
int (*do_ops)(union mm_context *,
struct host_vm_op *, int, int,
void **))
{ {
pgd_t *npgd; pte_t *pte;
pud_t *npud; int r, w, x, ret = 0;
pmd_t *npmd;
pte_t *npte;
union mm_context *mmu = &mm->context;
unsigned long addr, end;
int r, w, x;
struct host_vm_op ops[1];
void *flush = NULL;
int op_index = -1, last_op = ARRAY_SIZE(ops) - 1;
int ret = 0;
if(mm == NULL)
return;
ops[0].type = NONE; pte = pte_offset_kernel(pmd, addr);
for(addr = start_addr; addr < end_addr && !ret;){ do {
npgd = pgd_offset(mm, addr); r = pte_read(*pte);
if(!pgd_present(*npgd)){ w = pte_write(*pte);
end = ADD_ROUND(addr, PGDIR_SIZE); x = pte_exec(*pte);
if(end > end_addr) if (!pte_young(*pte)) {
end = end_addr; r = 0;
if(force || pgd_newpage(*npgd)){ w = 0;
ret = add_munmap(addr, end - addr, ops, } else if (!pte_dirty(*pte)) {
&op_index, last_op, mmu, w = 0;
&flush, do_ops);
pgd_mkuptodate(*npgd);
} }
addr = end; if(force || pte_newpage(*pte)){
continue; if(pte_present(*pte))
ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK,
PAGE_SIZE, r, w, x, ops,
op_index, last_op, mmu, flush,
do_ops);
else ret = add_munmap(addr, PAGE_SIZE, ops, op_index,
last_op, mmu, flush, do_ops);
} }
else if(pte_newprot(*pte))
ret = add_mprotect(addr, PAGE_SIZE, r, w, x, ops,
op_index, last_op, mmu, flush,
do_ops);
*pte = pte_mkuptodate(*pte);
} while (pte++, addr += PAGE_SIZE, ((addr != end) && !ret));
return ret;
}
npud = pud_offset(npgd, addr); static inline int update_pmd_range(pud_t *pud, unsigned long addr,
if(!pud_present(*npud)){ unsigned long end, struct host_vm_op *ops,
end = ADD_ROUND(addr, PUD_SIZE); int last_op, int *op_index, int force,
if(end > end_addr) union mm_context *mmu, void **flush,
end = end_addr; int (*do_ops)(union mm_context *,
if(force || pud_newpage(*npud)){ struct host_vm_op *, int, int,
ret = add_munmap(addr, end - addr, ops, void **))
&op_index, last_op, mmu, {
&flush, do_ops); pmd_t *pmd;
pud_mkuptodate(*npud); unsigned long next;
int ret = 0;
pmd = pmd_offset(pud, addr);
do {
next = pmd_addr_end(addr, end);
if(!pmd_present(*pmd)){
if(force || pmd_newpage(*pmd)){
ret = add_munmap(addr, next - addr, ops,
op_index, last_op, mmu,
flush, do_ops);
pmd_mkuptodate(*pmd);
} }
addr = end;
continue;
} }
else ret = update_pte_range(pmd, addr, next, ops, last_op,
op_index, force, mmu, flush,
do_ops);
} while (pmd++, addr = next, ((addr != end) && !ret));
return ret;
}
npmd = pmd_offset(npud, addr); static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
if(!pmd_present(*npmd)){ unsigned long end, struct host_vm_op *ops,
end = ADD_ROUND(addr, PMD_SIZE); int last_op, int *op_index, int force,
if(end > end_addr) union mm_context *mmu, void **flush,
end = end_addr; int (*do_ops)(union mm_context *,
if(force || pmd_newpage(*npmd)){ struct host_vm_op *, int, int,
ret = add_munmap(addr, end - addr, ops, void **))
&op_index, last_op, mmu, {
&flush, do_ops); pud_t *pud;
pmd_mkuptodate(*npmd); unsigned long next;
int ret = 0;
pud = pud_offset(pgd, addr);
do {
next = pud_addr_end(addr, end);
if(!pud_present(*pud)){
if(force || pud_newpage(*pud)){
ret = add_munmap(addr, next - addr, ops,
op_index, last_op, mmu,
flush, do_ops);
pud_mkuptodate(*pud);
} }
addr = end;
continue;
} }
else ret = update_pmd_range(pud, addr, next, ops, last_op,
op_index, force, mmu, flush,
do_ops);
} while (pud++, addr = next, ((addr != end) && !ret));
return ret;
}
npte = pte_offset_kernel(npmd, addr); void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
r = pte_read(*npte); unsigned long end_addr, int force,
w = pte_write(*npte); int (*do_ops)(union mm_context *, struct host_vm_op *,
x = pte_exec(*npte); int, int, void **))
if (!pte_young(*npte)) { {
r = 0; pgd_t *pgd;
w = 0; union mm_context *mmu = &mm->context;
} else if (!pte_dirty(*npte)) { struct host_vm_op ops[1];
w = 0; unsigned long addr = start_addr, next;
} int ret = 0, last_op = ARRAY_SIZE(ops) - 1, op_index = -1;
if(force || pte_newpage(*npte)){ void *flush = NULL;
if(pte_present(*npte)) unsigned long long start_time, end_time;
ret = add_mmap(addr,
pte_val(*npte) & PAGE_MASK, start_time = os_nsecs();
PAGE_SIZE, r, w, x, ops, ops[0].type = NONE;
&op_index, last_op, mmu, pgd = pgd_offset(mm, addr);
&flush, do_ops); do {
else ret = add_munmap(addr, PAGE_SIZE, ops, next = pgd_addr_end(addr, end_addr);
if(!pgd_present(*pgd)){
if (force || pgd_newpage(*pgd)){
ret = add_munmap(addr, next - addr, ops,
&op_index, last_op, mmu, &op_index, last_op, mmu,
&flush, do_ops); &flush, do_ops);
pgd_mkuptodate(*pgd);
} }
else if(pte_newprot(*npte))
ret = add_mprotect(addr, PAGE_SIZE, r, w, x, ops,
&op_index, last_op, mmu,
&flush, do_ops);
*npte = pte_mkuptodate(*npte);
addr += PAGE_SIZE;
} }
else ret = update_pud_range(pgd, addr, next, ops, last_op,
&op_index, force, mmu, &flush,
do_ops);
} while (pgd++, addr = next, ((addr != end_addr) && !ret));
end_time = os_nsecs();
log_info("total flush time - %Ld nsecs\n", end_time - start_time);
if(!ret) if(!ret)
ret = (*do_ops)(mmu, ops, op_index, 1, &flush); ret = (*do_ops)(mmu, ops, op_index, 1, &flush);
/* This is not an else because ret is modified above */ /* This is not an else because ret is modified above */
if(ret) { if(ret) {
printk("fix_range_common: failed, killing current process\n"); printk("fix_range_common: failed, killing current process\n");
force_sig(SIGKILL, current); force_sig(SIGKILL, current);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment