Commit 82622284 authored by David Daney's avatar David Daney Committed by Ralf Baechle

MIPS: Put PGD in C0_CONTEXT for 64-bit R2 processors.

Processors that support the mips64r2 ISA can in four instructions
convert a shifted PGD pointer stored in the upper bits of c0_context
into a usable pointer.  By doing this we save a memory load and
associated potential cache miss in the TLB exception handlers.

Since the upper bits of c0_context were holding the CPU number, we
move this to the upper bits of c0_xcontext which doesn't have enough
bits to hold the PGD pointer, but has plenty for the CPU number.
Signed-off-by: default avatarDavid Daney <ddaney@caviumnetworks.com>
Signed-off-by: default avatarRalf Baechle <ralf@linux-mips.org>
parent 92078e06
...@@ -1427,6 +1427,9 @@ config CPU_SUPPORTS_64BIT_KERNEL ...@@ -1427,6 +1427,9 @@ config CPU_SUPPORTS_64BIT_KERNEL
bool bool
config CPU_SUPPORTS_HUGEPAGES config CPU_SUPPORTS_HUGEPAGES
bool bool
config MIPS_PGD_C0_CONTEXT
bool
default y if 64BIT && CPU_MIPSR2
# #
# Set to y for ptrace access to watch registers. # Set to y for ptrace access to watch registers.
......
...@@ -24,6 +24,33 @@ ...@@ -24,6 +24,33 @@
#endif /* SMTC */ #endif /* SMTC */
#include <asm-generic/mm_hooks.h> #include <asm-generic/mm_hooks.h>
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
#define TLBMISS_HANDLER_SETUP_PGD(pgd) \
tlbmiss_handler_setup_pgd((unsigned long)(pgd))
static inline void tlbmiss_handler_setup_pgd(unsigned long pgd)
{
/* Check for swapper_pg_dir and convert to physical address. */
if ((pgd & CKSEG3) == CKSEG0)
pgd = CPHYSADDR(pgd);
write_c0_context(pgd << 11);
}
#define TLBMISS_HANDLER_SETUP() \
do { \
TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir); \
write_c0_xcontext((unsigned long) smp_processor_id() << 51); \
} while (0)
static inline unsigned long get_current_pgd(void)
{
return PHYS_TO_XKSEG_CACHED((read_c0_context() >> 11) & ~0xfffUL);
}
#else /* CONFIG_MIPS_PGD_C0_CONTEXT: using pgd_current*/
/* /*
* For the fast tlb miss handlers, we keep a per cpu array of pointers * For the fast tlb miss handlers, we keep a per cpu array of pointers
* to the current pgd for each processor. Also, the proc. id is stuffed * to the current pgd for each processor. Also, the proc. id is stuffed
...@@ -46,7 +73,7 @@ extern unsigned long pgd_current[]; ...@@ -46,7 +73,7 @@ extern unsigned long pgd_current[];
back_to_back_c0_hazard(); \ back_to_back_c0_hazard(); \
TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir) TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir)
#endif #endif
#endif /* CONFIG_MIPS_PGD_C0_CONTEXT*/
#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX) #if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)
#define ASID_INC 0x40 #define ASID_INC 0x40
......
...@@ -87,15 +87,19 @@ ...@@ -87,15 +87,19 @@
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
#ifdef CONFIG_MIPS_MT_SMTC #ifdef CONFIG_MIPS_MT_SMTC
#define PTEBASE_SHIFT 19 /* TCBIND */ #define PTEBASE_SHIFT 19 /* TCBIND */
#define CPU_ID_REG CP0_TCBIND
#define CPU_ID_MFC0 mfc0
#elif defined(CONFIG_MIPS_PGD_C0_CONTEXT)
#define PTEBASE_SHIFT 48 /* XCONTEXT */
#define CPU_ID_REG CP0_XCONTEXT
#define CPU_ID_MFC0 MFC0
#else #else
#define PTEBASE_SHIFT 23 /* CONTEXT */ #define PTEBASE_SHIFT 23 /* CONTEXT */
#define CPU_ID_REG CP0_CONTEXT
#define CPU_ID_MFC0 MFC0
#endif #endif
.macro get_saved_sp /* SMP variation */ .macro get_saved_sp /* SMP variation */
#ifdef CONFIG_MIPS_MT_SMTC CPU_ID_MFC0 k0, CPU_ID_REG
mfc0 k0, CP0_TCBIND
#else
MFC0 k0, CP0_CONTEXT
#endif
#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32) #if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
lui k1, %hi(kernelsp) lui k1, %hi(kernelsp)
#else #else
...@@ -111,11 +115,7 @@ ...@@ -111,11 +115,7 @@
.endm .endm
.macro set_saved_sp stackp temp temp2 .macro set_saved_sp stackp temp temp2
#ifdef CONFIG_MIPS_MT_SMTC CPU_ID_MFC0 \temp, CPU_ID_REG
mfc0 \temp, CP0_TCBIND
#else
MFC0 \temp, CP0_CONTEXT
#endif
LONG_SRL \temp, PTEBASE_SHIFT LONG_SRL \temp, PTEBASE_SHIFT
LONG_S \stackp, kernelsp(\temp) LONG_S \stackp, kernelsp(\temp)
.endm .endm
......
...@@ -462,7 +462,9 @@ void __init_refok free_initmem(void) ...@@ -462,7 +462,9 @@ void __init_refok free_initmem(void)
__pa_symbol(&__init_end)); __pa_symbol(&__init_end));
} }
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
unsigned long pgd_current[NR_CPUS]; unsigned long pgd_current[NR_CPUS];
#endif
/* /*
* On 64-bit we've got three-level pagetables with a slightly * On 64-bit we've got three-level pagetables with a slightly
* different layout ... * different layout ...
......
...@@ -160,6 +160,12 @@ static u32 tlb_handler[128] __cpuinitdata; ...@@ -160,6 +160,12 @@ static u32 tlb_handler[128] __cpuinitdata;
static struct uasm_label labels[128] __cpuinitdata; static struct uasm_label labels[128] __cpuinitdata;
static struct uasm_reloc relocs[128] __cpuinitdata; static struct uasm_reloc relocs[128] __cpuinitdata;
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
/*
* CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
* we cannot do r3000 under these circumstances.
*/
/* /*
* The R3000 TLB handler is simple. * The R3000 TLB handler is simple.
*/ */
...@@ -199,6 +205,7 @@ static void __cpuinit build_r3000_tlb_refill_handler(void) ...@@ -199,6 +205,7 @@ static void __cpuinit build_r3000_tlb_refill_handler(void)
dump_handler((u32 *)ebase, 32); dump_handler((u32 *)ebase, 32);
} }
#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
/* /*
* The R4000 TLB handler is much more complicated. We have two * The R4000 TLB handler is much more complicated. We have two
...@@ -497,8 +504,9 @@ static void __cpuinit ...@@ -497,8 +504,9 @@ static void __cpuinit
build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
unsigned int tmp, unsigned int ptr) unsigned int tmp, unsigned int ptr)
{ {
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
long pgdc = (long)pgd_current; long pgdc = (long)pgd_current;
#endif
/* /*
* The vmalloc handling is not in the hotpath. * The vmalloc handling is not in the hotpath.
*/ */
...@@ -506,7 +514,15 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, ...@@ -506,7 +514,15 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
uasm_il_bltz(p, r, tmp, label_vmalloc); uasm_il_bltz(p, r, tmp, label_vmalloc);
/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
#ifdef CONFIG_SMP #ifdef CONFIG_MIPS_PGD_C0_CONTEXT
/*
* &pgd << 11 stored in CONTEXT [23..63].
*/
UASM_i_MFC0(p, ptr, C0_CONTEXT);
uasm_i_dins(p, ptr, 0, 0, 23); /* Clear lower 23 bits of context. */
uasm_i_ori(p, ptr, ptr, 0x540); /* 1 0 1 0 1 << 6 xkphys cached */
uasm_i_drotr(p, ptr, ptr, 11);
#elif defined(CONFIG_SMP)
# ifdef CONFIG_MIPS_MT_SMTC # ifdef CONFIG_MIPS_MT_SMTC
/* /*
* SMTC uses TCBind value as "CPU" index * SMTC uses TCBind value as "CPU" index
...@@ -520,7 +536,7 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, ...@@ -520,7 +536,7 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
*/ */
uasm_i_dmfc0(p, ptr, C0_CONTEXT); uasm_i_dmfc0(p, ptr, C0_CONTEXT);
uasm_i_dsrl(p, ptr, ptr, 23); uasm_i_dsrl(p, ptr, ptr, 23);
#endif # endif
UASM_i_LA_mostly(p, tmp, pgdc); UASM_i_LA_mostly(p, tmp, pgdc);
uasm_i_daddu(p, ptr, ptr, tmp); uasm_i_daddu(p, ptr, ptr, tmp);
uasm_i_dmfc0(p, tmp, C0_BADVADDR); uasm_i_dmfc0(p, tmp, C0_BADVADDR);
...@@ -1033,6 +1049,7 @@ build_pte_modifiable(u32 **p, struct uasm_reloc **r, ...@@ -1033,6 +1049,7 @@ build_pte_modifiable(u32 **p, struct uasm_reloc **r,
iPTE_LW(p, pte, ptr); iPTE_LW(p, pte, ptr);
} }
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
/* /*
* R3000 style TLB load/store/modify handlers. * R3000 style TLB load/store/modify handlers.
*/ */
...@@ -1184,6 +1201,7 @@ static void __cpuinit build_r3000_tlb_modify_handler(void) ...@@ -1184,6 +1201,7 @@ static void __cpuinit build_r3000_tlb_modify_handler(void)
dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
} }
#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
/* /*
* R4000 style TLB load/store/modify handlers. * R4000 style TLB load/store/modify handlers.
...@@ -1400,6 +1418,7 @@ void __cpuinit build_tlb_refill_handler(void) ...@@ -1400,6 +1418,7 @@ void __cpuinit build_tlb_refill_handler(void)
case CPU_TX3912: case CPU_TX3912:
case CPU_TX3922: case CPU_TX3922:
case CPU_TX3927: case CPU_TX3927:
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
build_r3000_tlb_refill_handler(); build_r3000_tlb_refill_handler();
if (!run_once) { if (!run_once) {
build_r3000_tlb_load_handler(); build_r3000_tlb_load_handler();
...@@ -1407,6 +1426,9 @@ void __cpuinit build_tlb_refill_handler(void) ...@@ -1407,6 +1426,9 @@ void __cpuinit build_tlb_refill_handler(void)
build_r3000_tlb_modify_handler(); build_r3000_tlb_modify_handler();
run_once++; run_once++;
} }
#else
panic("No R3000 TLB refill handler");
#endif
break; break;
case CPU_R6000: case CPU_R6000:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment