tlbflush_64.c 10.8 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
/*
2
 * arch/sh/mm/tlb-flush_64.c
Linus Torvalds's avatar
Linus Torvalds committed
3 4 5
 *
 * Copyright (C) 2000, 2001  Paolo Alberelli
 * Copyright (C) 2003  Richard Curnow (/proc/tlb, bug fixes)
6
 * Copyright (C) 2003 - 2009 Paul Mundt
Linus Torvalds's avatar
Linus Torvalds committed
7
 *
8 9 10
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
Linus Torvalds's avatar
Linus Torvalds committed
11 12 13 14 15 16 17 18 19 20 21 22
 */
#include <linux/signal.h>
#include <linux/rwsem.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/smp.h>
23
#include <linux/perf_counter.h>
Linus Torvalds's avatar
Linus Torvalds committed
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
#include <linux/interrupt.h>
#include <asm/system.h>
#include <asm/io.h>
#include <asm/tlb.h>
#include <asm/uaccess.h>
#include <asm/pgalloc.h>
#include <asm/mmu_context.h>

extern void die(const char *,struct pt_regs *,long);

#define PFLAG(val,flag)   (( (val) & (flag) ) ? #flag : "" )
#define PPROT(flag) PFLAG(pgprot_val(prot),flag)

static inline void print_prots(pgprot_t prot)
{
	printk("prot is 0x%08lx\n",pgprot_val(prot));

	printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ),
	       PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER));
}

static inline void print_vma(struct vm_area_struct *vma)
{
	printk("vma start 0x%08lx\n", vma->vm_start);
	printk("vma end   0x%08lx\n", vma->vm_end);

	print_prots(vma->vm_page_prot);
	printk("vm_flags 0x%08lx\n", vma->vm_flags);
}

static inline void print_task(struct task_struct *tsk)
{
56
	printk("Task pid %d\n", task_pid_nr(tsk));
Linus Torvalds's avatar
Linus Torvalds committed
57 58 59 60 61
}

static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address)
{
	pgd_t *dir;
62
	pud_t *pud;
Linus Torvalds's avatar
Linus Torvalds committed
63 64 65 66 67
	pmd_t *pmd;
	pte_t *pte;
	pte_t entry;

	dir = pgd_offset(mm, address);
68
	if (pgd_none(*dir))
Linus Torvalds's avatar
Linus Torvalds committed
69 70
		return NULL;

71 72 73 74 75 76
	pud = pud_offset(dir, address);
	if (pud_none(*pud))
		return NULL;

	pmd = pmd_offset(pud, address);
	if (pmd_none(*pmd))
Linus Torvalds's avatar
Linus Torvalds committed
77 78 79 80
		return NULL;

	pte = pte_offset_kernel(pmd, address);
	entry = *pte;
81
	if (pte_none(entry) || !pte_present(entry))
Linus Torvalds's avatar
Linus Torvalds committed
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
		return NULL;

	return pte;
}

/*
 * This routine handles page faults.  It determines the address,
 * and the problem, and then passes it off to one of the appropriate
 * routines.
 */
asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
			      unsigned long textaccess, unsigned long address)
{
	struct task_struct *tsk;
	struct mm_struct *mm;
	struct vm_area_struct * vma;
	const struct exception_table_entry *fixup;
	pte_t *pte;
Nick Piggin's avatar
Nick Piggin committed
100
	int fault;
Linus Torvalds's avatar
Linus Torvalds committed
101 102 103 104

	/* SIM
	 * Note this is now called with interrupts still disabled
	 * This is to cope with being called for a missing IO port
Simon Arlott's avatar
Simon Arlott committed
105
	 * address with interrupts disabled. This should be fixed as
Linus Torvalds's avatar
Linus Torvalds committed
106 107 108 109 110 111 112 113 114 115 116 117 118
	 * soon as we have a better 'fast path' miss handler.
	 *
	 * Plus take care how you try and debug this stuff.
	 * For example, writing debug data to a port which you
	 * have just faulted on is not going to work.
	 */

	tsk = current;
	mm = tsk->mm;

	/* Not an IO address, so reenable interrupts */
	local_irq_enable();

119 120
	perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address);

Linus Torvalds's avatar
Linus Torvalds committed
121 122 123 124
	/*
	 * If we're in an interrupt or have no user
	 * context, we must not take the fault..
	 */
125
	if (in_atomic() || !mm)
Linus Torvalds's avatar
Linus Torvalds committed
126 127 128 129 130 131 132 133 134 135 136
		goto no_context;

	/* TLB misses upon some cache flushes get done under cli() */
	down_read(&mm->mmap_sem);

	vma = find_vma(mm, address);

	if (!vma) {
#ifdef DEBUG_FAULT
		print_task(tsk);
		printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
137
		       __func__, __LINE__,
Linus Torvalds's avatar
Linus Torvalds committed
138 139 140 141 142 143 144 145 146 147 148 149 150
		       address,regs->pc,textaccess,writeaccess);
		show_regs(regs);
#endif
		goto bad_area;
	}
	if (vma->vm_start <= address) {
		goto good_area;
	}

	if (!(vma->vm_flags & VM_GROWSDOWN)) {
#ifdef DEBUG_FAULT
		print_task(tsk);
		printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
151
		       __func__, __LINE__,
Linus Torvalds's avatar
Linus Torvalds committed
152 153 154 155 156 157 158 159 160 161 162
		       address,regs->pc,textaccess,writeaccess);
		show_regs(regs);

		print_vma(vma);
#endif
		goto bad_area;
	}
	if (expand_stack(vma, address)) {
#ifdef DEBUG_FAULT
		print_task(tsk);
		printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
163
		       __func__, __LINE__,
Linus Torvalds's avatar
Linus Torvalds committed
164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192
		       address,regs->pc,textaccess,writeaccess);
		show_regs(regs);
#endif
		goto bad_area;
	}
/*
 * Ok, we have a good vm_area for this memory access, so
 * we can handle it..
 */
good_area:
	if (textaccess) {
		if (!(vma->vm_flags & VM_EXEC))
			goto bad_area;
	} else {
		if (writeaccess) {
			if (!(vma->vm_flags & VM_WRITE))
				goto bad_area;
		} else {
			if (!(vma->vm_flags & VM_READ))
				goto bad_area;
		}
	}

	/*
	 * If for any reason at all we couldn't handle the fault,
	 * make sure we exit gracefully rather than endlessly redo
	 * the fault.
	 */
survive:
193
	fault = handle_mm_fault(mm, vma, address, writeaccess ? FAULT_FLAG_WRITE : 0);
Nick Piggin's avatar
Nick Piggin committed
194 195 196 197 198 199
	if (unlikely(fault & VM_FAULT_ERROR)) {
		if (fault & VM_FAULT_OOM)
			goto out_of_memory;
		else if (fault & VM_FAULT_SIGBUS)
			goto do_sigbus;
		BUG();
Linus Torvalds's avatar
Linus Torvalds committed
200
	}
201 202

	if (fault & VM_FAULT_MAJOR) {
Nick Piggin's avatar
Nick Piggin committed
203
		tsk->maj_flt++;
204 205 206
		perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0,
				     regs, address);
	} else {
Nick Piggin's avatar
Nick Piggin committed
207
		tsk->min_flt++;
208 209 210
		perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0,
				     regs, address);
	}
Nick Piggin's avatar
Nick Piggin committed
211

Linus Torvalds's avatar
Linus Torvalds committed
212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
	/* If we get here, the page fault has been handled.  Do the TLB refill
	   now from the newly-setup PTE, to avoid having to fault again right
	   away on the same instruction. */
	pte = lookup_pte (mm, address);
	if (!pte) {
		/* From empirical evidence, we can get here, due to
		   !pte_present(pte).  (e.g. if a swap-in occurs, and the page
		   is swapped back out again before the process that wanted it
		   gets rescheduled?) */
		goto no_pte;
	}

	__do_tlb_refill(address, textaccess, pte);

no_pte:

	up_read(&mm->mmap_sem);
	return;

/*
 * Something tried to access memory that isn't in our memory map..
 * Fix it, but check if it's kernel or user first..
 */
bad_area:
#ifdef DEBUG_FAULT
	printk("fault:bad area\n");
#endif
	up_read(&mm->mmap_sem);

	if (user_mode(regs)) {
		static int count=0;
		siginfo_t info;
		if (count < 4) {
			/* This is really to help debug faults when starting
			 * usermode, so only need a few */
			count++;
			printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n",
249
				address, task_pid_nr(current), current->comm,
Linus Torvalds's avatar
Linus Torvalds committed
250 251 252 253 254
				(unsigned long) regs->pc);
#if 0
			show_regs(regs);
#endif
		}
255
		if (is_global_init(tsk)) {
Linus Torvalds's avatar
Linus Torvalds committed
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
			panic("INIT had user mode bad_area\n");
		}
		tsk->thread.address = address;
		tsk->thread.error_code = writeaccess;
		info.si_signo = SIGSEGV;
		info.si_errno = 0;
		info.si_addr = (void *) address;
		force_sig_info(SIGSEGV, &info, tsk);
		return;
	}

no_context:
#ifdef DEBUG_FAULT
	printk("fault:No context\n");
#endif
	/* Are we prepared to handle this kernel fault?  */
	fixup = search_exception_tables(regs->pc);
	if (fixup) {
		regs->pc = fixup->fixup;
		return;
	}

/*
 * Oops. The kernel tried to access some bad page. We'll have to
 * terminate things with extreme prejudice.
 *
 */
	if (address < PAGE_SIZE)
		printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
	else
		printk(KERN_ALERT "Unable to handle kernel paging request");
	printk(" at virtual address %08lx\n", address);
	printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff);
	die("Oops", regs, writeaccess);
	do_exit(SIGKILL);

/*
 * We ran out of memory, or some other thing happened to us that made
 * us unable to handle the page fault gracefully.
 */
out_of_memory:
297
	if (is_global_init(current)) {
Linus Torvalds's avatar
Linus Torvalds committed
298 299 300 301 302 303
		panic("INIT out of memory\n");
		yield();
		goto survive;
	}
	printk("fault:Out of memory\n");
	up_read(&mm->mmap_sem);
304
	if (is_global_init(current)) {
Linus Torvalds's avatar
Linus Torvalds committed
305 306 307 308 309 310
		yield();
		down_read(&mm->mmap_sem);
		goto survive;
	}
	printk("VM: killing process %s\n", tsk->comm);
	if (user_mode(regs))
311
		do_group_exit(SIGKILL);
Linus Torvalds's avatar
Linus Torvalds committed
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
	goto no_context;

do_sigbus:
	printk("fault:Do sigbus\n");
	up_read(&mm->mmap_sem);

	/*
	 * Send a sigbus, regardless of whether we were in kernel
	 * or user mode.
	 */
	tsk->thread.address = address;
	tsk->thread.error_code = writeaccess;
	tsk->thread.trap_no = 14;
	force_sig(SIGBUS, tsk);

	/* Kernel mode? Handle exceptions or die */
	if (!user_mode(regs))
		goto no_context;
}

332
void local_flush_tlb_one(unsigned long asid, unsigned long page)
Linus Torvalds's avatar
Linus Torvalds committed
333 334 335 336 337 338 339
{
	unsigned long long match, pteh=0, lpage;
	unsigned long tlb;

	/*
	 * Sign-extend based on neff.
	 */
340
	lpage = neff_sign_extend(page);
341
	match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID;
Linus Torvalds's avatar
Linus Torvalds committed
342 343
	match |= lpage;

344 345 346 347
	for_each_itlb_entry(tlb) {
		asm volatile ("getcfg	%1, 0, %0"
			      : "=r" (pteh)
			      : "r" (tlb) );
Linus Torvalds's avatar
Linus Torvalds committed
348

349 350 351
		if (pteh == match) {
			__flush_tlb_slot(tlb);
			break;
Linus Torvalds's avatar
Linus Torvalds committed
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
		}
	}

	for_each_dtlb_entry(tlb) {
		asm volatile ("getcfg	%1, 0, %0"
			      : "=r" (pteh)
			      : "r" (tlb) );

		if (pteh == match) {
			__flush_tlb_slot(tlb);
			break;
		}

	}
}

368
void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
Linus Torvalds's avatar
Linus Torvalds committed
369 370 371 372 373 374
{
	unsigned long flags;

	if (vma->vm_mm) {
		page &= PAGE_MASK;
		local_irq_save(flags);
375
		local_flush_tlb_one(get_asid(), page);
Linus Torvalds's avatar
Linus Torvalds committed
376 377 378 379
		local_irq_restore(flags);
	}
}

380 381
void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
			   unsigned long end)
Linus Torvalds's avatar
Linus Torvalds committed
382 383 384 385
{
	unsigned long flags;
	unsigned long long match, pteh=0, pteh_epn, pteh_low;
	unsigned long tlb;
386
	unsigned int cpu = smp_processor_id();
Linus Torvalds's avatar
Linus Torvalds committed
387 388 389
	struct mm_struct *mm;

	mm = vma->vm_mm;
390
	if (cpu_context(cpu, mm) == NO_CONTEXT)
Linus Torvalds's avatar
Linus Torvalds committed
391 392 393 394 395 396 397
		return;

	local_irq_save(flags);

	start &= PAGE_MASK;
	end &= PAGE_MASK;

398
	match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID;
Linus Torvalds's avatar
Linus Torvalds committed
399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428

	/* Flush ITLB */
	for_each_itlb_entry(tlb) {
		asm volatile ("getcfg	%1, 0, %0"
			      : "=r" (pteh)
			      : "r" (tlb) );

		pteh_epn = pteh & PAGE_MASK;
		pteh_low = pteh & ~PAGE_MASK;

		if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
			__flush_tlb_slot(tlb);
	}

	/* Flush DTLB */
	for_each_dtlb_entry(tlb) {
		asm volatile ("getcfg	%1, 0, %0"
			      : "=r" (pteh)
			      : "r" (tlb) );

		pteh_epn = pteh & PAGE_MASK;
		pteh_low = pteh & ~PAGE_MASK;

		if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
			__flush_tlb_slot(tlb);
	}

	local_irq_restore(flags);
}

429
void local_flush_tlb_mm(struct mm_struct *mm)
Linus Torvalds's avatar
Linus Torvalds committed
430 431
{
	unsigned long flags;
432
	unsigned int cpu = smp_processor_id();
Linus Torvalds's avatar
Linus Torvalds committed
433

434
	if (cpu_context(cpu, mm) == NO_CONTEXT)
Linus Torvalds's avatar
Linus Torvalds committed
435 436 437 438
		return;

	local_irq_save(flags);

439 440 441
	cpu_context(cpu, mm) = NO_CONTEXT;
	if (mm == current->mm)
		activate_context(mm, cpu);
Linus Torvalds's avatar
Linus Torvalds committed
442 443 444 445

	local_irq_restore(flags);
}

446
void local_flush_tlb_all(void)
Linus Torvalds's avatar
Linus Torvalds committed
447 448 449 450 451 452 453
{
	/* Invalidate all, including shared pages, excluding fixed TLBs */
	unsigned long flags, tlb;

	local_irq_save(flags);

	/* Flush each ITLB entry */
454
	for_each_itlb_entry(tlb)
Linus Torvalds's avatar
Linus Torvalds committed
455 456 457
		__flush_tlb_slot(tlb);

	/* Flush each DTLB entry */
458
	for_each_dtlb_entry(tlb)
Linus Torvalds's avatar
Linus Torvalds committed
459 460 461 462 463
		__flush_tlb_slot(tlb);

	local_irq_restore(flags);
}

464
void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
Linus Torvalds's avatar
Linus Torvalds committed
465 466 467 468
{
        /* FIXME: Optimize this later.. */
        flush_tlb_all();
}
469 470 471 472

void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
{
}