c-r4k.c 34.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9
/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
 */
10
#include <linux/hardirq.h>
Linus Torvalds's avatar
Linus Torvalds committed
11
#include <linux/init.h>
12
#include <linux/highmem.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
#include <linux/kernel.h>
14
#include <linux/linkage.h>
Linus Torvalds's avatar
Linus Torvalds committed
15 16
#include <linux/sched.h>
#include <linux/mm.h>
17
#include <linux/module.h>
Linus Torvalds's avatar
Linus Torvalds committed
18 19 20 21
#include <linux/bitops.h>

#include <asm/bcache.h>
#include <asm/bootinfo.h>
22
#include <asm/cache.h>
Linus Torvalds's avatar
Linus Torvalds committed
23 24 25 26 27 28 29
#include <asm/cacheops.h>
#include <asm/cpu.h>
#include <asm/cpu-features.h>
#include <asm/io.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/r4kcache.h>
30
#include <asm/sections.h>
Linus Torvalds's avatar
Linus Torvalds committed
31 32 33
#include <asm/system.h>
#include <asm/mmu_context.h>
#include <asm/war.h>
34
#include <asm/cacheflush.h> /* for run_uncached() */
Linus Torvalds's avatar
Linus Torvalds committed
35

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56

/*
 * Special Variant of smp_call_function for use by cache functions:
 *
 *  o No return value
 *  o collapses to normal function call on UP kernels
 *  o collapses to normal function call on systems with a single shared
 *    primary cache.
 */
static inline void r4k_on_each_cpu(void (*func) (void *info), void *info,
                                   int retry, int wait)
{
	preempt_disable();

#if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC)
	smp_call_function(func, info, retry, wait);
#endif
	func(info);
	preempt_enable();
}

57 58 59 60 61 62
#if defined(CONFIG_MIPS_CMP)
#define cpu_has_safe_index_cacheops 0
#else
#define cpu_has_safe_index_cacheops 1
#endif

63 64 65 66 67 68
/*
 * Must die.
 */
static unsigned long icache_size __read_mostly;
static unsigned long dcache_size __read_mostly;
static unsigned long scache_size __read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
69 70 71 72

/*
 * Dummy cache handling routines for machines without boardcaches
 */
73
static void cache_noop(void) {}
Linus Torvalds's avatar
Linus Torvalds committed
74 75

static struct bcache_ops no_sc_ops = {
76 77 78 79
	.bc_enable = (void *)cache_noop,
	.bc_disable = (void *)cache_noop,
	.bc_wback_inv = (void *)cache_noop,
	.bc_inv = (void *)cache_noop
Linus Torvalds's avatar
Linus Torvalds committed
80 81 82 83
};

struct bcache_ops *bcops = &no_sc_ops;

84 85
#define cpu_is_r4600_v1_x()	((read_c0_prid() & 0xfffffff0) == 0x00002010)
#define cpu_is_r4600_v2_x()	((read_c0_prid() & 0xfffffff0) == 0x00002020)
Linus Torvalds's avatar
Linus Torvalds committed
86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102

#define R4600_HIT_CACHEOP_WAR_IMPL					\
do {									\
	if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())		\
		*(volatile unsigned long *)CKSEG1;			\
	if (R4600_V1_HIT_CACHEOP_WAR)					\
		__asm__ __volatile__("nop;nop;nop;nop");		\
} while (0)

static void (*r4k_blast_dcache_page)(unsigned long addr);

static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
{
	R4600_HIT_CACHEOP_WAR_IMPL;
	blast_dcache32_page(addr);
}

103
static void __cpuinit r4k_blast_dcache_page_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
104 105 106
{
	unsigned long  dc_lsize = cpu_dcache_line_size();

107 108 109
	if (dc_lsize == 0)
		r4k_blast_dcache_page = (void *)cache_noop;
	else if (dc_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
110 111 112 113 114 115 116
		r4k_blast_dcache_page = blast_dcache16_page;
	else if (dc_lsize == 32)
		r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
}

static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);

117
static void __cpuinit r4k_blast_dcache_page_indexed_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
118 119 120
{
	unsigned long dc_lsize = cpu_dcache_line_size();

121 122 123
	if (dc_lsize == 0)
		r4k_blast_dcache_page_indexed = (void *)cache_noop;
	else if (dc_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
124 125 126 127 128 129 130
		r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
	else if (dc_lsize == 32)
		r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
}

static void (* r4k_blast_dcache)(void);

131
static void __cpuinit r4k_blast_dcache_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
132 133 134
{
	unsigned long dc_lsize = cpu_dcache_line_size();

135 136 137
	if (dc_lsize == 0)
		r4k_blast_dcache = (void *)cache_noop;
	else if (dc_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
		r4k_blast_dcache = blast_dcache16;
	else if (dc_lsize == 32)
		r4k_blast_dcache = blast_dcache32;
}

/* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
#define JUMP_TO_ALIGN(order) \
	__asm__ __volatile__( \
		"b\t1f\n\t" \
		".align\t" #order "\n\t" \
		"1:\n\t" \
		)
#define CACHE32_UNROLL32_ALIGN	JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
#define CACHE32_UNROLL32_ALIGN2	JUMP_TO_ALIGN(11)

static inline void blast_r4600_v1_icache32(void)
{
	unsigned long flags;

	local_irq_save(flags);
	blast_icache32();
	local_irq_restore(flags);
}

static inline void tx49_blast_icache32(void)
{
	unsigned long start = INDEX_BASE;
	unsigned long end = start + current_cpu_data.icache.waysize;
	unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
	unsigned long ws_end = current_cpu_data.icache.ways <<
	                       current_cpu_data.icache.waybit;
	unsigned long ws, addr;

	CACHE32_UNROLL32_ALIGN2;
	/* I'm in even chunk.  blast odd chunks */
173 174
	for (ws = 0; ws < ws_end; ws += ws_inc)
		for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
175
			cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds's avatar
Linus Torvalds committed
176 177
	CACHE32_UNROLL32_ALIGN;
	/* I'm in odd chunk.  blast even chunks */
178 179
	for (ws = 0; ws < ws_end; ws += ws_inc)
		for (addr = start; addr < end; addr += 0x400 * 2)
180
			cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds's avatar
Linus Torvalds committed
181 182 183 184 185 186 187 188 189 190 191 192 193
}

static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
{
	unsigned long flags;

	local_irq_save(flags);
	blast_icache32_page_indexed(page);
	local_irq_restore(flags);
}

static inline void tx49_blast_icache32_page_indexed(unsigned long page)
{
194 195
	unsigned long indexmask = current_cpu_data.icache.waysize - 1;
	unsigned long start = INDEX_BASE + (page & indexmask);
Linus Torvalds's avatar
Linus Torvalds committed
196 197 198 199 200 201 202 203
	unsigned long end = start + PAGE_SIZE;
	unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
	unsigned long ws_end = current_cpu_data.icache.ways <<
	                       current_cpu_data.icache.waybit;
	unsigned long ws, addr;

	CACHE32_UNROLL32_ALIGN2;
	/* I'm in even chunk.  blast odd chunks */
204 205
	for (ws = 0; ws < ws_end; ws += ws_inc)
		for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
206
			cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds's avatar
Linus Torvalds committed
207 208
	CACHE32_UNROLL32_ALIGN;
	/* I'm in odd chunk.  blast even chunks */
209 210
	for (ws = 0; ws < ws_end; ws += ws_inc)
		for (addr = start; addr < end; addr += 0x400 * 2)
211
			cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds's avatar
Linus Torvalds committed
212 213 214 215
}

static void (* r4k_blast_icache_page)(unsigned long addr);

216
static void __cpuinit r4k_blast_icache_page_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
217 218 219
{
	unsigned long ic_lsize = cpu_icache_line_size();

220 221 222
	if (ic_lsize == 0)
		r4k_blast_icache_page = (void *)cache_noop;
	else if (ic_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
223 224 225 226 227 228 229 230 231 232
		r4k_blast_icache_page = blast_icache16_page;
	else if (ic_lsize == 32)
		r4k_blast_icache_page = blast_icache32_page;
	else if (ic_lsize == 64)
		r4k_blast_icache_page = blast_icache64_page;
}


static void (* r4k_blast_icache_page_indexed)(unsigned long addr);

233
static void __cpuinit r4k_blast_icache_page_indexed_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
234 235 236
{
	unsigned long ic_lsize = cpu_icache_line_size();

237 238 239
	if (ic_lsize == 0)
		r4k_blast_icache_page_indexed = (void *)cache_noop;
	else if (ic_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
240 241
		r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
	else if (ic_lsize == 32) {
Thiemo Seufer's avatar
Thiemo Seufer committed
242
		if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
Linus Torvalds's avatar
Linus Torvalds committed
243 244
			r4k_blast_icache_page_indexed =
				blast_icache32_r4600_v1_page_indexed;
Thiemo Seufer's avatar
Thiemo Seufer committed
245 246 247
		else if (TX49XX_ICACHE_INDEX_INV_WAR)
			r4k_blast_icache_page_indexed =
				tx49_blast_icache32_page_indexed;
Linus Torvalds's avatar
Linus Torvalds committed
248 249 250 251 252 253 254 255 256
		else
			r4k_blast_icache_page_indexed =
				blast_icache32_page_indexed;
	} else if (ic_lsize == 64)
		r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
}

static void (* r4k_blast_icache)(void);

257
static void __cpuinit r4k_blast_icache_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
258 259 260
{
	unsigned long ic_lsize = cpu_icache_line_size();

261 262 263
	if (ic_lsize == 0)
		r4k_blast_icache = (void *)cache_noop;
	else if (ic_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
264 265 266 267 268 269 270 271 272 273 274 275 276 277
		r4k_blast_icache = blast_icache16;
	else if (ic_lsize == 32) {
		if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
			r4k_blast_icache = blast_r4600_v1_icache32;
		else if (TX49XX_ICACHE_INDEX_INV_WAR)
			r4k_blast_icache = tx49_blast_icache32;
		else
			r4k_blast_icache = blast_icache32;
	} else if (ic_lsize == 64)
		r4k_blast_icache = blast_icache64;
}

static void (* r4k_blast_scache_page)(unsigned long addr);

278
static void __cpuinit r4k_blast_scache_page_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
279 280 281
{
	unsigned long sc_lsize = cpu_scache_line_size();

282
	if (scache_size == 0)
283
		r4k_blast_scache_page = (void *)cache_noop;
284
	else if (sc_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
285 286 287 288 289 290 291 292 293 294 295
		r4k_blast_scache_page = blast_scache16_page;
	else if (sc_lsize == 32)
		r4k_blast_scache_page = blast_scache32_page;
	else if (sc_lsize == 64)
		r4k_blast_scache_page = blast_scache64_page;
	else if (sc_lsize == 128)
		r4k_blast_scache_page = blast_scache128_page;
}

static void (* r4k_blast_scache_page_indexed)(unsigned long addr);

296
static void __cpuinit r4k_blast_scache_page_indexed_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
297 298 299
{
	unsigned long sc_lsize = cpu_scache_line_size();

300
	if (scache_size == 0)
301
		r4k_blast_scache_page_indexed = (void *)cache_noop;
302
	else if (sc_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
303 304 305 306 307 308 309 310 311 312 313
		r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
	else if (sc_lsize == 32)
		r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
	else if (sc_lsize == 64)
		r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
	else if (sc_lsize == 128)
		r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
}

static void (* r4k_blast_scache)(void);

314
static void __cpuinit r4k_blast_scache_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
315 316 317
{
	unsigned long sc_lsize = cpu_scache_line_size();

318
	if (scache_size == 0)
319
		r4k_blast_scache = (void *)cache_noop;
320
	else if (sc_lsize == 16)
Linus Torvalds's avatar
Linus Torvalds committed
321 322 323 324 325 326 327 328 329 330 331
		r4k_blast_scache = blast_scache16;
	else if (sc_lsize == 32)
		r4k_blast_scache = blast_scache32;
	else if (sc_lsize == 64)
		r4k_blast_scache = blast_scache64;
	else if (sc_lsize == 128)
		r4k_blast_scache = blast_scache128;
}

static inline void local_r4k___flush_cache_all(void * args)
{
332 333 334 335
#if defined(CONFIG_CPU_LOONGSON2)
	r4k_blast_scache();
	return;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
336 337 338
	r4k_blast_dcache();
	r4k_blast_icache();

339
	switch (current_cpu_type()) {
Linus Torvalds's avatar
Linus Torvalds committed
340 341 342 343 344 345
	case CPU_R4000SC:
	case CPU_R4000MC:
	case CPU_R4400SC:
	case CPU_R4400MC:
	case CPU_R10000:
	case CPU_R12000:
Kumba's avatar
Kumba committed
346
	case CPU_R14000:
Linus Torvalds's avatar
Linus Torvalds committed
347 348 349 350 351 352
		r4k_blast_scache();
	}
}

static void r4k___flush_cache_all(void)
{
353
	r4k_on_each_cpu(local_r4k___flush_cache_all, NULL, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
354 355
}

356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
static inline int has_valid_asid(const struct mm_struct *mm)
{
#if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_MIPS_MT_SMTC)
	int i;

	for_each_online_cpu(i)
		if (cpu_context(i, mm))
			return 1;

	return 0;
#else
	return cpu_context(smp_processor_id(), mm);
#endif
}

371 372 373 374 375 376 377 378 379 380
static void r4k__flush_cache_vmap(void)
{
	r4k_blast_dcache();
}

static void r4k__flush_cache_vunmap(void)
{
	r4k_blast_dcache();
}

Linus Torvalds's avatar
Linus Torvalds committed
381 382 383
static inline void local_r4k_flush_cache_range(void * args)
{
	struct vm_area_struct *vma = args;
384
	int exec = vma->vm_flags & VM_EXEC;
Linus Torvalds's avatar
Linus Torvalds committed
385

386
	if (!(has_valid_asid(vma->vm_mm)))
Linus Torvalds's avatar
Linus Torvalds committed
387 388
		return;

389
	r4k_blast_dcache();
390 391
	if (exec)
		r4k_blast_icache();
Linus Torvalds's avatar
Linus Torvalds committed
392 393 394 395 396
}

static void r4k_flush_cache_range(struct vm_area_struct *vma,
	unsigned long start, unsigned long end)
{
397
	int exec = vma->vm_flags & VM_EXEC;
398

399 400
	if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
		r4k_on_each_cpu(local_r4k_flush_cache_range, vma, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
401 402 403 404 405 406
}

static inline void local_r4k_flush_cache_mm(void * args)
{
	struct mm_struct *mm = args;

407
	if (!has_valid_asid(mm))
Linus Torvalds's avatar
Linus Torvalds committed
408 409 410 411 412
		return;

	/*
	 * Kludge alert.  For obscure reasons R4000SC and R4400SC go nuts if we
	 * only flush the primary caches but R10000 and R12000 behave sane ...
413 414
	 * R4000SC and R4400SC indexed S-cache ops also invalidate primary
	 * caches, so we can bail out early.
Linus Torvalds's avatar
Linus Torvalds committed
415
	 */
416 417 418 419
	if (current_cpu_type() == CPU_R4000SC ||
	    current_cpu_type() == CPU_R4000MC ||
	    current_cpu_type() == CPU_R4400SC ||
	    current_cpu_type() == CPU_R4400MC) {
Linus Torvalds's avatar
Linus Torvalds committed
420
		r4k_blast_scache();
421 422 423 424
		return;
	}

	r4k_blast_dcache();
Linus Torvalds's avatar
Linus Torvalds committed
425 426 427 428 429 430 431
}

static void r4k_flush_cache_mm(struct mm_struct *mm)
{
	if (!cpu_has_dc_aliases)
		return;

432
	r4k_on_each_cpu(local_r4k_flush_cache_mm, mm, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
433 434 435 436
}

struct flush_cache_page_args {
	struct vm_area_struct *vma;
437
	unsigned long addr;
438
	unsigned long pfn;
Linus Torvalds's avatar
Linus Torvalds committed
439 440 441 442 443 444
};

static inline void local_r4k_flush_cache_page(void *args)
{
	struct flush_cache_page_args *fcp_args = args;
	struct vm_area_struct *vma = fcp_args->vma;
445
	unsigned long addr = fcp_args->addr;
446
	struct page *page = pfn_to_page(fcp_args->pfn);
Linus Torvalds's avatar
Linus Torvalds committed
447 448 449
	int exec = vma->vm_flags & VM_EXEC;
	struct mm_struct *mm = vma->vm_mm;
	pgd_t *pgdp;
450
	pud_t *pudp;
Linus Torvalds's avatar
Linus Torvalds committed
451 452
	pmd_t *pmdp;
	pte_t *ptep;
453
	void *vaddr;
Linus Torvalds's avatar
Linus Torvalds committed
454

455 456 457 458
	/*
	 * If ownes no valid ASID yet, cannot possibly have gotten
	 * this page into the cache.
	 */
459
	if (!has_valid_asid(mm))
460 461
		return;

462 463 464 465 466
	addr &= PAGE_MASK;
	pgdp = pgd_offset(mm, addr);
	pudp = pud_offset(pgdp, addr);
	pmdp = pmd_offset(pudp, addr);
	ptep = pte_offset(pmdp, addr);
Linus Torvalds's avatar
Linus Torvalds committed
467 468 469 470 471

	/*
	 * If the page isn't marked valid, the page cannot possibly be
	 * in the cache.
	 */
472
	if (!(pte_present(*ptep)))
Linus Torvalds's avatar
Linus Torvalds committed
473 474
		return;

475 476 477 478 479 480 481 482 483 484 485 486
	if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
		vaddr = NULL;
	else {
		/*
		 * Use kmap_coherent or kmap_atomic to do flushes for
		 * another ASID than the current one.
		 */
		if (cpu_has_dc_aliases)
			vaddr = kmap_coherent(page, addr);
		else
			vaddr = kmap_atomic(page, KM_USER0);
		addr = (unsigned long)vaddr;
Linus Torvalds's avatar
Linus Torvalds committed
487 488 489
	}

	if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
490
		r4k_blast_dcache_page(addr);
491 492
		if (exec && !cpu_icache_snoops_remote_store)
			r4k_blast_scache_page(addr);
Linus Torvalds's avatar
Linus Torvalds committed
493 494
	}
	if (exec) {
495
		if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) {
Linus Torvalds's avatar
Linus Torvalds committed
496 497
			int cpu = smp_processor_id();

Thiemo Seufer's avatar
Thiemo Seufer committed
498 499
			if (cpu_context(cpu, mm) != 0)
				drop_mmu_context(mm, cpu);
Linus Torvalds's avatar
Linus Torvalds committed
500
		} else
501 502 503 504 505 506 507 508
			r4k_blast_icache_page(addr);
	}

	if (vaddr) {
		if (cpu_has_dc_aliases)
			kunmap_coherent();
		else
			kunmap_atomic(vaddr, KM_USER0);
Linus Torvalds's avatar
Linus Torvalds committed
509 510 511
	}
}

512 513
static void r4k_flush_cache_page(struct vm_area_struct *vma,
	unsigned long addr, unsigned long pfn)
Linus Torvalds's avatar
Linus Torvalds committed
514 515 516 517
{
	struct flush_cache_page_args args;

	args.vma = vma;
518
	args.addr = addr;
519
	args.pfn = pfn;
Linus Torvalds's avatar
Linus Torvalds committed
520

521
	r4k_on_each_cpu(local_r4k_flush_cache_page, &args, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
522 523 524 525 526 527 528 529 530
}

static inline void local_r4k_flush_data_cache_page(void * addr)
{
	r4k_blast_dcache_page((unsigned long) addr);
}

static void r4k_flush_data_cache_page(unsigned long addr)
{
531 532 533 534 535
	if (in_atomic())
		local_r4k_flush_data_cache_page((void *)addr);
	else
		r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr,
			        1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
536 537 538
}

struct flush_icache_range_args {
539 540
	unsigned long start;
	unsigned long end;
Linus Torvalds's avatar
Linus Torvalds committed
541 542 543 544 545 546 547 548 549
};

static inline void local_r4k_flush_icache_range(void *args)
{
	struct flush_icache_range_args *fir_args = args;
	unsigned long start = fir_args->start;
	unsigned long end = fir_args->end;

	if (!cpu_has_ic_fills_f_dc) {
550
		if (end - start >= dcache_size) {
Linus Torvalds's avatar
Linus Torvalds committed
551 552
			r4k_blast_dcache();
		} else {
553
			R4600_HIT_CACHEOP_WAR_IMPL;
554
			protected_blast_dcache_range(start, end);
Linus Torvalds's avatar
Linus Torvalds committed
555 556 557 558 559
		}
	}

	if (end - start > icache_size)
		r4k_blast_icache();
560 561
	else
		protected_blast_icache_range(start, end);
Linus Torvalds's avatar
Linus Torvalds committed
562 563
}

564
static void r4k_flush_icache_range(unsigned long start, unsigned long end)
Linus Torvalds's avatar
Linus Torvalds committed
565 566 567 568 569 570
{
	struct flush_icache_range_args args;

	args.start = start;
	args.end = end;

571
	r4k_on_each_cpu(local_r4k_flush_icache_range, &args, 1, 1);
572
	instruction_hazard();
Linus Torvalds's avatar
Linus Torvalds committed
573 574 575 576 577 578 579 580 581
}

#ifdef CONFIG_DMA_NONCOHERENT

static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
{
	/* Catch bad driver code */
	BUG_ON(size == 0);

582
	if (cpu_has_inclusive_pcaches) {
583
		if (size >= scache_size)
Linus Torvalds's avatar
Linus Torvalds committed
584
			r4k_blast_scache();
585 586
		else
			blast_scache_range(addr, addr + size);
Linus Torvalds's avatar
Linus Torvalds committed
587 588 589 590 591 592 593 594
		return;
	}

	/*
	 * Either no secondary cache or the available caches don't have the
	 * subset property so we have to flush the primary caches
	 * explicitly
	 */
595
	if (cpu_has_safe_index_cacheops && size >= dcache_size) {
Linus Torvalds's avatar
Linus Torvalds committed
596 597 598
		r4k_blast_dcache();
	} else {
		R4600_HIT_CACHEOP_WAR_IMPL;
599
		blast_dcache_range(addr, addr + size);
Linus Torvalds's avatar
Linus Torvalds committed
600 601 602 603 604 605 606 607 608 609
	}

	bc_wback_inv(addr, size);
}

static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
{
	/* Catch bad driver code */
	BUG_ON(size == 0);

610
	if (cpu_has_inclusive_pcaches) {
611
		if (size >= scache_size)
Linus Torvalds's avatar
Linus Torvalds committed
612
			r4k_blast_scache();
613
		else
614
			blast_inv_scache_range(addr, addr + size);
Linus Torvalds's avatar
Linus Torvalds committed
615 616 617
		return;
	}

618
	if (cpu_has_safe_index_cacheops && size >= dcache_size) {
Linus Torvalds's avatar
Linus Torvalds committed
619 620 621
		r4k_blast_dcache();
	} else {
		R4600_HIT_CACHEOP_WAR_IMPL;
622
		blast_inv_dcache_range(addr, addr + size);
Linus Torvalds's avatar
Linus Torvalds committed
623 624 625 626 627 628 629 630 631 632 633 634 635
	}

	bc_inv(addr, size);
}
#endif /* CONFIG_DMA_NONCOHERENT */

/*
 * While we're protected against bad userland addresses we don't care
 * very much about what happens in that case.  Usually a segmentation
 * fault will dump the process later on anyway ...
 */
static void local_r4k_flush_cache_sigtramp(void * arg)
{
Thiemo Seufer's avatar
Thiemo Seufer committed
636 637 638
	unsigned long ic_lsize = cpu_icache_line_size();
	unsigned long dc_lsize = cpu_dcache_line_size();
	unsigned long sc_lsize = cpu_scache_line_size();
Linus Torvalds's avatar
Linus Torvalds committed
639 640 641
	unsigned long addr = (unsigned long) arg;

	R4600_HIT_CACHEOP_WAR_IMPL;
642 643
	if (dc_lsize)
		protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
644
	if (!cpu_icache_snoops_remote_store && scache_size)
Linus Torvalds's avatar
Linus Torvalds committed
645
		protected_writeback_scache_line(addr & ~(sc_lsize - 1));
646 647
	if (ic_lsize)
		protected_flush_icache_line(addr & ~(ic_lsize - 1));
Linus Torvalds's avatar
Linus Torvalds committed
648 649 650 651 652
	if (MIPS4K_ICACHE_REFILL_WAR) {
		__asm__ __volatile__ (
			".set push\n\t"
			".set noat\n\t"
			".set mips3\n\t"
653
#ifdef CONFIG_32BIT
Linus Torvalds's avatar
Linus Torvalds committed
654 655
			"la	$at,1f\n\t"
#endif
656
#ifdef CONFIG_64BIT
Linus Torvalds's avatar
Linus Torvalds committed
657 658 659 660 661 662 663 664 665 666 667 668 669 670 671
			"dla	$at,1f\n\t"
#endif
			"cache	%0,($at)\n\t"
			"nop; nop; nop\n"
			"1:\n\t"
			".set pop"
			:
			: "i" (Hit_Invalidate_I));
	}
	if (MIPS_CACHE_SYNC_WAR)
		__asm__ __volatile__ ("sync");
}

static void r4k_flush_cache_sigtramp(unsigned long addr)
{
672
	r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1, 1);
Linus Torvalds's avatar
Linus Torvalds committed
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691
}

static void r4k_flush_icache_all(void)
{
	if (cpu_has_vtag_icache)
		r4k_blast_icache();
}

static inline void rm7k_erratum31(void)
{
	const unsigned long ic_lsize = 32;
	unsigned long addr;

	/* RM7000 erratum #31. The icache is screwed at startup. */
	write_c0_taglo(0);
	write_c0_taghi(0);

	for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
		__asm__ __volatile__ (
Thiemo Seufer's avatar
Thiemo Seufer committed
692
			".set push\n\t"
Linus Torvalds's avatar
Linus Torvalds committed
693 694 695 696 697 698 699 700 701 702 703 704 705 706
			".set noreorder\n\t"
			".set mips3\n\t"
			"cache\t%1, 0(%0)\n\t"
			"cache\t%1, 0x1000(%0)\n\t"
			"cache\t%1, 0x2000(%0)\n\t"
			"cache\t%1, 0x3000(%0)\n\t"
			"cache\t%2, 0(%0)\n\t"
			"cache\t%2, 0x1000(%0)\n\t"
			"cache\t%2, 0x2000(%0)\n\t"
			"cache\t%2, 0x3000(%0)\n\t"
			"cache\t%1, 0(%0)\n\t"
			"cache\t%1, 0x1000(%0)\n\t"
			"cache\t%1, 0x2000(%0)\n\t"
			"cache\t%1, 0x3000(%0)\n\t"
Thiemo Seufer's avatar
Thiemo Seufer committed
707
			".set pop\n"
Linus Torvalds's avatar
Linus Torvalds committed
708 709 710 711 712
			:
			: "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill));
	}
}

713
static char *way_string[] __cpuinitdata = { NULL, "direct mapped", "2-way",
Linus Torvalds's avatar
Linus Torvalds committed
714 715 716
	"3-way", "4-way", "5-way", "6-way", "7-way", "8-way"
};

717
static void __cpuinit probe_pcache(void)
Linus Torvalds's avatar
Linus Torvalds committed
718 719 720 721 722 723 724 725 726 727 728 729 730 731 732
{
	struct cpuinfo_mips *c = &current_cpu_data;
	unsigned int config = read_c0_config();
	unsigned int prid = read_c0_prid();
	unsigned long config1;
	unsigned int lsize;

	switch (c->cputype) {
	case CPU_R4600:			/* QED style two way caches? */
	case CPU_R4700:
	case CPU_R5000:
	case CPU_NEVADA:
		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 2;
733
		c->icache.waybit = __ffs(icache_size/2);
Linus Torvalds's avatar
Linus Torvalds committed
734 735 736 737

		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 2;
738
		c->dcache.waybit= __ffs(dcache_size/2);
Linus Torvalds's avatar
Linus Torvalds committed
739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769

		c->options |= MIPS_CPU_CACHE_CDEX_P;
		break;

	case CPU_R5432:
	case CPU_R5500:
		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 2;
		c->icache.waybit= 0;

		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 2;
		c->dcache.waybit = 0;

		c->options |= MIPS_CPU_CACHE_CDEX_P;
		break;

	case CPU_TX49XX:
		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 4;
		c->icache.waybit= 0;

		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 4;
		c->dcache.waybit = 0;

		c->options |= MIPS_CPU_CACHE_CDEX_P;
Atsushi Nemoto's avatar
Atsushi Nemoto committed
770
		c->options |= MIPS_CPU_PREFETCH;
Linus Torvalds's avatar
Linus Torvalds committed
771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794
		break;

	case CPU_R4000PC:
	case CPU_R4000SC:
	case CPU_R4000MC:
	case CPU_R4400PC:
	case CPU_R4400SC:
	case CPU_R4400MC:
	case CPU_R4300:
		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 1;
		c->icache.waybit = 0; 	/* doesn't matter */

		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 1;
		c->dcache.waybit = 0;	/* does not matter */

		c->options |= MIPS_CPU_CACHE_CDEX_P;
		break;

	case CPU_R10000:
	case CPU_R12000:
Kumba's avatar
Kumba committed
795
	case CPU_R14000:
Linus Torvalds's avatar
Linus Torvalds committed
796 797 798 799 800 801 802 803 804 805 806 807 808 809
		icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
		c->icache.linesz = 64;
		c->icache.ways = 2;
		c->icache.waybit = 0;

		dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
		c->dcache.linesz = 32;
		c->dcache.ways = 2;
		c->dcache.waybit = 0;

		c->options |= MIPS_CPU_PREFETCH;
		break;

	case CPU_VR4133:
810
		write_c0_config(config & ~VR41_CONF_P4K);
Linus Torvalds's avatar
Linus Torvalds committed
811 812 813 814
	case CPU_VR4131:
		/* Workaround for cache instruction bug of VR4131 */
		if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
		    c->processor_id == 0x0c82U) {
815 816 817
			config |= 0x00400000U;
			if (c->processor_id == 0x0c80U)
				config |= VR41_CONF_BP;
Linus Torvalds's avatar
Linus Torvalds committed
818
			write_c0_config(config);
819 820 821
		} else
			c->options |= MIPS_CPU_CACHE_CDEX_P;

Linus Torvalds's avatar
Linus Torvalds committed
822 823 824
		icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 2;
825
		c->icache.waybit = __ffs(icache_size/2);
Linus Torvalds's avatar
Linus Torvalds committed
826 827 828 829

		dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 2;
830
		c->dcache.waybit = __ffs(dcache_size/2);
Linus Torvalds's avatar
Linus Torvalds committed
831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858
		break;

	case CPU_VR41XX:
	case CPU_VR4111:
	case CPU_VR4121:
	case CPU_VR4122:
	case CPU_VR4181:
	case CPU_VR4181A:
		icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 1;
		c->icache.waybit = 0; 	/* doesn't matter */

		dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 1;
		c->dcache.waybit = 0;	/* does not matter */

		c->options |= MIPS_CPU_CACHE_CDEX_P;
		break;

	case CPU_RM7000:
		rm7k_erratum31();

	case CPU_RM9000:
		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		c->icache.ways = 4;
859
		c->icache.waybit = __ffs(icache_size / c->icache.ways);
Linus Torvalds's avatar
Linus Torvalds committed
860 861 862 863

		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		c->dcache.ways = 4;
864
		c->dcache.waybit = __ffs(dcache_size / c->dcache.ways);
Linus Torvalds's avatar
Linus Torvalds committed
865 866 867 868 869 870 871

#if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR)
		c->options |= MIPS_CPU_CACHE_CDEX_P;
#endif
		c->options |= MIPS_CPU_PREFETCH;
		break;

872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889
	case CPU_LOONGSON2:
		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
		if (prid & 0x3)
			c->icache.ways = 4;
		else
			c->icache.ways = 2;
		c->icache.waybit = 0;

		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
		if (prid & 0x3)
			c->dcache.ways = 4;
		else
			c->dcache.ways = 2;
		c->dcache.waybit = 0;
		break;

Linus Torvalds's avatar
Linus Torvalds committed
890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909
	default:
		if (!(config & MIPS_CONF_M))
			panic("Don't know how to probe P-caches on this cpu.");

		/*
		 * So we seem to be a MIPS32 or MIPS64 CPU
		 * So let's probe the I-cache ...
		 */
		config1 = read_c0_config1();

		if ((lsize = ((config1 >> 19) & 7)))
			c->icache.linesz = 2 << lsize;
		else
			c->icache.linesz = lsize;
		c->icache.sets = 64 << ((config1 >> 22) & 7);
		c->icache.ways = 1 + ((config1 >> 16) & 7);

		icache_size = c->icache.sets *
		              c->icache.ways *
		              c->icache.linesz;
910
		c->icache.waybit = __ffs(icache_size/c->icache.ways);
Linus Torvalds's avatar
Linus Torvalds committed
911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929

		if (config & 0x8)		/* VI bit */
			c->icache.flags |= MIPS_CACHE_VTAG;

		/*
		 * Now probe the MIPS32 / MIPS64 data cache.
		 */
		c->dcache.flags = 0;

		if ((lsize = ((config1 >> 10) & 7)))
			c->dcache.linesz = 2 << lsize;
		else
			c->dcache.linesz= lsize;
		c->dcache.sets = 64 << ((config1 >> 13) & 7);
		c->dcache.ways = 1 + ((config1 >> 7) & 7);

		dcache_size = c->dcache.sets *
		              c->dcache.ways *
		              c->dcache.linesz;
930
		c->dcache.waybit = __ffs(dcache_size/c->dcache.ways);
Linus Torvalds's avatar
Linus Torvalds committed
931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952

		c->options |= MIPS_CPU_PREFETCH;
		break;
	}

	/*
	 * Processor configuration sanity check for the R4000SC erratum
	 * #5.  With page sizes larger than 32kB there is no possibility
	 * to get a VCE exception anymore so we don't care about this
	 * misconfiguration.  The case is rather theoretical anyway;
	 * presumably no vendor is shipping his hardware in the "bad"
	 * configuration.
	 */
	if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 &&
	    !(config & CONF_SC) && c->icache.linesz != 16 &&
	    PAGE_SIZE <= 0x8000)
		panic("Improper R4000SC processor configuration detected");

	/* compute a couple of other cache variables */
	c->icache.waysize = icache_size / c->icache.ways;
	c->dcache.waysize = dcache_size / c->dcache.ways;

953 954 955 956
	c->icache.sets = c->icache.linesz ?
		icache_size / (c->icache.linesz * c->icache.ways) : 0;
	c->dcache.sets = c->dcache.linesz ?
		dcache_size / (c->dcache.linesz * c->dcache.ways) : 0;
Linus Torvalds's avatar
Linus Torvalds committed
957 958 959 960 961 962 963

	/*
	 * R10000 and R12000 P-caches are odd in a positive way.  They're 32kB
	 * 2-way virtually indexed so normally would suffer from aliases.  So
	 * normally they'd suffer from aliases but magic in the hardware deals
	 * with that for us so we don't need to take care ourselves.
	 */
964
	switch (c->cputype) {
965
	case CPU_20KC:
966
	case CPU_25KF:
967 968
	case CPU_SB1:
	case CPU_SB1A:
969
		c->dcache.flags |= MIPS_CACHE_PINDEX;
970 971
		break;

972 973
	case CPU_R10000:
	case CPU_R12000:
Kumba's avatar
Kumba committed
974
	case CPU_R14000:
975
		break;
976

977
	case CPU_24K:
978
	case CPU_34K:
979
	case CPU_74K:
980
	case CPU_1004K:
981 982 983 984 985 986
		if ((read_c0_config7() & (1 << 16))) {
			/* effectively physically indexed dcache,
			   thus no virtual aliases. */
			c->dcache.flags |= MIPS_CACHE_PINDEX;
			break;
		}
987
	default:
988 989
		if (c->dcache.waysize > PAGE_SIZE)
			c->dcache.flags |= MIPS_CACHE_ALIASES;
990
	}
Linus Torvalds's avatar
Linus Torvalds committed
991 992 993 994 995 996 997 998 999 1000

	switch (c->cputype) {
	case CPU_20KC:
		/*
		 * Some older 20Kc chips doesn't have the 'VI' bit in
		 * the config register.
		 */
		c->icache.flags |= MIPS_CACHE_VTAG;
		break;

Pete Popov's avatar
Pete Popov committed
1001
	case CPU_AU1000:
Linus Torvalds's avatar
Linus Torvalds committed
1002
	case CPU_AU1500:
Pete Popov's avatar
Pete Popov committed
1003 1004 1005
	case CPU_AU1100:
	case CPU_AU1550:
	case CPU_AU1200:
1006 1007
	case CPU_AU1210:
	case CPU_AU1250:
Linus Torvalds's avatar
Linus Torvalds committed
1008 1009 1010 1011
		c->icache.flags |= MIPS_CACHE_IC_F_DC;
		break;
	}

1012 1013 1014 1015 1016 1017 1018 1019
#ifdef  CONFIG_CPU_LOONGSON2
	/*
	 * LOONGSON2 has 4 way icache, but when using indexed cache op,
	 * one op will act on all 4 ways
	 */
	c->icache.ways = 1;
#endif

Linus Torvalds's avatar
Linus Torvalds committed
1020 1021
	printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
	       icache_size >> 10,
1022
	       cpu_has_vtag_icache ? "VIVT" : "VIPT",
Linus Torvalds's avatar
Linus Torvalds committed
1023 1024
	       way_string[c->icache.ways], c->icache.linesz);

1025 1026 1027 1028 1029 1030
	printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
	       dcache_size >> 10, way_string[c->dcache.ways],
	       (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT",
	       (c->dcache.flags & MIPS_CACHE_ALIASES) ?
			"cache aliases" : "no aliases",
	       c->dcache.linesz);
Linus Torvalds's avatar
Linus Torvalds committed
1031 1032 1033 1034 1035 1036 1037 1038
}

/*
 * If you even _breathe_ on this function, look at the gcc output and make sure
 * it does not pop things on and off the stack for the cache sizing loop that
 * executes in KSEG1 space or else you will crash and burn badly.  You have
 * been warned.
 */
1039
static int __cpuinit probe_scache(void)
Linus Torvalds's avatar
Linus Torvalds committed
1040 1041 1042 1043 1044 1045 1046 1047 1048
{
	unsigned long flags, addr, begin, end, pow2;
	unsigned int config = read_c0_config();
	struct cpuinfo_mips *c = &current_cpu_data;
	int tmp;

	if (config & CONF_SC)
		return 0;

1049
	begin = (unsigned long) &_stext;
Linus Torvalds's avatar
Linus Torvalds committed
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095
	begin &= ~((4 * 1024 * 1024) - 1);
	end = begin + (4 * 1024 * 1024);

	/*
	 * This is such a bitch, you'd think they would make it easy to do
	 * this.  Away you daemons of stupidity!
	 */
	local_irq_save(flags);

	/* Fill each size-multiple cache line with a valid tag. */
	pow2 = (64 * 1024);
	for (addr = begin; addr < end; addr = (begin + pow2)) {
		unsigned long *p = (unsigned long *) addr;
		__asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
		pow2 <<= 1;
	}

	/* Load first line with zero (therefore invalid) tag. */
	write_c0_taglo(0);
	write_c0_taghi(0);
	__asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
	cache_op(Index_Store_Tag_I, begin);
	cache_op(Index_Store_Tag_D, begin);
	cache_op(Index_Store_Tag_SD, begin);

	/* Now search for the wrap around point. */
	pow2 = (128 * 1024);
	tmp = 0;
	for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
		cache_op(Index_Load_Tag_SD, addr);
		__asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
		if (!read_c0_taglo())
			break;
		pow2 <<= 1;
	}
	local_irq_restore(flags);
	addr -= begin;

	scache_size = addr;
	c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22);
	c->scache.ways = 1;
	c->dcache.waybit = 0;		/* does not matter */

	return 1;
}

1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
#if defined(CONFIG_CPU_LOONGSON2)
static void __init loongson2_sc_init(void)
{
	struct cpuinfo_mips *c = &current_cpu_data;

	scache_size = 512*1024;
	c->scache.linesz = 32;
	c->scache.ways = 4;
	c->scache.waybit = 0;
	c->scache.waysize = scache_size / (c->scache.ways);
	c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
	pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
	       scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);

	c->options |= MIPS_CPU_INCLUSIVE_CACHES;
}
#endif

Linus Torvalds's avatar
Linus Torvalds committed
1114 1115
extern int r5k_sc_init(void);
extern int rm7k_sc_init(void);
1116
extern int mips_sc_init(void);
Linus Torvalds's avatar
Linus Torvalds committed
1117

1118
static void __cpuinit setup_scache(void)
Linus Torvalds's avatar
Linus Torvalds committed
1119 1120 1121 1122 1123 1124 1125 1126
{
	struct cpuinfo_mips *c = &current_cpu_data;
	unsigned int config = read_c0_config();
	int sc_present = 0;

	/*
	 * Do the probing thing on R4000SC and R4400SC processors.  Other
	 * processors don't have a S-cache that would be relevant to the
Joe Perches's avatar
Joe Perches committed
1127
	 * Linux memory management.
Linus Torvalds's avatar
Linus Torvalds committed
1128 1129 1130 1131 1132 1133
	 */
	switch (c->cputype) {
	case CPU_R4000SC:
	case CPU_R4000MC:
	case CPU_R4400SC:
	case CPU_R4400MC:
1134
		sc_present = run_uncached(probe_scache);
Linus Torvalds's avatar
Linus Torvalds committed
1135 1136 1137 1138 1139 1140
		if (sc_present)
			c->options |= MIPS_CPU_CACHE_CDEX_S;
		break;

	case CPU_R10000:
	case CPU_R12000:
Kumba's avatar
Kumba committed
1141
	case CPU_R14000:
Linus Torvalds's avatar
Linus Torvalds committed
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
		scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16);
		c->scache.linesz = 64 << ((config >> 13) & 1);
		c->scache.ways = 2;
		c->scache.waybit= 0;
		sc_present = 1;
		break;

	case CPU_R5000:
	case CPU_NEVADA:
#ifdef CONFIG_R5000_CPU_SCACHE
		r5k_sc_init();
#endif
                return;

	case CPU_RM7000:
	case CPU_RM9000:
#ifdef CONFIG_RM7000_CPU_SCACHE
		rm7k_sc_init();
#endif
		return;

1163 1164 1165 1166 1167 1168
#if defined(CONFIG_CPU_LOONGSON2)
	case CPU_LOONGSON2:
		loongson2_sc_init();
		return;
#endif

Linus Torvalds's avatar
Linus Torvalds committed
1169
	default:
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186
		if (c->isa_level == MIPS_CPU_ISA_M32R1 ||
		    c->isa_level == MIPS_CPU_ISA_M32R2 ||
		    c->isa_level == MIPS_CPU_ISA_M64R1 ||
		    c->isa_level == MIPS_CPU_ISA_M64R2) {
#ifdef CONFIG_MIPS_CPU_SCACHE
			if (mips_sc_init ()) {
				scache_size = c->scache.ways * c->scache.sets * c->scache.linesz;
				printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n",
				       scache_size >> 10,
				       way_string[c->scache.ways], c->scache.linesz);
			}
#else
			if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT))
				panic("Dunno how to handle MIPS32 / MIPS64 second level cache");
#endif
			return;
		}
Linus Torvalds's avatar
Linus Torvalds committed
1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200
		sc_present = 0;
	}

	if (!sc_present)
		return;

	/* compute a couple of other cache variables */
	c->scache.waysize = scache_size / c->scache.ways;

	c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);

	printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
	       scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);

1201
	c->options |= MIPS_CPU_INCLUSIVE_CACHES;
Linus Torvalds's avatar
Linus Torvalds committed
1202 1203
}

1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
void au1x00_fixup_config_od(void)
{
	/*
	 * c0_config.od (bit 19) was write only (and read as 0)
	 * on the early revisions of Alchemy SOCs.  It disables the bus
	 * transaction overlapping and needs to be set to fix various errata.
	 */
	switch (read_c0_prid()) {
	case 0x00030100: /* Au1000 DA */
	case 0x00030201: /* Au1000 HA */
	case 0x00030202: /* Au1000 HB */
	case 0x01030200: /* Au1500 AB */
	/*
	 * Au1100 errata actually keeps silence about this bit, so we set it
	 * just in case for those revisions that require it to be set according
	 * to arch/mips/au1000/common/cputable.c
	 */
	case 0x02030200: /* Au1100 AB */
	case 0x02030201: /* Au1100 BA */
	case 0x02030202: /* Au1100 BC */
		set_c0_config(1 << 19);
		break;
	}
}

1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
static int __cpuinitdata cca = -1;

static int __init cca_setup(char *str)
{
	get_option(&str, &cca);

	return 1;
}

__setup("cca=", cca_setup);

1240
static void __cpuinit coherency_setup(void)
Linus Torvalds's avatar
Linus Torvalds committed
1241
{
1242 1243 1244 1245 1246 1247
	if (cca < 0 || cca > 7)
		cca = read_c0_config() & CONF_CM_CMASK;
	_page_cachable_default = cca << _CACHE_SHIFT;

	pr_debug("Using cache attribute %d\n", cca);
	change_c0_config(CONF_CM_CMASK, cca);
Linus Torvalds's avatar
Linus Torvalds committed
1248 1249 1250 1251 1252 1253 1254 1255

	/*
	 * c0_status.cu=0 specifies that updates by the sc instruction use
	 * the coherency mode specified by the TLB; 1 means cachable
	 * coherent update on write will be used.  Not all processors have
	 * this bit and; some wire it to zero, others like Toshiba had the
	 * silly idea of putting something else there ...
	 */
1256
	switch (current_cpu_type()) {
Linus Torvalds's avatar
Linus Torvalds committed
1257 1258 1259 1260 1261 1262 1263 1264
	case CPU_R4000PC:
	case CPU_R4000SC:
	case CPU_R4000MC:
	case CPU_R4400PC:
	case CPU_R4400SC:
	case CPU_R4400MC:
		clear_c0_config(CONF_CU);
		break;
1265
	/*
Ralf Baechle's avatar
Ralf Baechle committed
1266
	 * We need to catch the early Alchemy SOCs with
1267 1268 1269 1270 1271 1272 1273
	 * the write-only co_config.od bit and set it back to one...
	 */
	case CPU_AU1000: /* rev. DA, HA, HB */
	case CPU_AU1100: /* rev. AB, BA, BC ?? */
	case CPU_AU1500: /* rev. AB */
		au1x00_fixup_config_od();
		break;
Linus Torvalds's avatar
Linus Torvalds committed
1274 1275 1276
	}
}

1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
#if defined(CONFIG_DMA_NONCOHERENT)

static int __cpuinitdata coherentio;

static int __init setcoherentio(char *str)
{
	coherentio = 1;

	return 1;
}

__setup("coherentio", setcoherentio);
#endif

1291
void __cpuinit r4k_cache_init(void)
Linus Torvalds's avatar
Linus Torvalds committed
1292 1293 1294
{
	extern void build_clear_page(void);
	extern void build_copy_page(void);
1295 1296
	extern char __weak except_vec2_generic;
	extern char __weak except_vec2_sb1;
Linus Torvalds's avatar
Linus Torvalds committed
1297 1298
	struct cpuinfo_mips *c = &current_cpu_data;

1299 1300 1301 1302 1303 1304 1305 1306 1307 1308
	switch (c->cputype) {
	case CPU_SB1:
	case CPU_SB1A:
		set_uncached_handler(0x100, &except_vec2_sb1, 0x80);
		break;

	default:
		set_uncached_handler(0x100, &except_vec2_generic, 0x80);
		break;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327

	probe_pcache();
	setup_scache();

	r4k_blast_dcache_page_setup();
	r4k_blast_dcache_page_indexed_setup();
	r4k_blast_dcache_setup();
	r4k_blast_icache_page_setup();
	r4k_blast_icache_page_indexed_setup();
	r4k_blast_icache_setup();
	r4k_blast_scache_page_setup();
	r4k_blast_scache_page_indexed_setup();
	r4k_blast_scache_setup();

	/*
	 * Some MIPS32 and MIPS64 processors have physically indexed caches.
	 * This code supports virtually indexed processors and will be
	 * unnecessarily inefficient on physically indexed processors.
	 */
1328 1329 1330 1331 1332 1333
	if (c->dcache.linesz)
		shm_align_mask = max_t( unsigned long,
					c->dcache.sets * c->dcache.linesz - 1,
					PAGE_SIZE - 1);
	else
		shm_align_mask = PAGE_SIZE-1;
1334 1335 1336 1337

	__flush_cache_vmap	= r4k__flush_cache_vmap;
	__flush_cache_vunmap	= r4k__flush_cache_vunmap;

1338
	flush_cache_all		= cache_noop;
Linus Torvalds's avatar
Linus Torvalds committed
1339 1340 1341 1342 1343 1344 1345
	__flush_cache_all	= r4k___flush_cache_all;
	flush_cache_mm		= r4k_flush_cache_mm;
	flush_cache_page	= r4k_flush_cache_page;
	flush_cache_range	= r4k_flush_cache_range;

	flush_cache_sigtramp	= r4k_flush_cache_sigtramp;
	flush_icache_all	= r4k_flush_icache_all;
1346
	local_flush_data_cache_page	= local_r4k_flush_data_cache_page;
Linus Torvalds's avatar
Linus Torvalds committed
1347 1348 1349
	flush_data_cache_page	= r4k_flush_data_cache_page;
	flush_icache_range	= r4k_flush_icache_range;

1350 1351 1352 1353 1354 1355 1356 1357 1358 1359
#if defined(CONFIG_DMA_NONCOHERENT)
	if (coherentio) {
		_dma_cache_wback_inv	= (void *)cache_noop;
		_dma_cache_wback	= (void *)cache_noop;
		_dma_cache_inv		= (void *)cache_noop;
	} else {
		_dma_cache_wback_inv	= r4k_dma_cache_wback_inv;
		_dma_cache_wback	= r4k_dma_cache_wback_inv;
		_dma_cache_inv		= r4k_dma_cache_inv;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1360 1361 1362 1363
#endif

	build_clear_page();
	build_copy_page();
1364
#if !defined(CONFIG_MIPS_CMP)
1365
	local_r4k___flush_cache_all(NULL);
1366
#endif
1367
	coherency_setup();
Linus Torvalds's avatar
Linus Torvalds committed
1368
}