init.c 49.1 KB
Newer Older
1
/*  $Id: init.c,v 1.209 2002/02/09 19:49:31 davem Exp $
Linus Torvalds's avatar
Linus Torvalds committed
2 3 4 5 6 7 8 9 10 11 12 13 14
 *  arch/sparc64/mm/init.c
 *
 *  Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
 *  Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
 */
 
#include <linux/config.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/mm.h>
Linus Torvalds's avatar
Linus Torvalds committed
15
#include <linux/slab.h>
Linus Torvalds's avatar
Linus Torvalds committed
16 17 18
#include <linux/blk.h>
#include <linux/swap.h>
#include <linux/swapctl.h>
Linus Torvalds's avatar
Linus Torvalds committed
19 20 21
#include <linux/pagemap.h>
#include <linux/fs.h>
#include <linux/seq_file.h>
Linus Torvalds's avatar
Linus Torvalds committed
22 23 24 25 26 27 28 29 30 31 32 33 34

#include <asm/head.h>
#include <asm/system.h>
#include <asm/page.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/oplib.h>
#include <asm/iommu.h>
#include <asm/io.h>
#include <asm/uaccess.h>
#include <asm/mmu_context.h>
#include <asm/dma.h>
#include <asm/starfire.h>
Linus Torvalds's avatar
Linus Torvalds committed
35
#include <asm/tlb.h>
Linus Torvalds's avatar
Linus Torvalds committed
36
#include <asm/spitfire.h>
Linus Torvalds's avatar
Linus Torvalds committed
37 38

mmu_gather_t mmu_gathers[NR_CPUS];
Linus Torvalds's avatar
Linus Torvalds committed
39 40 41 42 43 44 45 46 47 48

extern void device_scan(void);

struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS];

unsigned long *sparc64_valid_addr_bitmap;

/* Ugly, but necessary... -DaveM */
unsigned long phys_base;

Linus Torvalds's avatar
Linus Torvalds committed
49 50
enum ultra_tlb_layout tlb_type = spitfire;

Linus Torvalds's avatar
Linus Torvalds committed
51 52 53 54 55 56 57 58 59 60 61 62 63
/* get_new_mmu_context() uses "cache + 1".  */
spinlock_t ctx_alloc_lock = SPIN_LOCK_UNLOCKED;
unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;
#define CTX_BMAP_SLOTS (1UL << (CTX_VERSION_SHIFT - 6))
unsigned long mmu_context_bmap[CTX_BMAP_SLOTS];

/* References to section boundaries */
extern char __init_begin, __init_end, _start, _end, etext, edata;

/* Initial ramdisk setup */
extern unsigned int sparc_ramdisk_image;
extern unsigned int sparc_ramdisk_size;

Linus Torvalds's avatar
Linus Torvalds committed
64 65
struct page *mem_map_zero;

Linus Torvalds's avatar
Linus Torvalds committed
66 67
int bigkernel = 0;

68 69 70
/* XXX Tune this... */
#define PGT_CACHE_LOW	25
#define PGT_CACHE_HIGH	50
Linus Torvalds's avatar
Linus Torvalds committed
71

72 73 74 75
void check_pgt_cache(void)
{
	preempt_disable();
	if (pgtable_cache_size > PGT_CACHE_HIGH) {
Linus Torvalds's avatar
Linus Torvalds committed
76 77
		do {
#ifdef CONFIG_SMP
Linus Torvalds's avatar
Linus Torvalds committed
78
			if (pgd_quicklist)
79
				free_pgd_slow(get_pgd_fast());
Linus Torvalds's avatar
Linus Torvalds committed
80
#endif
Linus Torvalds's avatar
Linus Torvalds committed
81
			if (pte_quicklist[0])
82
				free_pte_slow(pte_alloc_one_fast(NULL, 0));
Linus Torvalds's avatar
Linus Torvalds committed
83
			if (pte_quicklist[1])
84 85
				free_pte_slow(pte_alloc_one_fast(NULL, 1 << (PAGE_SHIFT + 10)));
		} while (pgtable_cache_size > PGT_CACHE_LOW);
Linus Torvalds's avatar
Linus Torvalds committed
86
	}
87 88
#ifndef CONFIG_SMP
        if (pgd_cache_size > PGT_CACHE_HIGH / 4) {
Linus Torvalds's avatar
Linus Torvalds committed
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
		struct page *page, *page2;
                for (page2 = NULL, page = (struct page *)pgd_quicklist; page;) {
                        if ((unsigned long)page->pprev_hash == 3) {
                                if (page2)
                                        page2->next_hash = page->next_hash;
                                else
                                        (struct page *)pgd_quicklist = page->next_hash;
                                page->next_hash = NULL;
                                page->pprev_hash = NULL;
                                pgd_cache_size -= 2;
                                __free_page(page);
                                if (page2)
                                        page = page2->next_hash;
                                else
                                        page = (struct page *)pgd_quicklist;
104
                                if (pgd_cache_size <= PGT_CACHE_LOW / 4)
Linus Torvalds's avatar
Linus Torvalds committed
105 106 107 108 109 110 111 112
                                        break;
                                continue;
                        }
                        page2 = page;
                        page = page->next_hash;
                }
        }
#endif
113
	preempt_enable();
Linus Torvalds's avatar
Linus Torvalds committed
114 115
}

Linus Torvalds's avatar
Linus Torvalds committed
116
#ifdef CONFIG_DEBUG_DCFLUSH
Linus Torvalds's avatar
Linus Torvalds committed
117 118 119 120 121 122 123 124
atomic_t dcpage_flushes = ATOMIC_INIT(0);
#ifdef CONFIG_SMP
atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
#endif
#endif

__inline__ void flush_dcache_page_impl(struct page *page)
{
Linus Torvalds's avatar
Linus Torvalds committed
125
#ifdef CONFIG_DEBUG_DCFLUSH
Linus Torvalds's avatar
Linus Torvalds committed
126 127 128 129 130 131 132 133 134 135 136 137 138 139
	atomic_inc(&dcpage_flushes);
#endif

#if (L1DCACHE_SIZE > PAGE_SIZE)
	__flush_dcache_page(page->virtual,
			    ((tlb_type == spitfire) &&
			     page->mapping != NULL));
#else
	if (page->mapping != NULL &&
	    tlb_type == spitfire)
		__flush_icache_page(__pa(page->virtual));
#endif
}

Linus Torvalds's avatar
Linus Torvalds committed
140 141 142 143 144 145 146 147
#define PG_dcache_dirty		PG_arch_1

#define dcache_dirty_cpu(page) \
	(((page)->flags >> 24) & (NR_CPUS - 1UL))

static __inline__ void set_dcache_dirty(struct page *page)
{
	unsigned long mask = smp_processor_id();
148
	unsigned long non_cpu_bits = ~((NR_CPUS - 1UL) << 24UL);
Linus Torvalds's avatar
Linus Torvalds committed
149 150 151 152 153 154 155 156
	mask = (mask << 24) | (1UL << PG_dcache_dirty);
	__asm__ __volatile__("1:\n\t"
			     "ldx	[%2], %%g7\n\t"
			     "and	%%g7, %1, %%g5\n\t"
			     "or	%%g5, %0, %%g5\n\t"
			     "casx	[%2], %%g7, %%g5\n\t"
			     "cmp	%%g7, %%g5\n\t"
			     "bne,pn	%%xcc, 1b\n\t"
Linus Torvalds's avatar
Linus Torvalds committed
157
			     " membar	#StoreLoad | #StoreStore"
Linus Torvalds's avatar
Linus Torvalds committed
158 159 160 161 162 163 164 165 166 167 168 169 170
			     : /* no outputs */
			     : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)
			     : "g5", "g7");
}

static __inline__ void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu)
{
	unsigned long mask = (1UL << PG_dcache_dirty);

	__asm__ __volatile__("! test_and_clear_dcache_dirty\n"
			     "1:\n\t"
			     "ldx	[%2], %%g7\n\t"
			     "srlx	%%g7, 24, %%g5\n\t"
171
			     "and	%%g5, %3, %%g5\n\t"
Linus Torvalds's avatar
Linus Torvalds committed
172 173 174 175 176 177
			     "cmp	%%g5, %0\n\t"
			     "bne,pn	%%icc, 2f\n\t"
			     " andn	%%g7, %1, %%g5\n\t"
			     "casx	[%2], %%g7, %%g5\n\t"
			     "cmp	%%g7, %%g5\n\t"
			     "bne,pn	%%xcc, 1b\n\t"
Linus Torvalds's avatar
Linus Torvalds committed
178
			     " membar	#StoreLoad | #StoreStore\n"
Linus Torvalds's avatar
Linus Torvalds committed
179 180
			     "2:"
			     : /* no outputs */
181 182
			     : "r" (cpu), "r" (mask), "r" (&page->flags),
			       "i" (NR_CPUS - 1UL)
Linus Torvalds's avatar
Linus Torvalds committed
183 184 185
			     : "g5", "g7");
}

186 187
extern void __update_mmu_cache(unsigned long mmu_context_hw, unsigned long address, pte_t pte, int code);

Linus Torvalds's avatar
Linus Torvalds committed
188 189
void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
{
190 191
	struct page *page;
	unsigned long pfn;
Linus Torvalds's avatar
Linus Torvalds committed
192 193
	unsigned long pg_flags;

194 195 196
	pfn = pte_pfn(pte);
	if (pfn_valid(pfn) &&
	    (page = pfn_to_page(pfn), page->mapping) &&
Linus Torvalds's avatar
Linus Torvalds committed
197
	    ((pg_flags = page->flags) & (1UL << PG_dcache_dirty))) {
198
		int cpu = ((pg_flags >> 24) & (NR_CPUS - 1UL));
Linus Torvalds's avatar
Linus Torvalds committed
199

Linus Torvalds's avatar
Linus Torvalds committed
200 201 202
		/* This is just to optimize away some function calls
		 * in the SMP case.
		 */
Linus Torvalds's avatar
Linus Torvalds committed
203
		if (cpu == smp_processor_id())
Linus Torvalds's avatar
Linus Torvalds committed
204 205
			flush_dcache_page_impl(page);
		else
Linus Torvalds's avatar
Linus Torvalds committed
206
			smp_flush_dcache_page_impl(page, cpu);
Linus Torvalds's avatar
Linus Torvalds committed
207

Linus Torvalds's avatar
Linus Torvalds committed
208
		clear_dcache_dirty_cpu(page, cpu);
Linus Torvalds's avatar
Linus Torvalds committed
209
	}
210 211 212
	if (get_thread_fault_code())
		__update_mmu_cache(vma->vm_mm->context & TAG_CONTEXT_BITS,
				   address, pte, get_thread_fault_code());
Linus Torvalds's avatar
Linus Torvalds committed
213 214
}

Linus Torvalds's avatar
Linus Torvalds committed
215 216 217 218 219 220
void flush_dcache_page(struct page *page)
{
	int dirty = test_bit(PG_dcache_dirty, &page->flags);
	int dirty_cpu = dcache_dirty_cpu(page);

	if (page->mapping &&
221 222
	    list_empty(&page->mapping->i_mmap) &&
	    list_empty(&page->mapping->i_mmap_shared)) {
Linus Torvalds's avatar
Linus Torvalds committed
223 224 225
		if (dirty) {
			if (dirty_cpu == smp_processor_id())
				return;
Linus Torvalds's avatar
Linus Torvalds committed
226
			smp_flush_dcache_page_impl(page, dirty_cpu);
Linus Torvalds's avatar
Linus Torvalds committed
227 228 229 230 231 232 233 234 235 236 237 238
		}
		set_dcache_dirty(page);
	} else {
		/* We could delay the flush for the !page->mapping
		 * case too.  But that case is for exec env/arg
		 * pages and those are %99 certainly going to get
		 * faulted into the tlb (and thus flushed) anyways.
		 */
		flush_dcache_page_impl(page);
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
239 240 241 242 243 244 245 246 247 248 249 250 251 252
/* When shared+writable mmaps of files go away, we lose all dirty
 * page state, so we have to deal with D-cache aliasing here.
 *
 * This code relies on the fact that flush_cache_range() is always
 * called for an area composed by a single VMA.  It also assumes that
 * the MM's page_table_lock is held.
 */
static inline void flush_cache_pte_range(struct mm_struct *mm, pmd_t *pmd, unsigned long address, unsigned long size)
{
	unsigned long offset;
	pte_t *ptep;

	if (pmd_none(*pmd))
		return;
253
	ptep = pte_offset_map(pmd, address);
Linus Torvalds's avatar
Linus Torvalds committed
254 255 256 257 258 259 260 261 262 263 264
	offset = address & ~PMD_MASK;
	if (offset + size > PMD_SIZE)
		size = PMD_SIZE - offset;
	size &= PAGE_MASK;
	for (offset = 0; offset < size; ptep++, offset += PAGE_SIZE) {
		pte_t pte = *ptep;

		if (pte_none(pte))
			continue;

		if (pte_present(pte) && pte_dirty(pte)) {
265
			struct page *page;
Linus Torvalds's avatar
Linus Torvalds committed
266
			unsigned long pgaddr, uaddr;
267
			unsigned long pfn = pte_pfn(pte);
Linus Torvalds's avatar
Linus Torvalds committed
268

269 270 271 272
			if (!pfn_valid(pfn))
				continue;
			page = pfn_to_page(pfn);
			if (PageReserved(page) || !page->mapping)
Linus Torvalds's avatar
Linus Torvalds committed
273 274 275 276 277 278 279
				continue;
			pgaddr = (unsigned long) page_address(page);
			uaddr = address + offset;
			if ((pgaddr ^ uaddr) & (1 << 13))
				flush_dcache_page_all(mm, page);
		}
	}
280
	pte_unmap(ptep - 1);
Linus Torvalds's avatar
Linus Torvalds committed
281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
}

static inline void flush_cache_pmd_range(struct mm_struct *mm, pgd_t *dir, unsigned long address, unsigned long size)
{
	pmd_t *pmd;
	unsigned long end;

	if (pgd_none(*dir))
		return;
	pmd = pmd_offset(dir, address);
	end = address + size;
	if (end > ((address + PGDIR_SIZE) & PGDIR_MASK))
		end = ((address + PGDIR_SIZE) & PGDIR_MASK);
	do {
		flush_cache_pte_range(mm, pmd, address, end - address);
		address = (address + PMD_SIZE) & PMD_MASK;
		pmd++;
	} while (address < end);
}

void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
{
	struct mm_struct *mm = vma->vm_mm;
	pgd_t *dir = pgd_offset(mm, start);

	if (mm == current->mm)
		flushw_user();

	if (vma->vm_file == NULL ||
	    ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE)))
		return;

	do {
		flush_cache_pmd_range(mm, dir, start, end - start);
		start = (start + PGDIR_SIZE) & PGDIR_MASK;
		dir++;
	} while (start && (start < end));
}

Linus Torvalds's avatar
Linus Torvalds committed
320 321
void flush_icache_range(unsigned long start, unsigned long end)
{
Linus Torvalds's avatar
Linus Torvalds committed
322 323 324
	/* Cheetah has coherent I-cache. */
	if (tlb_type == spitfire) {
		unsigned long kaddr;
Linus Torvalds's avatar
Linus Torvalds committed
325

Linus Torvalds's avatar
Linus Torvalds committed
326 327 328
		for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE)
			__flush_icache_page(__get_phys(kaddr));
	}
Linus Torvalds's avatar
Linus Torvalds committed
329 330 331 332 333 334 335 336 337 338 339 340 341 342
}

void show_mem(void)
{
	printk("Mem-info:\n");
	show_free_areas();
	printk("Free swap:       %6dkB\n",
	       nr_swap_pages << (PAGE_SHIFT-10));
	printk("%ld pages of RAM\n", num_physpages);
	printk("%d free pages\n", nr_free_pages());
	printk("%d pages in page table cache\n",pgtable_cache_size);
#ifndef CONFIG_SMP
	printk("%d entries in page dir cache\n",pgd_cache_size);
#endif	
343
	printk("%ld buffermem pages\n", nr_buffermem_pages());
Linus Torvalds's avatar
Linus Torvalds committed
344 345
}

Linus Torvalds's avatar
Linus Torvalds committed
346
void mmu_info(struct seq_file *m)
Linus Torvalds's avatar
Linus Torvalds committed
347
{
Linus Torvalds's avatar
Linus Torvalds committed
348
	if (tlb_type == cheetah)
Linus Torvalds's avatar
Linus Torvalds committed
349
		seq_printf(m, "MMU Type\t: Cheetah\n");
Linus Torvalds's avatar
Linus Torvalds committed
350
	else if (tlb_type == spitfire)
Linus Torvalds's avatar
Linus Torvalds committed
351
		seq_printf(m, "MMU Type\t: Spitfire\n");
Linus Torvalds's avatar
Linus Torvalds committed
352
	else
Linus Torvalds's avatar
Linus Torvalds committed
353
		seq_printf(m, "MMU Type\t: ???\n");
Linus Torvalds's avatar
Linus Torvalds committed
354

Linus Torvalds's avatar
Linus Torvalds committed
355
#ifdef CONFIG_DEBUG_DCFLUSH
Linus Torvalds's avatar
Linus Torvalds committed
356 357
	seq_printf(m, "DCPageFlushes\t: %d\n",
		   atomic_read(&dcpage_flushes));
Linus Torvalds's avatar
Linus Torvalds committed
358
#ifdef CONFIG_SMP
Linus Torvalds's avatar
Linus Torvalds committed
359 360
	seq_printf(m, "DCPageFlushesXC\t: %d\n",
		   atomic_read(&dcpage_flushes_xcall));
Linus Torvalds's avatar
Linus Torvalds committed
361
#endif /* CONFIG_SMP */
Linus Torvalds's avatar
Linus Torvalds committed
362
#endif /* CONFIG_DEBUG_DCFLUSH */
Linus Torvalds's avatar
Linus Torvalds committed
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384
}

struct linux_prom_translation {
	unsigned long virt;
	unsigned long size;
	unsigned long data;
};

extern unsigned long prom_boot_page;
extern void prom_remap(unsigned long physpage, unsigned long virtpage, int mmu_ihandle);
extern int prom_get_mmu_ihandle(void);
extern void register_prom_callbacks(void);

/* Exported for SMP bootup purposes. */
unsigned long kern_locked_tte_data;

void __init early_pgtable_allocfail(char *type)
{
	prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
	prom_halt();
}

Linus Torvalds's avatar
Linus Torvalds committed
385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402
#define BASE_PAGE_SIZE 8192
static pmd_t *prompmd;

/*
 * Translate PROM's mapping we capture at boot time into physical address.
 * The second parameter is only set from prom_callback() invocations.
 */
unsigned long prom_virt_to_phys(unsigned long promva, int *error)
{
	pmd_t *pmdp = prompmd + ((promva >> 23) & 0x7ff);
	pte_t *ptep;
	unsigned long base;

	if (pmd_none(*pmdp)) {
		if (error)
			*error = 1;
		return(0);
	}
403
	ptep = (pte_t *)__pmd_page(*pmdp) + ((promva >> 13) & 0x3ff);
Linus Torvalds's avatar
Linus Torvalds committed
404 405 406 407 408 409 410 411 412 413 414 415 416
	if (!pte_present(*ptep)) {
		if (error)
			*error = 1;
		return(0);
	}
	if (error) {
		*error = 0;
		return(pte_val(*ptep));
	}
	base = pte_val(*ptep) & _PAGE_PADDR;
	return(base + (promva & (BASE_PAGE_SIZE - 1)));
}

Linus Torvalds's avatar
Linus Torvalds committed
417 418 419 420 421
static void inherit_prom_mappings(void)
{
	struct linux_prom_translation *trans;
	unsigned long phys_page, tte_vaddr, tte_data;
	void (*remap_func)(unsigned long, unsigned long, int);
Linus Torvalds's avatar
Linus Torvalds committed
422
	pmd_t *pmdp;
Linus Torvalds's avatar
Linus Torvalds committed
423 424
	pte_t *ptep;
	int node, n, i, tsz;
Linus Torvalds's avatar
Linus Torvalds committed
425
	extern unsigned int obp_iaddr_patch[2], obp_daddr_patch[2];
Linus Torvalds's avatar
Linus Torvalds committed
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448

	node = prom_finddevice("/virtual-memory");
	n = prom_getproplen(node, "translations");
	if (n == 0 || n == -1) {
		prom_printf("Couldn't get translation property\n");
		prom_halt();
	}
	n += 5 * sizeof(struct linux_prom_translation);
	for (tsz = 1; tsz < n; tsz <<= 1)
		/* empty */;
	trans = __alloc_bootmem(tsz, SMP_CACHE_BYTES, 0UL);
	if (trans == NULL) {
		prom_printf("inherit_prom_mappings: Cannot alloc translations.\n");
		prom_halt();
	}
	memset(trans, 0, tsz);

	if ((n = prom_getproperty(node, "translations", (char *)trans, tsz)) == -1) {
		prom_printf("Couldn't get translation property\n");
		prom_halt();
	}
	n = n / sizeof(*trans);

Linus Torvalds's avatar
Linus Torvalds committed
449 450 451 452 453 454 455
	/*
	 * The obp translations are saved based on 8k pagesize, since obp can use
	 * a mixture of pagesizes. Misses to the 0xf0000000 - 0x100000000, ie obp 
	 * range, are handled in entry.S and do not use the vpte scheme (see rant
	 * in inherit_locked_prom_mappings()).
	 */
#define OBP_PMD_SIZE 2048
Linus Torvalds's avatar
Linus Torvalds committed
456 457
	prompmd = __alloc_bootmem(OBP_PMD_SIZE, OBP_PMD_SIZE, 0UL);
	if (prompmd == NULL)
Linus Torvalds's avatar
Linus Torvalds committed
458
		early_pgtable_allocfail("pmd");
Linus Torvalds's avatar
Linus Torvalds committed
459
	memset(prompmd, 0, OBP_PMD_SIZE);
Linus Torvalds's avatar
Linus Torvalds committed
460 461 462
	for (i = 0; i < n; i++) {
		unsigned long vaddr;

Linus Torvalds's avatar
Linus Torvalds committed
463
		if (trans[i].virt >= LOW_OBP_ADDRESS && trans[i].virt < HI_OBP_ADDRESS) {
Linus Torvalds's avatar
Linus Torvalds committed
464
			for (vaddr = trans[i].virt;
Linus Torvalds's avatar
Linus Torvalds committed
465 466
			     ((vaddr < trans[i].virt + trans[i].size) && 
			     (vaddr < HI_OBP_ADDRESS));
Linus Torvalds's avatar
Linus Torvalds committed
467
			     vaddr += BASE_PAGE_SIZE) {
Linus Torvalds's avatar
Linus Torvalds committed
468 469
				unsigned long val;

Linus Torvalds's avatar
Linus Torvalds committed
470
				pmdp = prompmd + ((vaddr >> 23) & 0x7ff);
Linus Torvalds's avatar
Linus Torvalds committed
471
				if (pmd_none(*pmdp)) {
Linus Torvalds's avatar
Linus Torvalds committed
472 473
					ptep = __alloc_bootmem(BASE_PAGE_SIZE,
							       BASE_PAGE_SIZE,
Linus Torvalds's avatar
Linus Torvalds committed
474 475 476
							       0UL);
					if (ptep == NULL)
						early_pgtable_allocfail("pte");
Linus Torvalds's avatar
Linus Torvalds committed
477
					memset(ptep, 0, BASE_PAGE_SIZE);
Linus Torvalds's avatar
Linus Torvalds committed
478 479
					pmd_set(pmdp, ptep);
				}
480
				ptep = (pte_t *)__pmd_page(*pmdp) +
Linus Torvalds's avatar
Linus Torvalds committed
481
						((vaddr >> 13) & 0x3ff);
Linus Torvalds's avatar
Linus Torvalds committed
482 483 484 485 486 487 488 489

				val = trans[i].data;

				/* Clear diag TTE bits. */
				if (tlb_type == spitfire)
					val &= ~0x0003fe0000000000UL;

				set_pte (ptep, __pte(val | _PAGE_MODIFIED));
Linus Torvalds's avatar
Linus Torvalds committed
490
				trans[i].data += BASE_PAGE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
491 492 493
			}
		}
	}
Linus Torvalds's avatar
Linus Torvalds committed
494
	phys_page = __pa(prompmd);
Linus Torvalds's avatar
Linus Torvalds committed
495 496 497 498 499 500
	obp_iaddr_patch[0] |= (phys_page >> 10);
	obp_iaddr_patch[1] |= (phys_page & 0x3ff);
	flushi((long)&obp_iaddr_patch[0]);
	obp_daddr_patch[0] |= (phys_page >> 10);
	obp_daddr_patch[1] |= (phys_page & 0x3ff);
	flushi((long)&obp_daddr_patch[0]);
Linus Torvalds's avatar
Linus Torvalds committed
501 502 503 504 505 506 507 508 509 510 511

	/* Now fixup OBP's idea about where we really are mapped. */
	prom_printf("Remapping the kernel... ");

	/* Spitfire Errata #32 workaround */
	__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
			     "flush	%%g6"
			     : /* No outputs */
			     : "r" (0),
			     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

Linus Torvalds's avatar
Linus Torvalds committed
512 513 514 515 516 517 518
	switch (tlb_type) {
	default:
	case spitfire:
		phys_page = spitfire_get_dtlb_data(sparc64_highest_locked_tlbent());
		break;

	case cheetah:
Linus Torvalds's avatar
Linus Torvalds committed
519
		phys_page = cheetah_get_litlb_data(sparc64_highest_locked_tlbent());
Linus Torvalds's avatar
Linus Torvalds committed
520 521 522 523
		break;
	};

	phys_page &= _PAGE_PADDR;
Linus Torvalds's avatar
Linus Torvalds committed
524
	phys_page += ((unsigned long)&prom_boot_page -
Linus Torvalds's avatar
Linus Torvalds committed
525
		      (unsigned long)KERNBASE);
Linus Torvalds's avatar
Linus Torvalds committed
526

Linus Torvalds's avatar
Linus Torvalds committed
527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564
	if (tlb_type == spitfire) {
		/* Lock this into i/d tlb entry 59 */
		__asm__ __volatile__(
			"stxa	%%g0, [%2] %3\n\t"
			"stxa	%0, [%1] %4\n\t"
			"membar	#Sync\n\t"
			"flush	%%g6\n\t"
			"stxa	%%g0, [%2] %5\n\t"
			"stxa	%0, [%1] %6\n\t"
			"membar	#Sync\n\t"
			"flush	%%g6"
			: : "r" (phys_page | _PAGE_VALID | _PAGE_SZ8K | _PAGE_CP |
				 _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W),
			"r" (59 << 3), "r" (TLB_TAG_ACCESS),
			"i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS),
			"i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS)
			: "memory");
	} else if (tlb_type == cheetah) {
		/* Lock this into i/d tlb-0 entry 11 */
		__asm__ __volatile__(
			"stxa	%%g0, [%2] %3\n\t"
			"stxa	%0, [%1] %4\n\t"
			"membar	#Sync\n\t"
			"flush	%%g6\n\t"
			"stxa	%%g0, [%2] %5\n\t"
			"stxa	%0, [%1] %6\n\t"
			"membar	#Sync\n\t"
			"flush	%%g6"
			: : "r" (phys_page | _PAGE_VALID | _PAGE_SZ8K | _PAGE_CP |
				 _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W),
			"r" ((0 << 16) | (11 << 3)), "r" (TLB_TAG_ACCESS),
			"i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS),
			"i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS)
			: "memory");
	} else {
		/* Implement me :-) */
		BUG();
	}
Linus Torvalds's avatar
Linus Torvalds committed
565

Linus Torvalds's avatar
Linus Torvalds committed
566
	tte_vaddr = (unsigned long) KERNBASE;
Linus Torvalds's avatar
Linus Torvalds committed
567 568 569 570 571 572 573 574

	/* Spitfire Errata #32 workaround */
	__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
			     "flush	%%g6"
			     : /* No outputs */
			     : "r" (0),
			     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

Linus Torvalds's avatar
Linus Torvalds committed
575 576 577 578 579 580
	if (tlb_type == spitfire)
		tte_data = spitfire_get_dtlb_data(sparc64_highest_locked_tlbent());
	else
		tte_data = cheetah_get_ldtlb_data(sparc64_highest_locked_tlbent());

	kern_locked_tte_data = tte_data;
Linus Torvalds's avatar
Linus Torvalds committed
581 582 583 584 585 586 587 588 589 590 591 592

	remap_func = (void *)  ((unsigned long) &prom_remap -
				(unsigned long) &prom_boot_page);


	/* Spitfire Errata #32 workaround */
	__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
			     "flush	%%g6"
			     : /* No outputs */
			     : "r" (0),
			     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

Linus Torvalds's avatar
Linus Torvalds committed
593 594
	remap_func((tlb_type == spitfire ?
		    (spitfire_get_dtlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR) :
Linus Torvalds's avatar
Linus Torvalds committed
595
		    (cheetah_get_litlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR)),
Linus Torvalds's avatar
Linus Torvalds committed
596
		   (unsigned long) KERNBASE,
Linus Torvalds's avatar
Linus Torvalds committed
597 598
		   prom_get_mmu_ihandle());

Linus Torvalds's avatar
Linus Torvalds committed
599 600 601 602
	if (bigkernel)
		remap_func(((tte_data + 0x400000) & _PAGE_PADDR),
			(unsigned long) KERNBASE + 0x400000, prom_get_mmu_ihandle());

Linus Torvalds's avatar
Linus Torvalds committed
603 604 605 606 607
	/* Flush out that temporary mapping. */
	spitfire_flush_dtlb_nucleus_page(0x0);
	spitfire_flush_itlb_nucleus_page(0x0);

	/* Now lock us back into the TLBs via OBP. */
Linus Torvalds's avatar
Linus Torvalds committed
608 609
	prom_dtlb_load(sparc64_highest_locked_tlbent(), tte_data, tte_vaddr);
	prom_itlb_load(sparc64_highest_locked_tlbent(), tte_data, tte_vaddr);
Linus Torvalds's avatar
Linus Torvalds committed
610 611 612 613 614 615
	if (bigkernel) {
		prom_dtlb_load(sparc64_highest_locked_tlbent()-1, tte_data + 0x400000, 
								tte_vaddr + 0x400000);
		prom_itlb_load(sparc64_highest_locked_tlbent()-1, tte_data + 0x400000, 
								tte_vaddr + 0x400000);
	}
Linus Torvalds's avatar
Linus Torvalds committed
616 617 618 619 620 621 622 623 624 625 626 627 628

	/* Re-read translations property. */
	if ((n = prom_getproperty(node, "translations", (char *)trans, tsz)) == -1) {
		prom_printf("Couldn't get translation property\n");
		prom_halt();
	}
	n = n / sizeof(*trans);

	for (i = 0; i < n; i++) {
		unsigned long vaddr = trans[i].virt;
		unsigned long size = trans[i].size;

		if (vaddr < 0xf0000000UL) {
Linus Torvalds's avatar
Linus Torvalds committed
629
			unsigned long avoid_start = (unsigned long) KERNBASE;
Linus Torvalds's avatar
Linus Torvalds committed
630 631
			unsigned long avoid_end = avoid_start + (4 * 1024 * 1024);

Linus Torvalds's avatar
Linus Torvalds committed
632 633
			if (bigkernel)
				avoid_end += (4 * 1024 * 1024);
Linus Torvalds's avatar
Linus Torvalds committed
634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667
			if (vaddr < avoid_start) {
				unsigned long top = vaddr + size;

				if (top > avoid_start)
					top = avoid_start;
				prom_unmap(top - vaddr, vaddr);
			}
			if ((vaddr + size) > avoid_end) {
				unsigned long bottom = vaddr;

				if (bottom < avoid_end)
					bottom = avoid_end;
				prom_unmap((vaddr + size) - bottom, bottom);
			}
		}
	}

	prom_printf("done.\n");

	register_prom_callbacks();
}

/* The OBP specifications for sun4u mark 0xfffffffc00000000 and
 * upwards as reserved for use by the firmware (I wonder if this
 * will be the same on Cheetah...).  We use this virtual address
 * range for the VPTE table mappings of the nucleus so we need
 * to zap them when we enter the PROM.  -DaveM
 */
static void __flush_nucleus_vptes(void)
{
	unsigned long prom_reserved_base = 0xfffffffc00000000UL;
	int i;

	/* Only DTLB must be checked for VPTE entries. */
Linus Torvalds's avatar
Linus Torvalds committed
668 669 670 671 672 673 674 675 676 677 678 679 680 681
	if (tlb_type == spitfire) {
		for (i = 0; i < 63; i++) {
			unsigned long tag;

			/* Spitfire Errata #32 workaround */
			__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
					     "flush	%%g6"
					     : /* No outputs */
					     : "r" (0),
					     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

			tag = spitfire_get_dtlb_tag(i);
			if (((tag & ~(PAGE_MASK)) == 0) &&
			    ((tag &  (PAGE_MASK)) >= prom_reserved_base)) {
Linus Torvalds's avatar
Linus Torvalds committed
682 683
				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
						     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
684 685 686 687 688 689
						     : /* no outputs */
						     : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
				spitfire_put_dtlb_data(i, 0x0UL);
			}
		}
	} else if (tlb_type == cheetah) {
Linus Torvalds's avatar
Linus Torvalds committed
690
		for (i = 0; i < 512; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
691 692 693 694
			unsigned long tag = cheetah_get_dtlb_tag(i);

			if ((tag & ~PAGE_MASK) == 0 &&
			    (tag & PAGE_MASK) >= prom_reserved_base) {
Linus Torvalds's avatar
Linus Torvalds committed
695 696
				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
						     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
697 698 699 700
						     : /* no outputs */
						     : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
				cheetah_put_dtlb_data(i, 0x0UL);
			}
Linus Torvalds's avatar
Linus Torvalds committed
701
		}
Linus Torvalds's avatar
Linus Torvalds committed
702 703 704
	} else {
		/* Implement me :-) */
		BUG();
Linus Torvalds's avatar
Linus Torvalds committed
705 706 707
	}
}

708
static int prom_ditlb_set;
Linus Torvalds's avatar
Linus Torvalds committed
709 710 711 712 713
struct prom_tlb_entry {
	int		tlb_ent;
	unsigned long	tlb_tag;
	unsigned long	tlb_data;
};
Linus Torvalds's avatar
Linus Torvalds committed
714
struct prom_tlb_entry prom_itlb[16], prom_dtlb[16];
Linus Torvalds's avatar
Linus Torvalds committed
715 716 717 718 719 720 721

void prom_world(int enter)
{
	unsigned long pstate;
	int i;

	if (!enter)
722
		set_fs((mm_segment_t) { get_thread_current_ds() });
Linus Torvalds's avatar
Linus Torvalds committed
723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738

	if (!prom_ditlb_set)
		return;

	/* Make sure the following runs atomically. */
	__asm__ __volatile__("flushw\n\t"
			     "rdpr	%%pstate, %0\n\t"
			     "wrpr	%0, %1, %%pstate"
			     : "=r" (pstate)
			     : "i" (PSTATE_IE));

	if (enter) {
		/* Kick out nucleus VPTEs. */
		__flush_nucleus_vptes();

		/* Install PROM world. */
Linus Torvalds's avatar
Linus Torvalds committed
739
		for (i = 0; i < 16; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
740
			if (prom_dtlb[i].tlb_ent != -1) {
Linus Torvalds's avatar
Linus Torvalds committed
741 742
				__asm__ __volatile__("stxa %0, [%1] %2\n\t"
						     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
743 744
					: : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS),
					"i" (ASI_DMMU));
Linus Torvalds's avatar
Linus Torvalds committed
745 746 747 748 749 750
				if (tlb_type == spitfire)
					spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
							       prom_dtlb[i].tlb_data);
				else if (tlb_type == cheetah)
					cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
							       prom_dtlb[i].tlb_data);
Linus Torvalds's avatar
Linus Torvalds committed
751 752
			}
			if (prom_itlb[i].tlb_ent != -1) {
Linus Torvalds's avatar
Linus Torvalds committed
753 754
				__asm__ __volatile__("stxa %0, [%1] %2\n\t"
						     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
755 756 757 758 759 760 761 762 763
						     : : "r" (prom_itlb[i].tlb_tag),
						     "r" (TLB_TAG_ACCESS),
						     "i" (ASI_IMMU));
				if (tlb_type == spitfire)
					spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
							       prom_itlb[i].tlb_data);
				else if (tlb_type == cheetah)
					cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
							       prom_itlb[i].tlb_data);
Linus Torvalds's avatar
Linus Torvalds committed
764 765 766
			}
		}
	} else {
Linus Torvalds's avatar
Linus Torvalds committed
767
		for (i = 0; i < 16; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
768
			if (prom_dtlb[i].tlb_ent != -1) {
Linus Torvalds's avatar
Linus Torvalds committed
769 770
				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
						     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
771
					: : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
Linus Torvalds's avatar
Linus Torvalds committed
772 773 774 775
				if (tlb_type == spitfire)
					spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent, 0x0UL);
				else
					cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent, 0x0UL);
Linus Torvalds's avatar
Linus Torvalds committed
776 777
			}
			if (prom_itlb[i].tlb_ent != -1) {
Linus Torvalds's avatar
Linus Torvalds committed
778 779
				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
						     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
780 781 782 783 784 785
						     : : "r" (TLB_TAG_ACCESS),
						     "i" (ASI_IMMU));
				if (tlb_type == spitfire)
					spitfire_put_itlb_data(prom_itlb[i].tlb_ent, 0x0UL);
				else
					cheetah_put_litlb_data(prom_itlb[i].tlb_ent, 0x0UL);
Linus Torvalds's avatar
Linus Torvalds committed
786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813
			}
		}
	}
	__asm__ __volatile__("wrpr	%0, 0, %%pstate"
			     : : "r" (pstate));
}

void inherit_locked_prom_mappings(int save_p)
{
	int i;
	int dtlb_seen = 0;
	int itlb_seen = 0;

	/* Fucking losing PROM has more mappings in the TLB, but
	 * it (conveniently) fails to mention any of these in the
	 * translations property.  The only ones that matter are
	 * the locked PROM tlb entries, so we impose the following
	 * irrecovable rule on the PROM, it is allowed 8 locked
	 * entries in the ITLB and 8 in the DTLB.
	 *
	 * Supposedly the upper 16GB of the address space is
	 * reserved for OBP, BUT I WISH THIS WAS DOCUMENTED
	 * SOMEWHERE!!!!!!!!!!!!!!!!!  Furthermore the entire interface
	 * used between the client program and the firmware on sun5
	 * systems to coordinate mmu mappings is also COMPLETELY
	 * UNDOCUMENTED!!!!!! Thanks S(t)un!
	 */
	if (save_p) {
Linus Torvalds's avatar
Linus Torvalds committed
814
		for (i = 0; i < 16; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
815
			prom_itlb[i].tlb_ent = -1;
Linus Torvalds's avatar
Linus Torvalds committed
816
			prom_dtlb[i].tlb_ent = -1;
Linus Torvalds's avatar
Linus Torvalds committed
817 818
		}
	}
Linus Torvalds's avatar
Linus Torvalds committed
819
	if (tlb_type == spitfire) {
Linus Torvalds's avatar
Linus Torvalds committed
820 821
		int high = SPITFIRE_HIGHEST_LOCKED_TLBENT - bigkernel;
		for (i = 0; i < high; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
822
			unsigned long data;
Linus Torvalds's avatar
Linus Torvalds committed
823 824 825 826 827 828 829 830

			/* Spitfire Errata #32 workaround */
			__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
					     "flush	%%g6"
					     : /* No outputs */
					     : "r" (0),
					     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

Linus Torvalds's avatar
Linus Torvalds committed
831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
			data = spitfire_get_dtlb_data(i);
			if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
				unsigned long tag;

				/* Spitfire Errata #32 workaround */
				__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
						     "flush	%%g6"
						     : /* No outputs */
						     : "r" (0),
						     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

				tag = spitfire_get_dtlb_tag(i);
				if (save_p) {
					prom_dtlb[dtlb_seen].tlb_ent = i;
					prom_dtlb[dtlb_seen].tlb_tag = tag;
					prom_dtlb[dtlb_seen].tlb_data = data;
				}
Linus Torvalds's avatar
Linus Torvalds committed
848 849
				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
						     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
850 851
						     : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
				spitfire_put_dtlb_data(i, 0x0UL);
Linus Torvalds's avatar
Linus Torvalds committed
852

Linus Torvalds's avatar
Linus Torvalds committed
853 854 855 856
				dtlb_seen++;
				if (dtlb_seen > 15)
					break;
			}
Linus Torvalds's avatar
Linus Torvalds committed
857
		}
Linus Torvalds's avatar
Linus Torvalds committed
858

Linus Torvalds's avatar
Linus Torvalds committed
859
		for (i = 0; i < high; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
860
			unsigned long data;
Linus Torvalds's avatar
Linus Torvalds committed
861 862 863 864 865 866 867 868

			/* Spitfire Errata #32 workaround */
			__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
					     "flush	%%g6"
					     : /* No outputs */
					     : "r" (0),
					     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

Linus Torvalds's avatar
Linus Torvalds committed
869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885
			data = spitfire_get_itlb_data(i);
			if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
				unsigned long tag;

				/* Spitfire Errata #32 workaround */
				__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
						     "flush	%%g6"
						     : /* No outputs */
						     : "r" (0),
						     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

				tag = spitfire_get_itlb_tag(i);
				if (save_p) {
					prom_itlb[itlb_seen].tlb_ent = i;
					prom_itlb[itlb_seen].tlb_tag = tag;
					prom_itlb[itlb_seen].tlb_data = data;
				}
Linus Torvalds's avatar
Linus Torvalds committed
886 887
				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
						     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
888 889 890 891 892 893
						     : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
				spitfire_put_itlb_data(i, 0x0UL);

				itlb_seen++;
				if (itlb_seen > 15)
					break;
Linus Torvalds's avatar
Linus Torvalds committed
894
			}
Linus Torvalds's avatar
Linus Torvalds committed
895 896
		}
	} else if (tlb_type == cheetah) {
Linus Torvalds's avatar
Linus Torvalds committed
897 898 899
		int high = CHEETAH_HIGHEST_LOCKED_TLBENT - bigkernel;

		for (i = 0; i < high; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
900 901 902 903 904 905 906 907 908 909 910 911
			unsigned long data;

			data = cheetah_get_ldtlb_data(i);
			if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
				unsigned long tag;

				tag = cheetah_get_ldtlb_tag(i);
				if (save_p) {
					prom_dtlb[dtlb_seen].tlb_ent = i;
					prom_dtlb[dtlb_seen].tlb_tag = tag;
					prom_dtlb[dtlb_seen].tlb_data = data;
				}
Linus Torvalds's avatar
Linus Torvalds committed
912 913
				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
						     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
914 915
						     : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
				cheetah_put_ldtlb_data(i, 0x0UL);
Linus Torvalds's avatar
Linus Torvalds committed
916

Linus Torvalds's avatar
Linus Torvalds committed
917 918 919 920 921 922
				dtlb_seen++;
				if (dtlb_seen > 15)
					break;
			}
		}

Linus Torvalds's avatar
Linus Torvalds committed
923
		for (i = 0; i < high; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
924 925 926 927 928 929 930 931 932 933 934 935
			unsigned long data;

			data = cheetah_get_litlb_data(i);
			if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
				unsigned long tag;

				tag = cheetah_get_litlb_tag(i);
				if (save_p) {
					prom_itlb[itlb_seen].tlb_ent = i;
					prom_itlb[itlb_seen].tlb_tag = tag;
					prom_itlb[itlb_seen].tlb_data = data;
				}
Linus Torvalds's avatar
Linus Torvalds committed
936 937
				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
						     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
938 939 940 941 942 943 944
						     : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
				cheetah_put_litlb_data(i, 0x0UL);

				itlb_seen++;
				if (itlb_seen > 15)
					break;
			}
Linus Torvalds's avatar
Linus Torvalds committed
945
		}
Linus Torvalds's avatar
Linus Torvalds committed
946 947 948
	} else {
		/* Implement me :-) */
		BUG();
Linus Torvalds's avatar
Linus Torvalds committed
949 950 951 952 953 954 955 956 957 958
	}
	if (save_p)
		prom_ditlb_set = 1;
}

/* Give PROM back his world, done during reboots... */
void prom_reload_locked(void)
{
	int i;

Linus Torvalds's avatar
Linus Torvalds committed
959
	for (i = 0; i < 16; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
960
		if (prom_dtlb[i].tlb_ent != -1) {
Linus Torvalds's avatar
Linus Torvalds committed
961 962
			__asm__ __volatile__("stxa %0, [%1] %2\n\t"
					     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
963 964
				: : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS),
				"i" (ASI_DMMU));
Linus Torvalds's avatar
Linus Torvalds committed
965 966 967 968 969 970
			if (tlb_type == spitfire)
				spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
						       prom_dtlb[i].tlb_data);
			else if (tlb_type == cheetah)
				cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
						      prom_dtlb[i].tlb_data);
Linus Torvalds's avatar
Linus Torvalds committed
971 972 973
		}

		if (prom_itlb[i].tlb_ent != -1) {
Linus Torvalds's avatar
Linus Torvalds committed
974 975
			__asm__ __volatile__("stxa %0, [%1] %2\n\t"
					     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
976 977 978 979 980 981 982 983 984
					     : : "r" (prom_itlb[i].tlb_tag),
					     "r" (TLB_TAG_ACCESS),
					     "i" (ASI_IMMU));
			if (tlb_type == spitfire)
				spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
						       prom_itlb[i].tlb_data);
			else
				cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
						       prom_itlb[i].tlb_data);
Linus Torvalds's avatar
Linus Torvalds committed
985 986 987 988 989 990 991 992
		}
	}
}

void __flush_dcache_range(unsigned long start, unsigned long end)
{
	unsigned long va;

Linus Torvalds's avatar
Linus Torvalds committed
993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
	if (tlb_type == spitfire) {
		int n = 0;

		for (va = start; va < end; va += 32) {
			spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
			if (++n >= 512)
				break;
		}
	} else {
		start = __pa(start);
		end = __pa(end);
		for (va = start; va < end; va += 32)
			__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
					     "membar #Sync"
					     : /* no outputs */
					     : "r" (va),
					       "i" (ASI_DCACHE_INVALIDATE));
Linus Torvalds's avatar
Linus Torvalds committed
1010 1011 1012 1013 1014
	}
}

void __flush_cache_all(void)
{
Linus Torvalds's avatar
Linus Torvalds committed
1015 1016 1017
	/* Cheetah should be fine here too. */
	if (tlb_type == spitfire) {
		unsigned long va;
Linus Torvalds's avatar
Linus Torvalds committed
1018

Linus Torvalds's avatar
Linus Torvalds committed
1019 1020 1021 1022 1023
		flushw_all();
		for (va =  0; va < (PAGE_SIZE << 1); va += 32)
			spitfire_put_icache_tag(va, 0x0);
		__asm__ __volatile__("flush %g6");
	}
Linus Torvalds's avatar
Linus Torvalds committed
1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036
}

/* If not locked, zap it. */
void __flush_tlb_all(void)
{
	unsigned long pstate;
	int i;

	__asm__ __volatile__("flushw\n\t"
			     "rdpr	%%pstate, %0\n\t"
			     "wrpr	%0, %1, %%pstate"
			     : "=r" (pstate)
			     : "i" (PSTATE_IE));
Linus Torvalds's avatar
Linus Torvalds committed
1037 1038 1039 1040 1041 1042 1043 1044
	if (tlb_type == spitfire) {
		for (i = 0; i < 64; i++) {
			/* Spitfire Errata #32 workaround */
			__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
					     "flush	%%g6"
					     : /* No outputs */
					     : "r" (0),
					     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
Linus Torvalds's avatar
Linus Torvalds committed
1045

Linus Torvalds's avatar
Linus Torvalds committed
1046
			if (!(spitfire_get_dtlb_data(i) & _PAGE_L)) {
Linus Torvalds's avatar
Linus Torvalds committed
1047 1048
				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
						     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061
						     : /* no outputs */
						     : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
				spitfire_put_dtlb_data(i, 0x0UL);
			}

			/* Spitfire Errata #32 workaround */
			__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
					     "flush	%%g6"
					     : /* No outputs */
					     : "r" (0),
					     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

			if (!(spitfire_get_itlb_data(i) & _PAGE_L)) {
Linus Torvalds's avatar
Linus Torvalds committed
1062 1063
				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
						     "membar #Sync"
Linus Torvalds's avatar
Linus Torvalds committed
1064 1065 1066 1067
						     : /* no outputs */
						     : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
				spitfire_put_itlb_data(i, 0x0UL);
			}
Linus Torvalds's avatar
Linus Torvalds committed
1068
		}
Linus Torvalds's avatar
Linus Torvalds committed
1069 1070 1071
	} else if (tlb_type == cheetah) {
		cheetah_flush_dtlb_all();
		cheetah_flush_itlb_all();
Linus Torvalds's avatar
Linus Torvalds committed
1072 1073 1074 1075 1076 1077
	}
	__asm__ __volatile__("wrpr	%0, 0, %%pstate"
			     : : "r" (pstate));
}

/* Caller does TLB context flushing on local CPU if necessary.
Linus Torvalds's avatar
Linus Torvalds committed
1078
 * The caller also ensures that CTX_VALID(mm->context) is false.
Linus Torvalds's avatar
Linus Torvalds committed
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107
 *
 * We must be careful about boundary cases so that we never
 * let the user have CTX 0 (nucleus) or we ever use a CTX
 * version of zero (and thus NO_CONTEXT would not be caught
 * by version mis-match tests in mmu_context.h).
 */
void get_new_mmu_context(struct mm_struct *mm)
{
	unsigned long ctx, new_ctx;
	
	spin_lock(&ctx_alloc_lock);
	ctx = CTX_HWBITS(tlb_context_cache + 1);
	new_ctx = find_next_zero_bit(mmu_context_bmap, 1UL << CTX_VERSION_SHIFT, ctx);
	if (new_ctx >= (1UL << CTX_VERSION_SHIFT)) {
		new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
		if (new_ctx >= ctx) {
			int i;
			new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +
				CTX_FIRST_VERSION;
			if (new_ctx == 1)
				new_ctx = CTX_FIRST_VERSION;

			/* Don't call memset, for 16 entries that's just
			 * plain silly...
			 */
			mmu_context_bmap[0] = 3;
			mmu_context_bmap[1] = 0;
			mmu_context_bmap[2] = 0;
			mmu_context_bmap[3] = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1108
			for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {
Linus Torvalds's avatar
Linus Torvalds committed
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
				mmu_context_bmap[i + 0] = 0;
				mmu_context_bmap[i + 1] = 0;
				mmu_context_bmap[i + 2] = 0;
				mmu_context_bmap[i + 3] = 0;
			}
			goto out;
		}
	}
	mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
	new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
out:
	tlb_context_cache = new_ctx;
	spin_unlock(&ctx_alloc_lock);

	mm->context = new_ctx;
}

#ifndef CONFIG_SMP
struct pgtable_cache_struct pgt_quicklists;
#endif

Linus Torvalds's avatar
Linus Torvalds committed
1130 1131 1132 1133 1134 1135 1136 1137 1138 1139
/* OK, we have to color these pages. The page tables are accessed
 * by non-Dcache enabled mapping in the VPTE area by the dtlb_backend.S
 * code, as well as by PAGE_OFFSET range direct-mapped addresses by 
 * other parts of the kernel. By coloring, we make sure that the tlbmiss 
 * fast handlers do not get data from old/garbage dcache lines that 
 * correspond to an old/stale virtual address (user/kernel) that 
 * previously mapped the pagetable page while accessing vpte range 
 * addresses. The idea is that if the vpte color and PAGE_OFFSET range 
 * color is the same, then when the kernel initializes the pagetable 
 * using the later address range, accesses with the first address
Linus Torvalds's avatar
Linus Torvalds committed
1140
 * range will see the newly initialized data rather than the garbage.
Linus Torvalds's avatar
Linus Torvalds committed
1141
 */
Linus Torvalds's avatar
Linus Torvalds committed
1142 1143 1144 1145 1146
#if (L1DCACHE_SIZE > PAGE_SIZE)			/* is there D$ aliasing problem */
#define DC_ALIAS_SHIFT	1
#else
#define DC_ALIAS_SHIFT	0
#endif
1147
pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
Linus Torvalds's avatar
Linus Torvalds committed
1148
{
1149 1150 1151 1152 1153 1154 1155 1156 1157
	struct page *page;
	unsigned long color;

	{
		pte_t *ptep = pte_alloc_one_fast(mm, address);

		if (ptep)
			return ptep;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1158

1159 1160
	color = VPTE_COLOR(address);
	page = alloc_pages(GFP_KERNEL, DC_ALIAS_SHIFT);
Linus Torvalds's avatar
Linus Torvalds committed
1161 1162 1163 1164 1165
	if (page) {
		unsigned long *to_free;
		unsigned long paddr;
		pte_t *pte;

Linus Torvalds's avatar
Linus Torvalds committed
1166
#if (L1DCACHE_SIZE > PAGE_SIZE)			/* is there D$ aliasing problem */
Linus Torvalds's avatar
Linus Torvalds committed
1167
		set_page_count((page + 1), 1);
Linus Torvalds's avatar
Linus Torvalds committed
1168
#endif
Linus Torvalds's avatar
Linus Torvalds committed
1169
		paddr = (unsigned long) page_address(page);
Linus Torvalds's avatar
Linus Torvalds committed
1170
		memset((char *)paddr, 0, (PAGE_SIZE << DC_ALIAS_SHIFT));
Linus Torvalds's avatar
Linus Torvalds committed
1171 1172 1173 1174 1175 1176 1177 1178 1179

		if (!color) {
			pte = (pte_t *) paddr;
			to_free = (unsigned long *) (paddr + PAGE_SIZE);
		} else {
			pte = (pte_t *) (paddr + PAGE_SIZE);
			to_free = (unsigned long *) paddr;
		}

Linus Torvalds's avatar
Linus Torvalds committed
1180
#if (L1DCACHE_SIZE > PAGE_SIZE)			/* is there D$ aliasing problem */
Linus Torvalds's avatar
Linus Torvalds committed
1181
		/* Now free the other one up, adjust cache size. */
1182
		preempt_disable();
Linus Torvalds's avatar
Linus Torvalds committed
1183 1184 1185
		*to_free = (unsigned long) pte_quicklist[color ^ 0x1];
		pte_quicklist[color ^ 0x1] = to_free;
		pgtable_cache_size++;
1186
		preempt_enable();
Linus Torvalds's avatar
Linus Torvalds committed
1187
#endif
Linus Torvalds's avatar
Linus Torvalds committed
1188

Linus Torvalds's avatar
Linus Torvalds committed
1189
		return pte;
Linus Torvalds's avatar
Linus Torvalds committed
1190 1191 1192 1193 1194 1195 1196 1197
	}
	return NULL;
}

void sparc_ultra_dump_itlb(void)
{
        int slot;

Linus Torvalds's avatar
Linus Torvalds committed
1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
	if (tlb_type == spitfire) {
		printk ("Contents of itlb: ");
		for (slot = 0; slot < 14; slot++) printk ("    ");
		printk ("%2x:%016lx,%016lx\n",
			0,
			spitfire_get_itlb_tag(0), spitfire_get_itlb_data(0));
		for (slot = 1; slot < 64; slot+=3) {
			printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n", 
				slot,
				spitfire_get_itlb_tag(slot), spitfire_get_itlb_data(slot),
				slot+1,
				spitfire_get_itlb_tag(slot+1), spitfire_get_itlb_data(slot+1),
				slot+2,
				spitfire_get_itlb_tag(slot+2), spitfire_get_itlb_data(slot+2));
		}
	} else if (tlb_type == cheetah) {
		printk ("Contents of itlb0:\n");
		for (slot = 0; slot < 16; slot+=2) {
			printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
				slot,
				cheetah_get_litlb_tag(slot), cheetah_get_litlb_data(slot),
				slot+1,
				cheetah_get_litlb_tag(slot+1), cheetah_get_litlb_data(slot+1));
		}
		printk ("Contents of itlb2:\n");
		for (slot = 0; slot < 128; slot+=2) {
			printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
				slot,
				cheetah_get_itlb_tag(slot), cheetah_get_itlb_data(slot),
				slot+1,
				cheetah_get_itlb_tag(slot+1), cheetah_get_itlb_data(slot+1));
		}
	}
Linus Torvalds's avatar
Linus Torvalds committed
1231 1232 1233 1234 1235 1236
}

void sparc_ultra_dump_dtlb(void)
{
        int slot;

Linus Torvalds's avatar
Linus Torvalds committed
1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268
	if (tlb_type == spitfire) {
		printk ("Contents of dtlb: ");
		for (slot = 0; slot < 14; slot++) printk ("    ");
		printk ("%2x:%016lx,%016lx\n", 0,
			spitfire_get_dtlb_tag(0), spitfire_get_dtlb_data(0));
		for (slot = 1; slot < 64; slot+=3) {
			printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n", 
				slot,
				spitfire_get_dtlb_tag(slot), spitfire_get_dtlb_data(slot),
				slot+1,
				spitfire_get_dtlb_tag(slot+1), spitfire_get_dtlb_data(slot+1),
				slot+2,
				spitfire_get_dtlb_tag(slot+2), spitfire_get_dtlb_data(slot+2));
		}
	} else if (tlb_type == cheetah) {
		printk ("Contents of dtlb0:\n");
		for (slot = 0; slot < 16; slot+=2) {
			printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
				slot,
				cheetah_get_ldtlb_tag(slot), cheetah_get_ldtlb_data(slot),
				slot+1,
				cheetah_get_ldtlb_tag(slot+1), cheetah_get_ldtlb_data(slot+1));
		}
		printk ("Contents of dtlb2:\n");
		for (slot = 0; slot < 512; slot+=2) {
			printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
				slot,
				cheetah_get_dtlb_tag(slot), cheetah_get_dtlb_data(slot),
				slot+1,
				cheetah_get_dtlb_tag(slot+1), cheetah_get_dtlb_data(slot+1));
		}
	}
Linus Torvalds's avatar
Linus Torvalds committed
1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
}

extern unsigned long cmdline_memory_size;

unsigned long __init bootmem_init(unsigned long *pages_avail)
{
	unsigned long bootmap_size, start_pfn, end_pfn;
	unsigned long end_of_phys_memory = 0UL;
	unsigned long bootmap_pfn, bytes_avail, size;
	int i;

	bytes_avail = 0UL;
	for (i = 0; sp_banks[i].num_bytes != 0; i++) {
		end_of_phys_memory = sp_banks[i].base_addr +
			sp_banks[i].num_bytes;
		bytes_avail += sp_banks[i].num_bytes;
		if (cmdline_memory_size) {
			if (bytes_avail > cmdline_memory_size) {
				unsigned long slack = bytes_avail - cmdline_memory_size;

				bytes_avail -= slack;
				end_of_phys_memory -= slack;

				sp_banks[i].num_bytes -= slack;
				if (sp_banks[i].num_bytes == 0) {
					sp_banks[i].base_addr = 0xdeadbeef;
				} else {
					sp_banks[i+1].num_bytes = 0;
					sp_banks[i+1].base_addr = 0xdeadbeef;
				}
				break;
			}
		}
	}

	*pages_avail = bytes_avail >> PAGE_SHIFT;

	/* Start with page aligned address of last symbol in kernel
	 * image.  The kernel is hard mapped below PAGE_OFFSET in a
	 * 4MB locked TLB translation.
	 */
	start_pfn  = PAGE_ALIGN((unsigned long) &_end) -
Linus Torvalds's avatar
Linus Torvalds committed
1311
		((unsigned long) KERNBASE);
Linus Torvalds's avatar
Linus Torvalds committed
1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383

	/* Adjust up to the physical address where the kernel begins. */
	start_pfn += phys_base;

	/* Now shift down to get the real physical page frame number. */
	start_pfn >>= PAGE_SHIFT;
	
	bootmap_pfn = start_pfn;

	end_pfn = end_of_phys_memory >> PAGE_SHIFT;

#ifdef CONFIG_BLK_DEV_INITRD
	/* Now have to check initial ramdisk, so that bootmap does not overwrite it */
	if (sparc_ramdisk_image) {
		if (sparc_ramdisk_image >= (unsigned long)&_end - 2 * PAGE_SIZE)
			sparc_ramdisk_image -= KERNBASE;
		initrd_start = sparc_ramdisk_image + phys_base;
		initrd_end = initrd_start + sparc_ramdisk_size;
		if (initrd_end > end_of_phys_memory) {
			printk(KERN_CRIT "initrd extends beyond end of memory "
		                 	 "(0x%016lx > 0x%016lx)\ndisabling initrd\n",
			       initrd_end, end_of_phys_memory);
			initrd_start = 0;
		}
		if (initrd_start) {
			if (initrd_start >= (start_pfn << PAGE_SHIFT) &&
			    initrd_start < (start_pfn << PAGE_SHIFT) + 2 * PAGE_SIZE)
				bootmap_pfn = PAGE_ALIGN (initrd_end) >> PAGE_SHIFT;
		}
	}
#endif	
	/* Initialize the boot-time allocator. */
	bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap_pfn, phys_base>>PAGE_SHIFT, end_pfn);

	/* Now register the available physical memory with the
	 * allocator.
	 */
	for (i = 0; sp_banks[i].num_bytes != 0; i++)
		free_bootmem(sp_banks[i].base_addr,
			     sp_banks[i].num_bytes);

#ifdef CONFIG_BLK_DEV_INITRD
	if (initrd_start) {
		size = initrd_end - initrd_start;

		/* Resert the initrd image area. */
		reserve_bootmem(initrd_start, size);
		*pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;

		initrd_start += PAGE_OFFSET;
		initrd_end += PAGE_OFFSET;
	}
#endif
	/* Reserve the kernel text/data/bss. */
	size = (start_pfn << PAGE_SHIFT) - phys_base;
	reserve_bootmem(phys_base, size);
	*pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;

	/* Reserve the bootmem map.   We do not account for it
	 * in pages_avail because we will release that memory
	 * in free_all_bootmem.
	 */
	size = bootmap_size;
	reserve_bootmem((bootmap_pfn << PAGE_SHIFT), size);
	*pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;

	return end_pfn;
}

/* paging_init() sets up the page tables */

extern void sun_serial_setup(void);
Linus Torvalds's avatar
Linus Torvalds committed
1384
extern void cheetah_ecache_flush_init(void);
Linus Torvalds's avatar
Linus Torvalds committed
1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395

static unsigned long last_valid_pfn;

void __init paging_init(void)
{
	extern pmd_t swapper_pmd_dir[1024];
	extern unsigned int sparc64_vpte_patchme1[1];
	extern unsigned int sparc64_vpte_patchme2[1];
	unsigned long alias_base = phys_base + PAGE_OFFSET;
	unsigned long second_alias_page = 0;
	unsigned long pt, flags, end_pfn, pages_avail;
Linus Torvalds's avatar
Linus Torvalds committed
1396
	unsigned long shift = alias_base - ((unsigned long)KERNBASE);
Linus Torvalds's avatar
Linus Torvalds committed
1397
	unsigned long real_end;
Linus Torvalds's avatar
Linus Torvalds committed
1398 1399

	set_bit(0, mmu_context_bmap);
Linus Torvalds's avatar
Linus Torvalds committed
1400 1401

	real_end = (unsigned long)&_end;
Linus Torvalds's avatar
Linus Torvalds committed
1402 1403
	if ((real_end > ((unsigned long)KERNBASE + 0x400000)))
		bigkernel = 1;
Linus Torvalds's avatar
Linus Torvalds committed
1404 1405 1406 1407 1408
#ifdef CONFIG_BLK_DEV_INITRD
	if (sparc_ramdisk_image)
		real_end = (PAGE_ALIGN(real_end) + PAGE_ALIGN(sparc_ramdisk_size));
#endif

Linus Torvalds's avatar
Linus Torvalds committed
1409 1410 1411 1412 1413 1414 1415
	/* We assume physical memory starts at some 4mb multiple,
	 * if this were not true we wouldn't boot up to this point
	 * anyways.
	 */
	pt  = phys_base | _PAGE_VALID | _PAGE_SZ4MB;
	pt |= _PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W;
	__save_and_cli(flags);
Linus Torvalds's avatar
Linus Torvalds committed
1416
	if (tlb_type == spitfire) {
Linus Torvalds's avatar
Linus Torvalds committed
1417 1418 1419 1420 1421 1422 1423 1424
		__asm__ __volatile__(
	"	stxa	%1, [%0] %3\n"
	"	stxa	%2, [%5] %4\n"
	"	membar	#Sync\n"
	"	flush	%%g6\n"
	"	nop\n"
	"	nop\n"
	"	nop\n"
Linus Torvalds's avatar
Linus Torvalds committed
1425
		: /* No outputs */
Linus Torvalds's avatar
Linus Torvalds committed
1426 1427
		: "r" (TLB_TAG_ACCESS), "r" (alias_base), "r" (pt),
		  "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (61 << 3)
Linus Torvalds's avatar
Linus Torvalds committed
1428
		: "memory");
Linus Torvalds's avatar
Linus Torvalds committed
1429
		if (real_end >= KERNBASE + 0x340000) {
Linus Torvalds's avatar
Linus Torvalds committed
1430
			second_alias_page = alias_base + 0x400000;
Linus Torvalds's avatar
Linus Torvalds committed
1431 1432 1433 1434 1435 1436 1437 1438
			__asm__ __volatile__(
		"	stxa	%1, [%0] %3\n"
		"	stxa	%2, [%5] %4\n"
		"	membar	#Sync\n"
		"	flush	%%g6\n"
		"	nop\n"
		"	nop\n"
		"	nop\n"
Linus Torvalds's avatar
Linus Torvalds committed
1439 1440 1441 1442 1443 1444
			: /* No outputs */
			: "r" (TLB_TAG_ACCESS), "r" (second_alias_page), "r" (pt + 0x400000),
			  "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (60 << 3)
			: "memory");
		}
	} else if (tlb_type == cheetah) {
Linus Torvalds's avatar
Linus Torvalds committed
1445 1446 1447 1448 1449 1450 1451 1452
		__asm__ __volatile__(
	"	stxa	%1, [%0] %3\n"
	"	stxa	%2, [%5] %4\n"
	"	membar	#Sync\n"
	"	flush	%%g6\n"
	"	nop\n"
	"	nop\n"
	"	nop\n"
Linus Torvalds's avatar
Linus Torvalds committed
1453 1454 1455 1456
		: /* No outputs */
		: "r" (TLB_TAG_ACCESS), "r" (alias_base), "r" (pt),
		  "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" ((0<<16) | (13<<3))
		: "memory");
Linus Torvalds's avatar
Linus Torvalds committed
1457
		if (real_end >= KERNBASE + 0x340000) {
Linus Torvalds's avatar
Linus Torvalds committed
1458
			second_alias_page = alias_base + 0x400000;
Linus Torvalds's avatar
Linus Torvalds committed
1459 1460 1461 1462 1463 1464 1465 1466
			__asm__ __volatile__(
		"	stxa	%1, [%0] %3\n"
		"	stxa	%2, [%5] %4\n"
		"	membar	#Sync\n"
		"	flush	%%g6\n"
		"	nop\n"
		"	nop\n"
		"	nop\n"
Linus Torvalds's avatar
Linus Torvalds committed
1467 1468 1469 1470 1471
			: /* No outputs */
			: "r" (TLB_TAG_ACCESS), "r" (second_alias_page), "r" (pt + 0x400000),
			  "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" ((0<<16) | (12<<3))
			: "memory");
		}
Linus Torvalds's avatar
Linus Torvalds committed
1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
	}
	__restore_flags(flags);
	
	/* Now set kernel pgd to upper alias so physical page computations
	 * work.
	 */
	init_mm.pgd += ((shift) / (sizeof(pgd_t)));
	
	memset(swapper_pmd_dir, 0, sizeof(swapper_pmd_dir));

	/* Now can init the kernel/bad page tables. */
	pgd_set(&swapper_pg_dir[0], swapper_pmd_dir + (shift / sizeof(pgd_t)));
	
	sparc64_vpte_patchme1[0] |= (pgd_val(init_mm.pgd[0]) >> 10);
	sparc64_vpte_patchme2[0] |= (pgd_val(init_mm.pgd[0]) & 0x3ff);
	flushi((long)&sparc64_vpte_patchme1[0]);
	
	/* Setup bootmem... */
	pages_avail = 0;
	last_valid_pfn = end_pfn = bootmem_init(&pages_avail);

#ifdef CONFIG_SUN_SERIAL
	/* This does not logically belong here, but we need to
	 * call it at the moment we are able to use the bootmem
	 * allocator.
	 */
	sun_serial_setup();
#endif

	/* Inherit non-locked OBP mappings. */
	inherit_prom_mappings();
	
	/* Ok, we can use our TLB miss and window trap handlers safely.
	 * We need to do a quick peek here to see if we are on StarFire
	 * or not, so setup_tba can setup the IRQ globals correctly (it
	 * needs to get the hard smp processor id correctly).
	 */
	{
		extern void setup_tba(int);
		setup_tba(this_is_starfire);
	}

	inherit_locked_prom_mappings(1);
	
	/* We only created DTLB mapping of this stuff. */
	spitfire_flush_dtlb_nucleus_page(alias_base);
	if (second_alias_page)
		spitfire_flush_dtlb_nucleus_page(second_alias_page);

Linus Torvalds's avatar
Linus Torvalds committed
1521
	__flush_tlb_all();
Linus Torvalds's avatar
Linus Torvalds committed
1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693

	{
		unsigned long zones_size[MAX_NR_ZONES];
		unsigned long zholes_size[MAX_NR_ZONES];
		unsigned long npages;
		int znum;

		for (znum = 0; znum < MAX_NR_ZONES; znum++)
			zones_size[znum] = zholes_size[znum] = 0;

		npages = end_pfn - (phys_base >> PAGE_SHIFT);
		zones_size[ZONE_DMA] = npages;
		zholes_size[ZONE_DMA] = npages - pages_avail;

		free_area_init_node(0, NULL, NULL, zones_size,
				    phys_base, zholes_size);
	}

	device_scan();
}

/* Ok, it seems that the prom can allocate some more memory chunks
 * as a side effect of some prom calls we perform during the
 * boot sequence.  My most likely theory is that it is from the
 * prom_set_traptable() call, and OBP is allocating a scratchpad
 * for saving client program register state etc.
 */
void __init sort_memlist(struct linux_mlist_p1275 *thislist)
{
	int swapi = 0;
	int i, mitr;
	unsigned long tmpaddr, tmpsize;
	unsigned long lowest;

	for (i = 0; thislist[i].theres_more != 0; i++) {
		lowest = thislist[i].start_adr;
		for (mitr = i+1; thislist[mitr-1].theres_more != 0; mitr++)
			if (thislist[mitr].start_adr < lowest) {
				lowest = thislist[mitr].start_adr;
				swapi = mitr;
			}
		if (lowest == thislist[i].start_adr)
			continue;
		tmpaddr = thislist[swapi].start_adr;
		tmpsize = thislist[swapi].num_bytes;
		for (mitr = swapi; mitr > i; mitr--) {
			thislist[mitr].start_adr = thislist[mitr-1].start_adr;
			thislist[mitr].num_bytes = thislist[mitr-1].num_bytes;
		}
		thislist[i].start_adr = tmpaddr;
		thislist[i].num_bytes = tmpsize;
	}
}

void __init rescan_sp_banks(void)
{
	struct linux_prom64_registers memlist[64];
	struct linux_mlist_p1275 avail[64], *mlist;
	unsigned long bytes, base_paddr;
	int num_regs, node = prom_finddevice("/memory");
	int i;

	num_regs = prom_getproperty(node, "available",
				    (char *) memlist, sizeof(memlist));
	num_regs = (num_regs / sizeof(struct linux_prom64_registers));
	for (i = 0; i < num_regs; i++) {
		avail[i].start_adr = memlist[i].phys_addr;
		avail[i].num_bytes = memlist[i].reg_size;
		avail[i].theres_more = &avail[i + 1];
	}
	avail[i - 1].theres_more = NULL;
	sort_memlist(avail);

	mlist = &avail[0];
	i = 0;
	bytes = mlist->num_bytes;
	base_paddr = mlist->start_adr;
  
	sp_banks[0].base_addr = base_paddr;
	sp_banks[0].num_bytes = bytes;

	while (mlist->theres_more != NULL){
		i++;
		mlist = mlist->theres_more;
		bytes = mlist->num_bytes;
		if (i >= SPARC_PHYS_BANKS-1) {
			printk ("The machine has more banks than "
				"this kernel can support\n"
				"Increase the SPARC_PHYS_BANKS "
				"setting (currently %d)\n",
				SPARC_PHYS_BANKS);
			i = SPARC_PHYS_BANKS-1;
			break;
		}
    
		sp_banks[i].base_addr = mlist->start_adr;
		sp_banks[i].num_bytes = mlist->num_bytes;
	}

	i++;
	sp_banks[i].base_addr = 0xdeadbeefbeefdeadUL;
	sp_banks[i].num_bytes = 0;

	for (i = 0; sp_banks[i].num_bytes != 0; i++)
		sp_banks[i].num_bytes &= PAGE_MASK;
}

static void __init taint_real_pages(void)
{
	struct sparc_phys_banks saved_sp_banks[SPARC_PHYS_BANKS];
	int i;

	for (i = 0; i < SPARC_PHYS_BANKS; i++) {
		saved_sp_banks[i].base_addr =
			sp_banks[i].base_addr;
		saved_sp_banks[i].num_bytes =
			sp_banks[i].num_bytes;
	}

	rescan_sp_banks();

	/* Find changes discovered in the sp_bank rescan and
	 * reserve the lost portions in the bootmem maps.
	 */
	for (i = 0; saved_sp_banks[i].num_bytes; i++) {
		unsigned long old_start, old_end;

		old_start = saved_sp_banks[i].base_addr;
		old_end = old_start +
			saved_sp_banks[i].num_bytes;
		while (old_start < old_end) {
			int n;

			for (n = 0; sp_banks[n].num_bytes; n++) {
				unsigned long new_start, new_end;

				new_start = sp_banks[n].base_addr;
				new_end = new_start + sp_banks[n].num_bytes;

				if (new_start <= old_start &&
				    new_end >= (old_start + PAGE_SIZE)) {
					set_bit (old_start >> 22,
						 sparc64_valid_addr_bitmap);
					goto do_next_page;
				}
			}
			reserve_bootmem(old_start, PAGE_SIZE);

		do_next_page:
			old_start += PAGE_SIZE;
		}
	}
}

void __init mem_init(void)
{
	unsigned long codepages, datapages, initpages;
	unsigned long addr, last;
	int i;

	i = last_valid_pfn >> ((22 - PAGE_SHIFT) + 6);
	i += 1;
	sparc64_valid_addr_bitmap = (unsigned long *)
		__alloc_bootmem(i << 3, SMP_CACHE_BYTES, 0UL);
	if (sparc64_valid_addr_bitmap == NULL) {
		prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
		prom_halt();
	}
	memset(sparc64_valid_addr_bitmap, 0, i << 3);

	addr = PAGE_OFFSET + phys_base;
	last = PAGE_ALIGN((unsigned long)&_end) -
Linus Torvalds's avatar
Linus Torvalds committed
1694
		((unsigned long) KERNBASE);
Linus Torvalds's avatar
Linus Torvalds committed
1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705
	last += PAGE_OFFSET + phys_base;
	while (addr < last) {
		set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap);
		addr += PAGE_SIZE;
	}

	taint_real_pages();

	max_mapnr = last_valid_pfn - (phys_base >> PAGE_SHIFT);
	high_memory = __va(last_valid_pfn << PAGE_SHIFT);

Linus Torvalds's avatar
Linus Torvalds committed
1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719
	num_physpages = free_all_bootmem() - 1;

	/*
	 * Set up the zero page, mark it reserved, so that page count
	 * is not manipulated when freeing the page from user ptes.
	 */
	mem_map_zero = _alloc_pages(GFP_KERNEL, 0);
	if (mem_map_zero == NULL) {
		prom_printf("paging_init: Cannot alloc zero page.\n");
		prom_halt();
	}
	SetPageReserved(mem_map_zero);
	clear_page(page_address(mem_map_zero));

Linus Torvalds's avatar
Linus Torvalds committed
1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732
	codepages = (((unsigned long) &etext) - ((unsigned long)&_start));
	codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT;
	datapages = (((unsigned long) &edata) - ((unsigned long)&etext));
	datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT;
	initpages = (((unsigned long) &__init_end) - ((unsigned long) &__init_begin));
	initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT;

#ifndef CONFIG_SMP
	{
		/* Put empty_pg_dir on pgd_quicklist */
		extern pgd_t empty_pg_dir[1024];
		unsigned long addr = (unsigned long)empty_pg_dir;
		unsigned long alias_base = phys_base + PAGE_OFFSET -
Linus Torvalds's avatar
Linus Torvalds committed
1733
			(long)(KERNBASE);
Linus Torvalds's avatar
Linus Torvalds committed
1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747
		
		memset(empty_pg_dir, 0, sizeof(empty_pg_dir));
		addr += alias_base;
		free_pgd_fast((pgd_t *)addr);
		num_physpages++;
	}
#endif

	printk("Memory: %uk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
	       nr_free_pages() << (PAGE_SHIFT-10),
	       codepages << (PAGE_SHIFT-10),
	       datapages << (PAGE_SHIFT-10), 
	       initpages << (PAGE_SHIFT-10), 
	       PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT));
Linus Torvalds's avatar
Linus Torvalds committed
1748 1749 1750

	if (tlb_type == cheetah)
		cheetah_ecache_flush_init();
Linus Torvalds's avatar
Linus Torvalds committed
1751 1752 1753 1754
}

void free_initmem (void)
{
Linus Torvalds's avatar
Linus Torvalds committed
1755
	unsigned long addr, initend;
Linus Torvalds's avatar
Linus Torvalds committed
1756

Linus Torvalds's avatar
Linus Torvalds committed
1757 1758 1759 1760 1761 1762
	/*
	 * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
	 */
	addr = PAGE_ALIGN((unsigned long)(&__init_begin));
	initend = (unsigned long)(&__init_end) & PAGE_MASK;
	for (; addr < initend; addr += PAGE_SIZE) {
Linus Torvalds's avatar
Linus Torvalds committed
1763 1764 1765 1766 1767
		unsigned long page;
		struct page *p;

		page = (addr +
			((unsigned long) __va(phys_base)) -
Linus Torvalds's avatar
Linus Torvalds committed
1768
			((unsigned long) KERNBASE));
Linus Torvalds's avatar
Linus Torvalds committed
1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792
		p = virt_to_page(page);

		ClearPageReserved(p);
		set_page_count(p, 1);
		__free_page(p);
		num_physpages++;
	}
}

#ifdef CONFIG_BLK_DEV_INITRD
void free_initrd_mem(unsigned long start, unsigned long end)
{
	if (start < end)
		printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
	for (; start < end; start += PAGE_SIZE) {
		struct page *p = virt_to_page(start);

		ClearPageReserved(p);
		set_page_count(p, 1);
		__free_page(p);
		num_physpages++;
	}
}
#endif