[LV-kern-commit] CVS: kernel-2.4/arch/sparc64/mm Makefile,1.1.1.1,1.2 extable.c,1.1.1.1,1.2 fault.c,1.1.1.1,1.2 generic.c,1.1.1.1,1.2 init.c,1.1.1.2,1.2 modutil.c,1.1.1.2,1.2 ultra.S,1.1.1.1,1.2

[Andy P. <at...@us...>]

Update of /cvsroot/linux-vax/kernel-2.4/arch/sparc64/mm
In directory usw-pr-cvs1:/tmp/cvs-serv32333/sparc64/mm

Modified Files:
	Makefile extable.c fault.c generic.c init.c modutil.c ultra.S 
Log Message:
synch 2.4.15 commit 46


Index: Makefile
===================================================================
RCS file: /cvsroot/linux-vax/kernel-2.4/arch/sparc64/mm/Makefile,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2

Index: extable.c
===================================================================
RCS file: /cvsroot/linux-vax/kernel-2.4/arch/sparc64/mm/extable.c,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- extable.c	14 Jan 2001 19:45:13 -0000	1.1.1.1
+++ extable.c	10 Apr 2002 15:21:24 -0000	1.2
@@ -43,27 +43,32 @@
         return 0;
 }
 
+extern spinlock_t modlist_lock;
+
 unsigned long
 search_exception_table(unsigned long addr, unsigned long *g2)
 {
-	unsigned long ret;
+	unsigned long ret = 0, flags;
 
 #ifndef CONFIG_MODULES
 	/* There is only the kernel to search.  */
 	ret = search_one_table(__start___ex_table,
 			       __stop___ex_table-1, addr, g2);
-	if (ret) return ret;
+	return ret;
 #else
 	/* The kernel is the last "module" -- no need to treat it special.  */
 	struct module *mp;
+
+	spin_lock_irqsave(&modlist_lock, flags);
 	for (mp = module_list; mp != NULL; mp = mp->next) {
-		if (mp->ex_table_start == NULL)
+		if (mp->ex_table_start == NULL || !(mp->flags & (MOD_RUNNING | MOD_INITIALIZING)))
 			continue;
 		ret = search_one_table(mp->ex_table_start,
 				       mp->ex_table_end-1, addr, g2);
-		if (ret) return ret;
+		if (ret)
+			break;
 	}
+	spin_unlock_irqrestore(&modlist_lock, flags);
+	return ret;
 #endif
-
-	return 0;
 }

Index: fault.c
===================================================================
RCS file: /cvsroot/linux-vax/kernel-2.4/arch/sparc64/mm/fault.c,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- fault.c	14 Jan 2001 19:45:14 -0000	1.1.1.1
+++ fault.c	10 Apr 2002 15:21:24 -0000	1.2
@@ -22,11 +22,58 @@
 #include <asm/openprom.h>
 #include <asm/oplib.h>
 #include <asm/uaccess.h>
+#include <asm/asi.h>
+#include <asm/lsu.h>
 
 #define ELEMENTS(arr) (sizeof (arr)/sizeof (arr[0]))
 
 extern struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS];
 
+/*
+ * To debug kernel during syscall entry.
+ */
+void syscall_trace_entry(struct pt_regs *regs)
+{
+	printk("scall entry: %s[%d]/cpu%d: %d\n", current->comm, current->pid, smp_processor_id(), (int) regs->u_regs[UREG_G1]);
+}
+
+/*
+ * To debug kernel during syscall exit.
+ */
+void syscall_trace_exit(struct pt_regs *regs)
+{
+	printk("scall exit: %s[%d]/cpu%d: %d\n", current->comm, current->pid, smp_processor_id(), (int) regs->u_regs[UREG_G1]);
+}
+
+/*
+ * To debug kernel to catch accesses to certain virtual/physical addresses.
+ * Mode = 0 selects physical watchpoints, mode = 1 selects virtual watchpoints.
+ * flags = VM_READ watches memread accesses, flags = VM_WRITE watches memwrite accesses.
+ * Caller passes in a 64bit aligned addr, with mask set to the bytes that need to be
+ * watched. This is only useful on a single cpu machine for now. After the watchpoint
+ * is detected, the process causing it will be killed, thus preventing an infinite loop.
+ */
+void set_brkpt(unsigned long addr, unsigned char mask, int flags, int mode)
+{
+	unsigned long lsubits = LSU_CONTROL_IC|LSU_CONTROL_DC|LSU_CONTROL_IM|LSU_CONTROL_DM;
+
+	__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
+			     "membar	#Sync"
+			     : /* no outputs */
+			     : "r" (addr), "r" (mode ? VIRT_WATCHPOINT : PHYS_WATCHPOINT),
+			       "i" (ASI_DMMU));
+	lsubits |= ((unsigned long)mask << (mode ? 25 : 33));
+	if (flags & VM_READ)
+		lsubits |= (mode ? LSU_CONTROL_VR : LSU_CONTROL_PR);
+	if (flags & VM_WRITE)
+		lsubits |= (mode ? LSU_CONTROL_VW : LSU_CONTROL_PW);
+	__asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
+			     "membar #Sync"
+			     : /* no outputs */
+			     : "r" (lsubits), "i" (ASI_LSU_CONTROL)
+			     : "memory");
+}
+
 /* Nice, simple, prom library does all the sweating for us. ;) */
 unsigned long __init prom_probe_memory (void)
 {
@@ -92,6 +139,13 @@
 	die_if_kernel("Oops", regs);
 }
 
+/*
+ * We now make sure that mmap_sem is held in all paths that call 
+ * this. Additionally, to prevent kswapd from ripping ptes from
+ * under us, raise interrupts around the time that we look at the
+ * pte, kswapd will have to wait to get his smp ipi response from
+ * us. This saves us having to get page_table_lock.
+ */
 static unsigned int get_user_insn(unsigned long tpc)
 {
 	pgd_t *pgdp = pgd_offset(current->mm, tpc);
@@ -99,13 +153,17 @@
 	pte_t *ptep, pte;
 	unsigned long pa;
 	u32 insn = 0;
+	unsigned long pstate;
 
 	if (pgd_none(*pgdp))
-		goto out;
+		goto outret;
 	pmdp = pmd_offset(pgdp, tpc);
 	if (pmd_none(*pmdp))
-		goto out;
+		goto outret;
 	ptep = pte_offset(pmdp, tpc);
+	__asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
+	__asm__ __volatile__("wrpr %0, %1, %%pstate"
+				: : "r" (pstate), "i" (PSTATE_IE));
 	pte = *ptep;
 	if (!pte_present(pte))
 		goto out;
@@ -119,6 +177,8 @@
 			     : "r" (pa), "i" (ASI_PHYS_USE_EC));
 
 out:
+	__asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate));
+outret:
 	return insn;
 }
 
@@ -137,21 +197,28 @@
 extern int handle_ldf_stq(u32, struct pt_regs *);
 extern int handle_ld_nf(u32, struct pt_regs *);
 
-static void do_kernel_fault(struct pt_regs *regs, int si_code, int fault_code,
-			    unsigned int insn, unsigned long address)
+static inline unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn)
 {
-	unsigned long g2;
-	unsigned char asi = ASI_P;
-		
 	if (!insn) {
+		if (!regs->tpc || (regs->tpc & 0x3))
+			return 0;
 		if (regs->tstate & TSTATE_PRIV) {
-			if (!regs->tpc || (regs->tpc & 0x3))
-				goto cannot_handle;
 			insn = *(unsigned int *)regs->tpc;
 		} else {
 			insn = get_user_insn(regs->tpc);
 		}
 	}
+	return insn;
+}
+
+static void do_kernel_fault(struct pt_regs *regs, int si_code, int fault_code,
+			    unsigned int insn, unsigned long address)
+{
+	unsigned long g2;
+	unsigned char asi = ASI_P;
+ 
+	if ((!insn) && (regs->tstate & TSTATE_PRIV))
+		goto cannot_handle;
 
 	/* If user insn could be read (thus insn is zero), that
 	 * is fine.  We will just gun down the process with a signal
@@ -232,9 +299,14 @@
 	 * context, we must not take the fault..
 	 */
 	if (in_interrupt() || !mm)
-		goto handle_kernel_fault;
+		goto intr_or_no_mm;
 
-	down(&mm->mmap_sem);
+	if ((current->thread.flags & SPARC_FLAG_32BIT) != 0) {
+		regs->tpc &= 0xffffffff;
+		address &= 0xffffffff;
+	}
+
+	down_read(&mm->mmap_sem);
 	vma = find_vma(mm, address);
 	if (!vma)
 		goto bad_area;
@@ -250,16 +322,9 @@
 	if (((fault_code &
 	      (FAULT_CODE_DTLB | FAULT_CODE_WRITE | FAULT_CODE_WINFIXUP)) == FAULT_CODE_DTLB) &&
 	    (vma->vm_flags & VM_WRITE) != 0) {
-		unsigned long tpc = regs->tpc;
-
-		if (tpc & 0x3)
+		insn = get_fault_insn(regs, 0);
+		if (!insn)
 			goto continue_fault;
-
-		if (regs->tstate & TSTATE_PRIV)
-			insn = *(unsigned int *)tpc;
-		else
-			insn = get_user_insn(tpc);
-
 		if ((insn & 0xc0200000) == 0xc0200000 &&
 		    (insn & 0x1780000) != 0x1680000) {
 			/* Don't bother updating thread struct value,
@@ -286,7 +351,12 @@
 	if (fault_code & FAULT_CODE_WRITE) {
 		if (!(vma->vm_flags & VM_WRITE))
 			goto bad_area;
-		if ((vma->vm_flags & VM_EXEC) != 0 &&
+
+		/* Spitfire has an icache which does not snoop
+		 * processor stores.  Later processors do...
+		 */
+		if (tlb_type == spitfire &&
+		    (vma->vm_flags & VM_EXEC) != 0 &&
 		    vma->vm_file != NULL)
 			current->thread.use_blkcommit = 1;
 	} else {
@@ -308,7 +378,7 @@
 		goto out_of_memory;
 	}
 
-	up(&mm->mmap_sem);
+	up_read(&mm->mmap_sem);
 	goto fault_done;
 
 	/*
@@ -316,7 +386,8 @@
 	 * Fix it, but check if it's kernel or user first..
 	 */
 bad_area:
-	up(&mm->mmap_sem);
+	insn = get_fault_insn(regs, insn);
+	up_read(&mm->mmap_sem);
 
 handle_kernel_fault:
 	do_kernel_fault(regs, si_code, fault_code, insn, address);
@@ -328,14 +399,20 @@
  * us unable to handle the page fault gracefully.
  */
 out_of_memory:
-	up(&mm->mmap_sem);
+	insn = get_fault_insn(regs, insn);
+	up_read(&mm->mmap_sem);
 	printk("VM: killing process %s\n", current->comm);
 	if (!(regs->tstate & TSTATE_PRIV))
 		do_exit(SIGKILL);
 	goto handle_kernel_fault;
 
+intr_or_no_mm:
+	insn = get_fault_insn(regs, 0);
+	goto handle_kernel_fault;
+
 do_sigbus:
-	up(&mm->mmap_sem);
+	insn = get_fault_insn(regs, insn);
+	up_read(&mm->mmap_sem);
 
 	/*
 	 * Send a sigbus, regardless of whether we were in kernel

Index: generic.c
===================================================================
RCS file: /cvsroot/linux-vax/kernel-2.4/arch/sparc64/mm/generic.c,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- generic.c	14 Jan 2001 19:45:15 -0000	1.1.1.1
+++ generic.c	10 Apr 2002 15:21:24 -0000	1.2
@@ -8,6 +8,7 @@
 #include <linux/kernel.h>
 #include <linux/mm.h>
 #include <linux/swap.h>
+#include <linux/pagemap.h>
 
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
@@ -21,11 +22,7 @@
 		struct page *ptpage = pte_page(page);
 		if ((!VALID_PAGE(ptpage)) || PageReserved(ptpage))
 			return;
-		/* 
-		 * free_page() used to be able to clear swap cache
-		 * entries.  We may now have to do it manually.  
-		 */
-		free_page_and_swap_cache(ptpage);
+		page_cache_release(ptpage);
 		return;
 	}
 	swap_free(pte_to_swp_entry(page));
@@ -104,12 +101,10 @@
 		end = PGDIR_SIZE;
 	offset -= address;
 	do {
-		pte_t * pte = pte_alloc(pmd, address);
+		pte_t * pte = pte_alloc(current->mm, pmd, address);
 		if (!pte)
 			return -ENOMEM;
-		spin_lock(&current->mm->page_table_lock);
 		io_remap_pte_range(pte, address, end - address, address + offset, prot, space);
-		spin_unlock(&current->mm->page_table_lock);
 		address = (address + PMD_SIZE) & PMD_MASK;
 		pmd++;
 	} while (address < end);
@@ -122,13 +117,16 @@
 	pgd_t * dir;
 	unsigned long beg = from;
 	unsigned long end = from + size;
+	struct mm_struct *mm = current->mm;
 
 	prot = __pgprot(pg_iobits);
 	offset -= from;
-	dir = pgd_offset(current->mm, from);
-	flush_cache_range(current->mm, beg, end);
+	dir = pgd_offset(mm, from);
+	flush_cache_range(mm, beg, end);
+
+	spin_lock(&mm->page_table_lock);
 	while (from < end) {
-		pmd_t *pmd = pmd_alloc(dir, from);
+		pmd_t *pmd = pmd_alloc(current->mm, dir, from);
 		error = -ENOMEM;
 		if (!pmd)
 			break;
@@ -138,6 +136,8 @@
 		from = (from + PGDIR_SIZE) & PGDIR_MASK;
 		dir++;
 	}
+	spin_unlock(&mm->page_table_lock);
+
 	flush_tlb_range(current->mm, beg, end);
 	return error;
 }

Index: init.c
===================================================================
RCS file: /cvsroot/linux-vax/kernel-2.4/arch/sparc64/mm/init.c,v
retrieving revision 1.1.1.2
retrieving revision 1.2
diff -u -r1.1.1.2 -r1.2
--- init.c	25 Feb 2001 23:15:20 -0000	1.1.1.2
+++ init.c	10 Apr 2002 15:21:24 -0000	1.2
@@ -16,6 +16,9 @@
 #include <linux/blk.h>
 #include <linux/swap.h>
 #include <linux/swapctl.h>
+#include <linux/pagemap.h>
+#include <linux/fs.h>
+#include <linux/seq_file.h>
 
 #include <asm/head.h>
 #include <asm/system.h>
@@ -27,9 +30,12 @@
 #include <asm/io.h>
[...1406 lines suppressed...]
+	unsigned long addr, initend;
 
-	addr = (unsigned long)(&__init_begin);
-	for (; addr < (unsigned long)(&__init_end); addr += PAGE_SIZE) {
+	/*
+	 * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
+	 */
+	addr = PAGE_ALIGN((unsigned long)(&__init_begin));
+	initend = (unsigned long)(&__init_end) & PAGE_MASK;
+	for (; addr < initend; addr += PAGE_SIZE) {
 		unsigned long page;
 		struct page *p;
 
 		page = (addr +
 			((unsigned long) __va(phys_base)) -
-			((unsigned long) &empty_zero_page));
+			((unsigned long) KERNBASE));
 		p = virt_to_page(page);
 
 		ClearPageReserved(p);

Index: modutil.c
===================================================================
RCS file: /cvsroot/linux-vax/kernel-2.4/arch/sparc64/mm/modutil.c,v
retrieving revision 1.1.1.2
retrieving revision 1.2
diff -u -r1.1.1.2 -r1.2
--- modutil.c	25 Feb 2001 23:15:20 -0000	1.1.1.2
+++ modutil.c	10 Apr 2002 15:21:24 -0000	1.2
@@ -10,7 +10,10 @@
 
 #include <asm/uaccess.h>
 #include <asm/system.h>
-#include <asm/vaddrs.h>
+
+#define  MODULES_VADDR	0x0000000001000000ULL  /* Where to map modules */
+#define  MODULES_LEN	0x000000007f000000ULL
+#define  MODULES_END	0x0000000080000000ULL
 
 static struct vm_struct * modvmlist = NULL;
 
@@ -59,7 +62,7 @@
 	*p = area;
 
 	if (vmalloc_area_pages(VMALLOC_VMADDR(addr), size, GFP_KERNEL, PAGE_KERNEL)) {
-		vfree(addr);
+		module_unmap(addr);
 		return NULL;
 	}
 	return addr;

Index: ultra.S
===================================================================
RCS file: /cvsroot/linux-vax/kernel-2.4/arch/sparc64/mm/ultra.S,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- ultra.S	14 Jan 2001 19:45:13 -0000	1.1.1.1
+++ ultra.S	10 Apr 2002 15:21:24 -0000	1.2
@@ -9,6 +9,23 @@
 #include <asm/pgtable.h>
 #include <asm/page.h>
 #include <asm/spitfire.h>
+#include <asm/mmu_context.h>
+
+	/* Basically, all this madness has to do with the
+	 * fact that Cheetah does not support IMMU flushes
+	 * out of the secondary context.  Someone needs to
+	 * throw a south lake birthday party for the folks
+	 * in Microelectronics who refused to fix this shit.
+	 */
+#define BRANCH_IF_CHEETAH(tmp1, tmp2, label)		\
+	rdpr	%ver, %tmp1;				\
+	sethi	%hi(0x003e0014), %tmp2;			\
+	srlx	%tmp1, 32, %tmp1;			\
+	or	%tmp2, %lo(0x003e0014), %tmp2;		\
+	cmp	%tmp1, %tmp2;				\
+	be,pn	%icc, label;				\
+	 nop;						\
+	nop;
 
 	/* This file is meant to be read efficiently by the CPU, not humans.
 	 * Staraj sie tego nikomu nie pierdolnac...
@@ -16,38 +33,78 @@
 	.text
 	.align		32
 	.globl		__flush_tlb_page, __flush_tlb_mm, __flush_tlb_range
-__flush_tlb_page: /* %o0=(ctx & 0x3ff), %o1=page&PAGE_MASK, %o2=SECONDARY_CONTEXT */
-/*IC1*/	ldxa		[%o2] ASI_DMMU, %g2
+__flush_tlb_page: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=page&PAGE_MASK, %o2=SECONDARY_CONTEXT */
+/*IC1*/	BRANCH_IF_CHEETAH(g2, g3, __cheetah_flush_tlb_page)
+__spitfire_flush_tlb_page:
+/*IC2*/	ldxa		[%o2] ASI_DMMU, %g2
 	cmp		%g2, %o0
-	bne,pn		%icc, __flush_tlb_page_slow
+	bne,pn		%icc, __spitfire_flush_tlb_page_slow
 	 or		%o1, 0x10, %g3
 	stxa		%g0, [%g3] ASI_DMMU_DEMAP
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
 	retl
 	 flush		%g6
-__flush_tlb_mm: /* %o0=(ctx & 0x3ff), %o1=SECONDARY_CONTEXT */
-/*IC2*/	ldxa		[%o1] ASI_DMMU, %g2
+__cheetah_flush_tlb_page:
+/*IC3*/	rdpr		%pstate, %g5
+	andn		%g5, PSTATE_IE, %g2
+	wrpr		%g2, 0x0, %pstate
+	wrpr		%g0, 1, %tl
+	mov		PRIMARY_CONTEXT, %o2
+	ldxa		[%o2] ASI_DMMU, %g2
+	stxa		%o0, [%o2] ASI_DMMU
+	stxa		%g0, [%o1] ASI_DMMU_DEMAP
+/*IC4*/	stxa		%g0, [%o1] ASI_IMMU_DEMAP
+	stxa		%g2, [%o2] ASI_DMMU
+	flush		%g6
+	wrpr		%g0, 0, %tl
+	retl
+	 wrpr		%g5, 0x0, %pstate
+	nop
+	nop
+__flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
+/*IC5*/	BRANCH_IF_CHEETAH(g2, g3, __cheetah_flush_tlb_mm)
+__spitfire_flush_tlb_mm:
+/*IC6*/	ldxa		[%o1] ASI_DMMU, %g2
 	cmp		%g2, %o0
-	bne,pn		%icc, __flush_tlb_mm_slow
+	bne,pn		%icc, __spitfire_flush_tlb_mm_slow
 	 mov		0x50, %g3
 	stxa		%g0, [%g3] ASI_DMMU_DEMAP
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
 	retl
 	 flush		%g6
-__flush_tlb_range: /* %o0=(ctx&0x3ff), %o1=start&PAGE_MASK, %o2=SECONDARY_CONTEXT,
+__cheetah_flush_tlb_mm:
+/*IC7*/	rdpr		%pstate, %g5
+	andn		%g5, PSTATE_IE, %g2
+	wrpr		%g2, 0x0, %pstate
+	wrpr		%g0, 1, %tl
+	mov		PRIMARY_CONTEXT, %o2
+	mov		0x40, %g3
+	ldxa		[%o2] ASI_DMMU, %g2
+	stxa		%o0, [%o2] ASI_DMMU
+/*IC8*/	stxa		%g0, [%g3] ASI_DMMU_DEMAP
+	stxa		%g0, [%g3] ASI_IMMU_DEMAP
+	stxa		%g2, [%o2] ASI_DMMU
+	flush		%g6
+	wrpr		%g0, 0, %tl
+	retl
+	 wrpr		%g5, 0x0, %pstate
+	nop
+__flush_tlb_range: /* %o0=(ctx&TAG_CONTEXT_BITS), %o1=start&PAGE_MASK, %o2=SECONDARY_CONTEXT,
 		    * %o3=end&PAGE_MASK, %o4=PAGE_SIZE, %o5=(end - start)
 		    */
+/*IC9*/	BRANCH_IF_CHEETAH(g2, g3, __cheetah_flush_tlb_range)
+__spitfire_flush_tlb_range:
 #define TLB_MAGIC	207 /* Students, do you know how I calculated this?  -DaveM */
-/*IC3*/	cmp		%o5, %o4
+/*IC10*/cmp		%o5, %o4
 	bleu,pt		%xcc, __flush_tlb_page
-	 srlx		%o5, 13, %g5
+	 srlx		%o5, PAGE_SHIFT, %g5
 	cmp		%g5, TLB_MAGIC
-	bgeu,pn		%icc, __flush_tlb_range_constant_time
+	bgeu,pn		%icc, __spitfire_flush_tlb_range_constant_time
 	 or		%o1, 0x10, %g5
 	ldxa		[%o2] ASI_DMMU, %g2
 	cmp		%g2, %o0
-__flush_tlb_range_page_by_page:
-/*IC4*/	bne,pn		%icc, __flush_tlb_range_pbp_slow
+__spitfire_flush_tlb_range_page_by_page:
+/*IC11*/bne,pn		%icc, __spitfire_flush_tlb_range_pbp_slow
 	 sub		%o5, %o4, %o5
 1:	stxa		%g0, [%g5 + %o5] ASI_DMMU_DEMAP
 	stxa		%g0, [%g5 + %o5] ASI_IMMU_DEMAP
@@ -55,10 +112,11 @@
 	 sub		%o5, %o4, %o5
 	retl
 	 flush		%g6
-__flush_tlb_range_constant_time: /* %o0=ctx, %o1=start, %o3=end */
-/*IC5*/	rdpr		%pstate, %g1
+__spitfire_flush_tlb_range_constant_time: /* %o0=ctx, %o1=start, %o3=end */
+/*IC12*/rdpr		%pstate, %g1
 	wrpr		%g1, PSTATE_IE, %pstate
 	mov		TLB_TAG_ACCESS, %g3
+	/* XXX Spitfire dependency... */
 	mov		(62 << 3), %g2
 
 	/* Spitfire Errata #32 workaround. */
@@ -67,18 +125,18 @@
 	flush		%g6
 
 1:	ldxa		[%g2] ASI_ITLB_TAG_READ, %o4
-	and		%o4, 0x3ff, %o5
+	and		%o4, TAG_CONTEXT_BITS, %o5
 	cmp		%o5, %o0
 	bne,pt		%icc, 2f
-/*IC6*/	 andn		%o4, 0x3ff, %o4
+/*IC13*/ andn		%o4, TAG_CONTEXT_BITS, %o4
 	cmp		%o4, %o1
 	blu,pt		%xcc, 2f
 	 cmp		%o4, %o3
 	blu,pn		%xcc, 4f
 2:	 ldxa		[%g2] ASI_DTLB_TAG_READ, %o4
-	and		%o4, 0x3ff, %o5
+	and		%o4, TAG_CONTEXT_BITS, %o5
 	cmp		%o5, %o0
-/*IC7*/	andn		%o4, 0x3ff, %o4
+/*IC14*/andn		%o4, TAG_CONTEXT_BITS, %o4
 	bne,pt		%icc, 3f
 	 cmp		%o4, %o1
 	blu,pt		%xcc, 3f
@@ -86,7 +144,7 @@
 	blu,pn		%xcc, 5f
 	 nop
 3:	brnz,pt		%g2, 1b
-/*IC8*/	 sub		%g2, (1 << 3), %g2
+/*IC15*/ sub		%g2, (1 << 3), %g2
 	retl
 	 wrpr		%g1, 0x0, %pstate
 4:	stxa		%g0, [%g3] ASI_IMMU
@@ -102,7 +160,7 @@
 	 nop
 
 5:	stxa		%g0, [%g3] ASI_DMMU
-/*IC9*/	stxa		%g0, [%g2] ASI_DTLB_DATA_ACCESS
+/*IC16*/stxa		%g0, [%g2] ASI_DTLB_DATA_ACCESS
 	flush		%g6
 
 	/* Spitfire Errata #32 workaround. */
@@ -114,46 +172,81 @@
 	 nop
 
 	.align		32
-__flush_tlb_mm_slow:
-/*IC10*/rdpr		%pstate, %g1
+__cheetah_flush_tlb_range:
+	cmp		%o5, %o4
+	bleu,pt		%xcc, __cheetah_flush_tlb_page
+	 nop
+/*IC17*/rdpr		%pstate, %g5
+	andn		%g5, PSTATE_IE, %g2
+	wrpr		%g2, 0x0, %pstate
+	wrpr		%g0, 1, %tl
+	mov		PRIMARY_CONTEXT, %o2
+	sub		%o5, %o4, %o5
+	ldxa		[%o2] ASI_DMMU, %g2
+	stxa		%o0, [%o2] ASI_DMMU
+
+/*IC18*/
+1:	stxa		%g0, [%o1 + %o5] ASI_DMMU_DEMAP
+	stxa		%g0, [%o1 + %o5] ASI_IMMU_DEMAP
+	membar		#Sync
+	brnz,pt		%o5, 1b
+	 sub		%o5, %o4, %o5
+
+	stxa		%g2, [%o2] ASI_DMMU
+	flush		%g6
+	wrpr		%g0, 0, %tl
+	retl
+/*IC19*/ wrpr		%g5, 0x0, %pstate
+
+__spitfire_flush_tlb_mm_slow:
+	rdpr		%pstate, %g1
 	wrpr		%g1, PSTATE_IE, %pstate
 	stxa		%o0, [%o1] ASI_DMMU
 	stxa		%g0, [%g3] ASI_DMMU_DEMAP
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
 	flush		%g6
 	stxa		%g2, [%o1] ASI_DMMU
-	flush		%g6
-/*IC11*/retl
+/*IC18*/flush		%g6
+	retl
 	 wrpr		%g1, 0, %pstate
 
-	.align		32
-__flush_tlb_page_slow:
-/*IC12*/rdpr		%pstate, %g1
+__spitfire_flush_tlb_page_slow:
+	rdpr		%pstate, %g1
 	wrpr		%g1, PSTATE_IE, %pstate
 	stxa		%o0, [%o2] ASI_DMMU
 	stxa		%g0, [%g3] ASI_DMMU_DEMAP
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
-	flush		%g6
+/*IC20*/flush		%g6
 	stxa		%g2, [%o2] ASI_DMMU
 	flush		%g6
-/*IC13*/retl
+	retl
 	 wrpr		%g1, 0, %pstate
 
-	.align		32
-__flush_tlb_range_pbp_slow:
-/*IC13*/rdpr		%pstate, %g1
+__spitfire_flush_tlb_range_pbp_slow:
+	rdpr		%pstate, %g1
 	wrpr		%g1, PSTATE_IE, %pstate
 	stxa		%o0, [%o2] ASI_DMMU
+/*IC21*/
 2:	stxa		%g0, [%g5 + %o5] ASI_DMMU_DEMAP
 	stxa		%g0, [%g5 + %o5] ASI_IMMU_DEMAP
 	brnz,pt		%o5, 2b
 	 sub		%o5, %o4, %o5
 	flush		%g6
-/*IC14*/stxa		%g2, [%o2] ASI_DMMU
+	stxa		%g2, [%o2] ASI_DMMU
 	flush		%g6
 	retl
-	 wrpr		%g1, 0x0, %pstate
+/*IC22*/ wrpr		%g1, 0x0, %pstate
 
+/*
+ * The following code flushes one page_size worth.
+ */
+#if (PAGE_SHIFT == 13)
+#define ITAG_MASK 0xfe
+#elif (PAGE_SHIFT == 16)
+#define ITAG_MASK 0x7fe
+#else
+#error unsupported PAGE_SIZE
+#endif
 	.align		32
 	.globl		__flush_icache_page
 __flush_icache_page:	/* %o0 = phys_page */
@@ -167,7 +260,7 @@
 	or		%o0, %g1, %o0		! VALID+phys-addr comparitor
 
 	sllx		%g2, 1, %g2
-	andn		%g2, 0xfe, %g2		! IC_tag mask
+	andn		%g2, ITAG_MASK, %g2	! IC_tag mask
 	nop
 	nop
 	nop
@@ -210,6 +303,37 @@
 	.globl		__flush_dcache_page
 __flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
 	sub		%o0, %g4, %o0
+	
+	rdpr		%ver, %g1
+	sethi		%hi(0x003e0014), %g2
+	srlx		%g1, 32, %g1
+	or		%g2, %lo(0x003e0014), %g2
+	cmp		%g1, %g2
+	bne,pt		%icc, flush_dcpage_spitfire
+	 nop
+
+flush_dcpage_cheetah:
+	sethi		%hi(PAGE_SIZE), %o4
+1:	subcc		%o4, (1 << 5), %o4
+	stxa		%g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
+	bne,pt		%icc, 1b
+	 nop
+	membar		#Sync
+	/* I-cache flush never needed on Cheetah, see callers. */
+	retl
+	 nop
+
+#if (PAGE_SHIFT == 13)
+#define DTAG_MASK 0x3
+#elif (PAGE_SHIFT == 16)
+#define DTAG_MASK 0x1f
+#elif (PAGE_SHIFT == 19)
+#define DTAG_MASK 0xff
+#elif (PAGE_SHIFT == 22)
+#define DTAG_MASK 0x3ff
+#endif
+
+flush_dcpage_spitfire:
 	clr		%o4
 	srlx		%o0, 11, %o0
 	sethi		%hi(1 << 14), %o2
@@ -219,19 +343,19 @@
 	add		%o4, (1 << 5), %o4		! IEU0
 	ldxa		[%o4] ASI_DCACHE_TAG, %g2	! LSU	Group	o3 available
 	add		%o4, (1 << 5), %o4		! IEU0
-	andn		%o3, 0x3, %o3			! IEU1
+	andn		%o3, DTAG_MASK, %o3		! IEU1
 	ldxa		[%o4] ASI_DCACHE_TAG, %g3	! LSU	Group
 	add		%o4, (1 << 5), %o4		! IEU0
-	andn		%g1, 0x3, %g1			! IEU1
+	andn		%g1, DTAG_MASK, %g1		! IEU1
 	cmp		%o0, %o3			! IEU1	Group
 	be,a,pn		%xcc, dflush1			! CTI
 	 sub		%o4, (4 << 5), %o4		! IEU0	(Group)
 	cmp		%o0, %g1			! IEU1	Group
-	andn		%g2, 0x3, %g2			! IEU0
+	andn		%g2, DTAG_MASK, %g2		! IEU0
 	be,a,pn		%xcc, dflush2			! CTI
 	 sub		%o4, (3 << 5), %o4		! IEU0	(Group)
 	cmp		%o0, %g2			! IEU1	Group
-	andn		%g3, 0x3, %g3			! IEU0
+	andn		%g3, DTAG_MASK, %g3		! IEU0
 	be,a,pn		%xcc, dflush3			! CTI
 	 sub		%o4, (2 << 5), %o4		! IEU0	(Group)
 	cmp		%o0, %g3			! IEU1	Group
@@ -284,14 +408,14 @@
 	.globl		__update_mmu_cache
 __update_mmu_cache:	/* %o0=vma, %o1=address, %o2=pte */
 	ldub		[%g6 + AOFF_task_thread + AOFF_thread_fault_code], %o3
-	srlx		%o1, 13, %o1
+	srlx		%o1, PAGE_SHIFT, %o1
 	ldx		[%o0 + 0x0], %o4		/* XXX vma->vm_mm */
 	brz,pn		%o3, 1f
-	 sllx		%o1, 13, %o0
+	 sllx		%o1, PAGE_SHIFT, %o0
 	ldx		[%o4 + AOFF_mm_context], %o5
 	andcc		%o3, FAULT_CODE_DTLB, %g0
 	mov		%o2, %o1
-	and		%o5, 0x3ff, %o5
+	and		%o5, TAG_CONTEXT_BITS, %o5
 	bne,pt		%xcc, __prefill_dtlb
 	 or		%o0, %o5, %o0
 	ba,a,pt		%xcc, __prefill_itlb
@@ -317,18 +441,18 @@
 	.align		32
 	.globl		xcall_flush_tlb_page, xcall_flush_tlb_mm, xcall_flush_tlb_range
 xcall_flush_tlb_page:
-	mov		SECONDARY_CONTEXT, %g2
-	or		%g1, 0x10, %g4
+	mov		PRIMARY_CONTEXT, %g2
 	ldxa		[%g2] ASI_DMMU, %g3
 	stxa		%g5, [%g2] ASI_DMMU
-	stxa		%g0, [%g4] ASI_DMMU_DEMAP
-	stxa		%g0, [%g4] ASI_IMMU_DEMAP
+	stxa		%g0, [%g1] ASI_DMMU_DEMAP
+	stxa		%g0, [%g1] ASI_IMMU_DEMAP
 	stxa		%g3, [%g2] ASI_DMMU
 	retry
+	nop
 
 xcall_flush_tlb_mm:
-	mov		SECONDARY_CONTEXT, %g2
-	mov		0x50, %g4
+	mov		PRIMARY_CONTEXT, %g2
+	mov		0x40, %g4
 	ldxa		[%g2] ASI_DMMU, %g3
 	stxa		%g5, [%g2] ASI_DMMU
 	stxa		%g0, [%g4] ASI_DMMU_DEMAP
@@ -337,26 +461,27 @@
 	retry
 
 xcall_flush_tlb_range:
-	sethi		%hi(8192 - 1), %g2
-	or		%g2, %lo(8192 - 1), %g2
+	sethi		%hi(PAGE_SIZE - 1), %g2
+	or		%g2, %lo(PAGE_SIZE - 1), %g2
 	andn		%g1, %g2, %g1
 	andn		%g7, %g2, %g7
 	sub		%g7, %g1, %g3
 	add		%g2, 1, %g2
-	orcc		%g1, 0x10, %g1
-	srlx		%g3, 13, %g4
-
+	srlx		%g3, PAGE_SHIFT, %g4
 	cmp		%g4, 96
+
 	bgu,pn		%icc, xcall_flush_tlb_mm
-	 mov		SECONDARY_CONTEXT, %g4
+	 mov		PRIMARY_CONTEXT, %g4
 	ldxa		[%g4] ASI_DMMU, %g7
 	sub		%g3, %g2, %g3
 	stxa		%g5, [%g4] ASI_DMMU
 	nop
 	nop
+	nop
 
 1:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
 	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
+	membar		#Sync
 	brnz,pt		%g3, 1b
 	 sub		%g3, %g2, %g3
 	stxa		%g7, [%g4] ASI_DMMU
@@ -378,6 +503,53 @@
 	b,pt		%xcc, rtrap
 	 clr		%l6
 
+	.align		32
+	.globl		xcall_flush_dcache_page_cheetah
+xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
+	sethi		%hi(PAGE_SIZE), %g3
+1:	subcc		%g3, (1 << 5), %g3
+	stxa		%g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
+	bne,pt		%icc, 1b
+	 nop
+	membar		#Sync
+	retry
+	nop
+
+	.globl		xcall_flush_dcache_page_spitfire
+xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
+				     %g7 == kernel page virtual address
+				     %g5 == (page->mapping != NULL)  */
+#if (L1DCACHE_SIZE > PAGE_SIZE)
+	srlx		%g1, (13 - 2), %g1	! Form tag comparitor
+	sethi		%hi(L1DCACHE_SIZE), %g3	! D$ size == 16K
+	sub		%g3, (1 << 5), %g3	! D$ linesize == 32
+1:	ldxa		[%g3] ASI_DCACHE_TAG, %g2
+	andcc		%g2, 0x3, %g0
+	be,pn		%xcc, 2f
+	 andn		%g2, 0x3, %g2
+	cmp		%g2, %g1
+
+	bne,pt		%xcc, 2f
+	 nop
+	stxa		%g0, [%g3] ASI_DCACHE_TAG
+	membar		#Sync
+2:	cmp		%g3, 0
+	bne,pt		%xcc, 1b
+	 sub		%g3, (1 << 5), %g3
+
+	brz,pn		%g5, 2f
+#endif /* L1DCACHE_SIZE > PAGE_SIZE */
+	 sethi		%hi(PAGE_SIZE), %g3
+
+1:	flush		%g7
+	subcc		%g3, (1 << 5), %g3
+	bne,pt		%icc, 1b
+	 add		%g7, (1 << 5), %g7
+
+2:	retry
+	nop
+	nop
+
 	.globl		xcall_capture
 xcall_capture:
 	rdpr		%pstate, %g2
@@ -433,7 +605,8 @@
 	/* These two are not performance critical... */
 	.globl		xcall_flush_tlb_all
 xcall_flush_tlb_all:
-
+	BRANCH_IF_CHEETAH(g2, g3, __cheetah_xcall_flush_tlb_all)
+__spitfire_xcall_flush_tlb_all:
 	/* Spitfire Errata #32 workaround. */
 	sethi		%hi(errata32_hwbug), %g4
 	stx		%g0, [%g4 + %lo(errata32_hwbug)]
@@ -475,17 +648,33 @@
 	flush		%g6
 	retry
 
+__cheetah_xcall_flush_tlb_all:
+	mov		0x80, %g2
+	stxa		%g0, [%g2] ASI_DMMU_DEMAP
+	stxa		%g0, [%g2] ASI_IMMU_DEMAP
+	retry
+
 	.globl		xcall_flush_cache_all
 xcall_flush_cache_all:
+	BRANCH_IF_CHEETAH(g2, g3, __cheetah_xcall_flush_cache_all)
+__spitfire_xcall_flush_cache_all:
 	sethi		%hi(16383), %g2
 	or		%g2, %lo(16383), %g2
 	clr		%g3
 1:	stxa		%g0, [%g3] ASI_IC_TAG
+	membar		#Sync
 	add		%g3, 32, %g3
 	cmp		%g3, %g2
 	bleu,pt		%xcc, 1b
 	 nop
 	flush		%g6
+	retry
+
+	/* Cheetah's caches are fully coherent in the sense that
+	 * caches are flushed here.  We need to verify this and
+	 * really just not even send out the xcall at the top level.
+	 */
+__cheetah_xcall_flush_cache_all:
 	retry
 
 	.globl		xcall_call_function