Menu
▾
▴
[LV-kern-commit] CVS: kernel-2.4/arch/ia64/mm extable.c,1.1.1.1,1.2 fault.c,1.1.1.1,1.2 init.c,1.1.1.1,1.2 tlb.c,1.1.1.1,1.2
|
From: Andy P. <at...@us...> - 2002-04-10 18:36:13
|
Update of /cvsroot/linux-vax/kernel-2.4/arch/ia64/mm
In directory usw-pr-cvs1:/tmp/cvs-serv12040/ia64/mm
Modified Files:
extable.c fault.c init.c tlb.c
Log Message:
synch 2.4.15 commit 36
Index: extable.c
===================================================================
RCS file: /cvsroot/linux-vax/kernel-2.4/arch/ia64/mm/extable.c,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- extable.c 14 Jan 2001 19:50:52 -0000 1.1.1.1
+++ extable.c 10 Apr 2002 14:27:22 -0000 1.2
@@ -1,13 +1,14 @@
/*
* Kernel exception handling table support. Derived from arch/alpha/mm/extable.c.
*
- * Copyright (C) 1998, 1999 Hewlett-Packard Co
- * Copyright (C) 1998, 1999 David Mosberger-Tang <da...@hp...>
+ * Copyright (C) 1998, 1999, 2001 Hewlett-Packard Co
+ * Copyright (C) 1998, 1999, 2001 David Mosberger-Tang <da...@hp...>
*/
#include <linux/config.h>
-#include <linux/module.h>
+
#include <asm/uaccess.h>
+#include <asm/module.h>
extern const struct exception_table_entry __start___ex_table[];
extern const struct exception_table_entry __stop___ex_table[];
@@ -15,54 +16,66 @@
static inline const struct exception_table_entry *
search_one_table (const struct exception_table_entry *first,
const struct exception_table_entry *last,
- signed long value)
+ unsigned long ip, unsigned long gp)
{
- /* Abort early if the search value is out of range. */
- if (value != (signed int)value)
- return 0;
-
while (first <= last) {
const struct exception_table_entry *mid;
long diff;
- /*
- * We know that first and last are both kernel virtual
- * pointers (region 7) so first+last will cause an
- * overflow. We fix that by calling __va() on the
- * result, which will ensure that the top two bits get
- * set again.
- */
- mid = (void *) __va((((__u64) first + (__u64) last)/2/sizeof(*mid))*sizeof(*mid));
- diff = mid->addr - value;
+
+ mid = &first[(last - first)/2];
+ diff = (mid->addr + gp) - ip;
if (diff == 0)
return mid;
else if (diff < 0)
- first = mid+1;
+ first = mid + 1;
else
- last = mid-1;
+ last = mid - 1;
}
return 0;
}
-register unsigned long gp __asm__("gp");
+#ifndef CONFIG_MODULES
+register unsigned long main_gp __asm__("gp");
+#endif
-const struct exception_table_entry *
+struct exception_fixup
search_exception_table (unsigned long addr)
{
-#ifndef CONFIG_MODULE
+ const struct exception_table_entry *entry;
+ struct exception_fixup fix = { 0 };
+
+#ifndef CONFIG_MODULES
/* There is only the kernel to search. */
- return search_one_table(__start___ex_table, __stop___ex_table - 1, addr - gp);
+ entry = search_one_table(__start___ex_table, __stop___ex_table - 1, addr, main_gp);
+ if (entry)
+ fix.cont = entry->cont + main_gp;
+ return fix;
#else
- struct exception_table_entry *ret;
- /* The kernel is the last "module" -- no need to treat it special. */
+ struct archdata *archdata;
struct module *mp;
+ /* The kernel is the last "module" -- no need to treat it special. */
for (mp = module_list; mp ; mp = mp->next) {
if (!mp->ex_table_start)
continue;
- ret = search_one_table(mp->ex_table_start, mp->ex_table_end - 1, addr - mp->gp);
- if (ret)
- return ret;
+ archdata = (struct archdata *) mp->archdata_start;
+ entry = search_one_table(mp->ex_table_start, mp->ex_table_end - 1,
+ addr, (unsigned long) archdata->gp);
+ if (entry) {
+ fix.cont = entry->cont + (unsigned long) archdata->gp;
+ return fix;
+ }
}
- return 0;
#endif
+ return fix;
+}
+
+void
+handle_exception (struct pt_regs *regs, struct exception_fixup fix)
+{
+ regs->r8 = -EFAULT;
+ if (fix.cont & 4)
+ regs->r9 = 0;
+ regs->cr_iip = (long) fix.cont & ~0xf;
+ ia64_psr(regs)->ri = fix.cont & 0x3; /* set continuation slot number */
}
Index: fault.c
===================================================================
RCS file: /cvsroot/linux-vax/kernel-2.4/arch/ia64/mm/fault.c,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- fault.c 14 Jan 2001 19:50:53 -0000 1.1.1.1
+++ fault.c 10 Apr 2002 14:27:22 -0000 1.2
@@ -1,8 +1,8 @@
/*
* MMU fault handling support.
*
- * Copyright (C) 1998-2000 Hewlett-Packard Co
- * Copyright (C) 1998-2000 David Mosberger-Tang <da...@hp...>
+ * Copyright (C) 1998-2001 Hewlett-Packard Co
+ * David Mosberger-Tang <da...@hp...>
*/
#include <linux/sched.h>
#include <linux/kernel.h>
@@ -16,7 +16,7 @@
#include <asm/uaccess.h>
#include <asm/hardirq.h>
-extern void die_if_kernel (char *, struct pt_regs *, long);
+extern void die (char *, struct pt_regs *, long);
/*
* This routine is analogous to expand_stack() but instead grows the
@@ -46,21 +46,20 @@
void
ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *regs)
{
- struct mm_struct *mm = current->mm;
- const struct exception_table_entry *fix;
+ int signal = SIGSEGV, code = SEGV_MAPERR;
struct vm_area_struct *vma, *prev_vma;
+ struct mm_struct *mm = current->mm;
+ struct exception_fixup fix;
struct siginfo si;
- int signal = SIGSEGV;
unsigned long mask;
/*
- * If we're in an interrupt or have no user
- * context, we must not take the fault..
+ * If we're in an interrupt or have no user context, we must not take the fault..
*/
if (in_interrupt() || !mm)
goto no_context;
- down(&mm->mmap_sem);
+ down_read(&mm->mmap_sem);
vma = find_vma_prev(mm, address, &prev_vma);
if (!vma)
@@ -71,6 +70,8 @@
goto check_expansion;
good_area:
+ code = SEGV_ACCERR;
+
/* OK, we've got a good vm_area for this memory area. Check the access permissions: */
# define VM_READ_BIT 0
@@ -89,12 +90,13 @@
if ((vma->vm_flags & mask) != mask)
goto bad_area;
+ survive:
/*
* If for any reason at all we couldn't handle the fault, make
* sure we exit gracefully rather than endlessly redo the
* fault.
*/
- switch (handle_mm_fault(mm, vma, address, mask) != 0) {
+ switch (handle_mm_fault(mm, vma, address, mask)) {
case 1:
++current->min_flt;
break;
@@ -112,7 +114,7 @@
default:
goto out_of_memory;
}
- up(&mm->mmap_sem);
+ up_read(&mm->mmap_sem);
return;
check_expansion:
@@ -135,7 +137,7 @@
goto good_area;
bad_area:
- up(&mm->mmap_sem);
+ up_read(&mm->mmap_sem);
if (isr & IA64_ISR_SP) {
/*
* This fault was due to a speculative load set the "ed" bit in the psr to
@@ -147,7 +149,7 @@
if (user_mode(regs)) {
si.si_signo = signal;
si.si_errno = 0;
- si.si_code = SI_KERNEL;
+ si.si_code = code;
si.si_addr = (void *) address;
force_sig_info(signal, &si, current);
return;
@@ -163,29 +165,40 @@
return;
}
+#ifdef GAS_HAS_LOCAL_TAGS
+ fix = search_exception_table(regs->cr_iip + ia64_psr(regs)->ri);
+#else
fix = search_exception_table(regs->cr_iip);
- if (fix) {
- regs->r8 = -EFAULT;
- if (fix->skip & 1) {
- regs->r9 = 0;
- }
- regs->cr_iip += ((long) fix->skip) & ~15;
- regs->cr_ipsr &= ~IA64_PSR_RI; /* clear exception slot number */
+#endif
+ if (fix.cont) {
+ handle_exception(regs, fix);
return;
}
/*
- * Oops. The kernel tried to access some bad page. We'll have
- * to terminate things with extreme prejudice.
+ * Oops. The kernel tried to access some bad page. We'll have to terminate things
+ * with extreme prejudice.
*/
- printk(KERN_ALERT "Unable to handle kernel paging request at "
- "virtual address %016lx\n", address);
- die_if_kernel("Oops", regs, isr);
+ bust_spinlocks(1);
+
+ if (address < PAGE_SIZE)
+ printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
+ else
+ printk(KERN_ALERT "Unable to handle kernel paging request at "
+ "virtual address %016lx\n", address);
+ die("Oops", regs, isr);
+ bust_spinlocks(0);
do_exit(SIGKILL);
return;
out_of_memory:
- up(&mm->mmap_sem);
+ up_read(&mm->mmap_sem);
+ if (current->pid == 1) {
+ current->policy |= SCHED_YIELD;
+ schedule();
+ down_read(&mm->mmap_sem);
+ goto survive;
+ }
printk("VM: killing process %s\n", current->comm);
if (user_mode(regs))
do_exit(SIGKILL);
Index: init.c
===================================================================
RCS file: /cvsroot/linux-vax/kernel-2.4/arch/ia64/mm/init.c,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- init.c 14 Jan 2001 19:50:55 -0000 1.1.1.1
+++ init.c 10 Apr 2002 14:27:22 -0000 1.2
@@ -1,8 +1,8 @@
/*
* Initialize MMU support.
*
- * Copyright (C) 1998-2000 Hewlett-Packard Co
- * Copyright (C) 1998-2000 David Mosberger-Tang <da...@hp...>
+ * Copyright (C) 1998-2001 Hewlett-Packard Co
+ * Copyright (C) 1998-2001 David Mosberger-Tang <da...@hp...>
*/
#include <linux/config.h>
#include <linux/kernel.h>
@@ -23,116 +23,36 @@
#include <asm/pgalloc.h>
#include <asm/sal.h>
#include <asm/system.h>
+#include <asm/uaccess.h>
+#include <asm/tlb.h>
+
+mmu_gather_t mmu_gathers[NR_CPUS];
/* References to section boundaries: */
extern char _stext, _etext, _edata, __init_begin, __init_end;
-/*
- * These are allocated in head.S so that we get proper page alignment.
- * If you change the size of these then change head.S as well.
- */
-extern char empty_bad_page[PAGE_SIZE];
-extern pmd_t empty_bad_pmd_table[PTRS_PER_PMD];
-extern pte_t empty_bad_pte_table[PTRS_PER_PTE];
-
extern void ia64_tlb_init (void);
-static unsigned long totalram_pages;
-
-/*
- * Fill in empty_bad_pmd_table with entries pointing to
- * empty_bad_pte_table and return the address of this PMD table.
- */
-static pmd_t *
-get_bad_pmd_table (void)
-{
- pmd_t v;
- int i;
-
- pmd_set(&v, empty_bad_pte_table);
-
- for (i = 0; i < PTRS_PER_PMD; ++i)
- empty_bad_pmd_table[i] = v;
-
- return empty_bad_pmd_table;
-}
-
-/*
- * Fill in empty_bad_pte_table with PTEs pointing to empty_bad_page
- * and return the address of this PTE table.
- */
-static pte_t *
-get_bad_pte_table (void)
-{
- pte_t v;
- int i;
-
- set_pte(&v, pte_mkdirty(mk_pte_phys(__pa(empty_bad_page), PAGE_SHARED)));
-
- for (i = 0; i < PTRS_PER_PTE; ++i)
- empty_bad_pte_table[i] = v;
+unsigned long MAX_DMA_ADDRESS = PAGE_OFFSET + 0x100000000UL;
- return empty_bad_pte_table;
-}
-
-void
-__handle_bad_pgd (pgd_t *pgd)
-{
- pgd_ERROR(*pgd);
- pgd_set(pgd, get_bad_pmd_table());
-}
-
-void
-__handle_bad_pmd (pmd_t *pmd)
-{
- pmd_ERROR(*pmd);
- pmd_set(pmd, get_bad_pte_table());
-}
-
-/*
- * Allocate and initialize an L3 directory page and set
- * the L2 directory entry PMD to the newly allocated page.
- */
-pte_t*
-get_pte_slow (pmd_t *pmd, unsigned long offset)
-{
- pte_t *pte;
-
- pte = (pte_t *) __get_free_page(GFP_KERNEL);
- if (pmd_none(*pmd)) {
- if (pte) {
- /* everything A-OK */
- clear_page(pte);
- pmd_set(pmd, pte);
- return pte + offset;
- }
- pmd_set(pmd, get_bad_pte_table());
- return NULL;
- }
- free_page((unsigned long) pte);
- if (pmd_bad(*pmd)) {
- __handle_bad_pmd(pmd);
- return NULL;
- }
- return (pte_t *) pmd_page(*pmd) + offset;
-}
+static unsigned long totalram_pages;
int
do_check_pgt_cache (int low, int high)
{
int freed = 0;
- if (pgtable_cache_size > high) {
- do {
- if (pgd_quicklist)
- free_page((unsigned long)get_pgd_fast()), ++freed;
- if (pmd_quicklist)
- free_page((unsigned long)get_pmd_fast()), ++freed;
- if (pte_quicklist)
- free_page((unsigned long)get_pte_fast()), ++freed;
- } while (pgtable_cache_size > low);
- }
- return freed;
+ if (pgtable_cache_size > high) {
+ do {
+ if (pgd_quicklist)
+ free_page((unsigned long)pgd_alloc_one_fast(0)), ++freed;
+ if (pmd_quicklist)
+ free_page((unsigned long)pmd_alloc_one_fast(0, 0)), ++freed;
+ if (pte_quicklist)
+ free_page((unsigned long)pte_alloc_one_fast(0, 0)), ++freed;
+ } while (pgtable_cache_size > low);
+ }
+ return freed;
}
/*
@@ -188,21 +108,21 @@
{
/*
* EFI uses 4KB pages while the kernel can use 4KB or bigger.
- * Thus EFI and the kernel may have different page sizes. It is
- * therefore possible to have the initrd share the same page as
- * the end of the kernel (given current setup).
+ * Thus EFI and the kernel may have different page sizes. It is
+ * therefore possible to have the initrd share the same page as
+ * the end of the kernel (given current setup).
*
* To avoid freeing/using the wrong page (kernel sized) we:
- * - align up the beginning of initrd
- * - keep the end untouched
+ * - align up the beginning of initrd
+ * - align down the end of initrd
*
* | |
* |=============| a000
* | |
* | |
* | | 9000
- * |/////////////|
- * |/////////////|
+ * |/////////////|
+ * |/////////////|
* |=============| 8000
* |///INITRD////|
* |/////////////|
@@ -211,18 +131,21 @@
* |KKKKKKKKKKKKK|
* |=============| 6000
* |KKKKKKKKKKKKK|
- * |KKKKKKKKKKKKK|
+ * |KKKKKKKKKKKKK|
* K=kernel using 8KB pages
- *
+ *
* In this example, we must free page 8000 ONLY. So we must align up
* initrd_start and keep initrd_end as is.
*/
start = PAGE_ALIGN(start);
+ end = end & PAGE_MASK;
if (start < end)
printk ("Freeing initrd memory: %ldkB freed\n", (end - start) >> 10);
for (; start < end; start += PAGE_SIZE) {
+ if (!VALID_PAGE(virt_to_page(start)))
+ continue;
clear_bit(PG_reserved, &virt_to_page(start)->flags);
set_page_count(virt_to_page(start), 1);
free_page(start);
@@ -244,13 +167,40 @@
}
void
-show_mem (void)
+show_mem(void)
{
int i, total = 0, reserved = 0;
int shared = 0, cached = 0;
printk("Mem-info:\n");
show_free_areas();
+
+#ifdef CONFIG_DISCONTIGMEM
+ {
+ pg_data_t *pgdat = pgdat_list;
+
+ printk("Free swap: %6dkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
+ do {
+ printk("Node ID: %d\n", pgdat->node_id);
+ for(i = 0; i < pgdat->node_size; i++) {
+ if (PageReserved(pgdat->node_mem_map+i))
+ reserved++;
+ else if (PageSwapCache(pgdat->node_mem_map+i))
+ cached++;
+ else if (page_count(pgdat->node_mem_map + i))
+ shared += page_count(pgdat->node_mem_map + i) - 1;
+ }
+ printk("\t%d pages of RAM\n", pgdat->node_size);
+ printk("\t%d reserved pages\n", reserved);
+ printk("\t%d pages shared\n", shared);
+ printk("\t%d pages swap cached\n", cached);
+ pgdat = pgdat->node_next;
+ } while (pgdat);
+ printk("Total of %ld pages in page table cache\n", pgtable_cache_size);
+ show_buffers();
+ printk("%d free buffer pages\n", nr_free_buffer_pages());
+ }
+#else /* !CONFIG_DISCONTIGMEM */
printk("Free swap: %6dkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
i = max_mapnr;
while (i-- > 0) {
@@ -268,6 +218,7 @@
printk("%d pages swap cached\n", cached);
printk("%ld pages in page table cache\n", pgtable_cache_size);
show_buffers();
+#endif /* !CONFIG_DISCONTIGMEM */
}
/*
@@ -286,52 +237,58 @@
page_address(page));
pgd = pgd_offset_k(address); /* note: this is NOT pgd_offset()! */
- pmd = pmd_alloc(pgd, address);
- if (!pmd) {
- __free_page(page);
- panic("Out of memory.");
- return 0;
- }
- pte = pte_alloc(pmd, address);
- if (!pte) {
- __free_page(page);
- panic("Out of memory.");
- return 0;
- }
- if (!pte_none(*pte)) {
- pte_ERROR(*pte);
- __free_page(page);
- return 0;
+
+ spin_lock(&init_mm.page_table_lock);
+ {
+ pmd = pmd_alloc(&init_mm, pgd, address);
+ if (!pmd)
+ goto out;
+ pte = pte_alloc(&init_mm, pmd, address);
+ if (!pte)
+ goto out;
+ if (!pte_none(*pte)) {
+ pte_ERROR(*pte);
+ goto out;
+ }
+ flush_page_to_ram(page);
+ set_pte(pte, mk_pte(page, PAGE_GATE));
}
- flush_page_to_ram(page);
- set_pte(pte, mk_pte(page, PAGE_GATE));
+ out: spin_unlock(&init_mm.page_table_lock);
/* no need for flush_tlb */
return page;
}
void __init
-ia64_rid_init (void)
+ia64_mmu_init (void *my_cpu_data)
{
unsigned long flags, rid, pta, impl_va_bits;
+ extern void __init tlb_init (void);
#ifdef CONFIG_DISABLE_VHPT
# define VHPT_ENABLE_BIT 0
#else
# define VHPT_ENABLE_BIT 1
#endif
- /* Set up the kernel identity mappings (regions 6 & 7) and the vmalloc area (region 5): */
+ /*
+ * Set up the kernel identity mapping for regions 6 and 5. The mapping for region
+ * 7 is setup up in _start().
+ */
ia64_clear_ic(flags);
rid = ia64_rid(IA64_REGION_ID_KERNEL, __IA64_UNCACHED_OFFSET);
- ia64_set_rr(__IA64_UNCACHED_OFFSET, (rid << 8) | (_PAGE_SIZE_256M << 2));
-
- rid = ia64_rid(IA64_REGION_ID_KERNEL, PAGE_OFFSET);
- ia64_set_rr(PAGE_OFFSET, (rid << 8) | (_PAGE_SIZE_256M << 2));
+ ia64_set_rr(__IA64_UNCACHED_OFFSET, (rid << 8) | (IA64_GRANULE_SHIFT << 2));
rid = ia64_rid(IA64_REGION_ID_KERNEL, VMALLOC_START);
ia64_set_rr(VMALLOC_START, (rid << 8) | (PAGE_SHIFT << 2) | 1);
+ /* ensure rr6 is up-to-date before inserting the PERCPU_ADDR translation: */
+ ia64_srlz_d();
+
+ ia64_itr(0x2, IA64_TR_PERCPU_DATA, PERCPU_ADDR,
+ pte_val(mk_pte_phys(__pa(my_cpu_data), PAGE_KERNEL)), PAGE_SHIFT);
+
__restore_flags(flags);
+ ia64_srlz_i();
/*
* Check if the virtually mapped linear page table (VMLPT) overlaps with a mapped
@@ -356,7 +313,7 @@
# define vmlpt_bits (impl_va_bits - PAGE_SHIFT + pte_bits)
# define POW2(n) (1ULL << (n))
- impl_va_bits = ffz(~my_cpu_data.unimpl_va_mask);
+ impl_va_bits = ffz(~(local_cpu_data->unimpl_va_mask | (7UL << 61)));
if (impl_va_bits < 51 || impl_va_bits > 61)
panic("CPU has bogus IMPL_VA_MSB value of %lu!\n", impl_va_bits - 1);
@@ -374,6 +331,8 @@
* enabled.
*/
ia64_set_pta(pta | (0 << 8) | (vmlpt_bits << 2) | VHPT_ENABLE_BIT);
+
+ ia64_tlb_init();
}
/*
@@ -390,7 +349,7 @@
memset(zones_size, 0, sizeof(zones_size));
- max_dma = (PAGE_ALIGN(MAX_DMA_ADDRESS) >> PAGE_SHIFT);
+ max_dma = virt_to_phys((void *) MAX_DMA_ADDRESS) >> PAGE_SHIFT;
if (max_low_pfn < max_dma)
zones_size[ZONE_DMA] = max_low_pfn;
else {
@@ -428,6 +387,7 @@
{
extern char __start_gate_section[];
long reserved_pages, codesize, datasize, initsize;
+ unsigned long num_pgt_pages;
#ifdef CONFIG_PCI
/*
@@ -460,6 +420,19 @@
(unsigned long) nr_free_pages() << (PAGE_SHIFT - 10),
max_mapnr << (PAGE_SHIFT - 10), codesize >> 10, reserved_pages << (PAGE_SHIFT - 10),
datasize >> 10, initsize >> 10);
+
+ /*
+ * Allow for enough (cached) page table pages so that we can map the entire memory
+ * at least once. Each task also needs a couple of page tables pages, so add in a
+ * fudge factor for that (don't use "threads-max" here; that would be wrong!).
+ * Don't allow the cache to be more than 10% of total memory, though.
+ */
+# define NUM_TASKS 500 /* typical number of tasks */
+ num_pgt_pages = nr_free_pages() / PTRS_PER_PGD + NUM_TASKS;
+ if (num_pgt_pages > nr_free_pages() / 10)
+ num_pgt_pages = nr_free_pages() / 10;
+ if (num_pgt_pages > pgt_cache_water[1])
+ pgt_cache_water[1] = num_pgt_pages;
/* install the gate page in the global page table: */
put_gate_page(virt_to_page(__start_gate_section), GATE_ADDR);
Index: tlb.c
===================================================================
RCS file: /cvsroot/linux-vax/kernel-2.4/arch/ia64/mm/tlb.c,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- tlb.c 14 Jan 2001 19:50:55 -0000 1.1.1.1
+++ tlb.c 10 Apr 2002 14:27:22 -0000 1.2
@@ -1,11 +1,11 @@
/*
* TLB support routines.
*
- * Copyright (C) 1998-2000 Hewlett-Packard Co
- * Copyright (C) 1998-2000 David Mosberger-Tang <da...@hp...>
+ * Copyright (C) 1998-2001 Hewlett-Packard Co
+ * David Mosberger-Tang <da...@hp...>
*
- * 08/02/00 A. Mallick <asi...@in...>
- * Modified RID allocation for SMP
+ * 08/02/00 A. Mallick <asi...@in...>
+ * Modified RID allocation for SMP
* Goutham Rao <gou...@in...>
* IPI based ptc implementation and A-step IPI implementation.
*/
@@ -41,88 +41,6 @@
};
/*
- * Seralize usage of ptc.g
- */
-spinlock_t ptcg_lock = SPIN_LOCK_UNLOCKED; /* see <asm/pgtable.h> */
-
-#if defined(CONFIG_SMP) && !defined(CONFIG_ITANIUM_PTCG)
-
-#include <linux/irq.h>
-
-unsigned long flush_end, flush_start, flush_nbits, flush_rid;
-atomic_t flush_cpu_count;
-
-/*
- * flush_tlb_no_ptcg is called with ptcg_lock locked
- */
-static inline void
-flush_tlb_no_ptcg (unsigned long start, unsigned long end, unsigned long nbits)
-{
- extern void smp_send_flush_tlb (void);
- unsigned long saved_tpr = 0;
- unsigned long flags;
-
- /*
- * Some times this is called with interrupts disabled and causes
- * dead-lock; to avoid this we enable interrupt and raise the TPR
- * to enable ONLY IPI.
- */
- __save_flags(flags);
- if (!(flags & IA64_PSR_I)) {
- saved_tpr = ia64_get_tpr();
- ia64_srlz_d();
- ia64_set_tpr(IPI_IRQ - 16);
- ia64_srlz_d();
- local_irq_enable();
- }
-
- spin_lock(&ptcg_lock);
- flush_rid = ia64_get_rr(start);
- ia64_srlz_d();
- flush_start = start;
- flush_end = end;
- flush_nbits = nbits;
- atomic_set(&flush_cpu_count, smp_num_cpus - 1);
- smp_send_flush_tlb();
- /*
- * Purge local TLB entries. ALAT invalidation is done in ia64_leave_kernel.
- */
- do {
- asm volatile ("ptc.l %0,%1" :: "r"(start), "r"(nbits<<2) : "memory");
- start += (1UL << nbits);
- } while (start < end);
-
- ia64_srlz_i(); /* srlz.i implies srlz.d */
-
- /*
- * Wait for other CPUs to finish purging entries.
- */
-#if (defined(CONFIG_ITANIUM_ASTEP_SPECIFIC) || defined(CONFIG_ITANIUM_BSTEP_SPECIFIC))
- {
- unsigned long start = ia64_get_itc();
- while (atomic_read(&flush_cpu_count) > 0) {
- if ((ia64_get_itc() - start) > 40000UL) {
- atomic_set(&flush_cpu_count, smp_num_cpus - 1);
- smp_send_flush_tlb();
- start = ia64_get_itc();
- }
- }
- }
-#else
- while (atomic_read(&flush_cpu_count)) {
- /* Nothing */
- }
-#endif
- if (!(flags & IA64_PSR_I)) {
- local_irq_disable();
- ia64_set_tpr(saved_tpr);
- ia64_srlz_d();
- }
-}
-
-#endif /* CONFIG_SMP && !CONFIG_ITANIUM_PTCG */
-
-/*
* Acquire the ia64_ctx.lock before calling this function!
*/
void
@@ -148,7 +66,7 @@
if (tsk_context == ia64_ctx.next) {
if (++ia64_ctx.next >= ia64_ctx.limit) {
/* empty range: reset the range limit and start over */
- if (ia64_ctx.next > max_ctx)
+ if (ia64_ctx.next > max_ctx)
ia64_ctx.next = 300;
ia64_ctx.limit = max_ctx + 1;
goto repeat;
@@ -161,16 +79,36 @@
flush_tlb_all();
}
+static inline void
+ia64_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbits)
+{
+ static spinlock_t ptcg_lock = SPIN_LOCK_UNLOCKED;
+
+ /* HW requires global serialization of ptc.ga. */
+ spin_lock(&ptcg_lock);
+ {
+ do {
+ /*
+ * Flush ALAT entries also.
+ */
+ asm volatile ("ptc.ga %0,%1;;srlz.i;;" :: "r"(start), "r"(nbits<<2)
+ : "memory");
+ start += (1UL << nbits);
+ } while (start < end);
+ }
+ spin_unlock(&ptcg_lock);
+}
+
void
__flush_tlb_all (void)
{
unsigned long i, j, flags, count0, count1, stride0, stride1, addr;
- addr = my_cpu_data.ptce_base;
- count0 = my_cpu_data.ptce_count[0];
- count1 = my_cpu_data.ptce_count[1];
- stride0 = my_cpu_data.ptce_stride[0];
- stride1 = my_cpu_data.ptce_stride[1];
+ addr = local_cpu_data->ptce_base;
+ count0 = local_cpu_data->ptce_count[0];
+ count1 = local_cpu_data->ptce_count[1];
+ stride0 = local_cpu_data->ptce_stride[0];
+ stride1 = local_cpu_data->ptce_stride[1];
local_irq_save(flags);
for (i = 0; i < count0; ++i) {
@@ -221,23 +159,15 @@
}
start &= ~((1UL << nbits) - 1);
-#if defined(CONFIG_SMP) && !defined(CONFIG_ITANIUM_PTCG)
- flush_tlb_no_ptcg(start, end, nbits);
-#else
- spin_lock(&ptcg_lock);
- do {
# ifdef CONFIG_SMP
- /*
- * Flush ALAT entries also.
- */
- asm volatile ("ptc.ga %0,%1;;srlz.i;;" :: "r"(start), "r"(nbits<<2) : "memory");
+ platform_global_tlb_purge(start, end, nbits);
# else
+ do {
asm volatile ("ptc.l %0,%1" :: "r"(start), "r"(nbits<<2) : "memory");
-# endif
start += (1UL << nbits);
} while (start < end);
-#endif /* CONFIG_SMP && !defined(CONFIG_ITANIUM_PTCG) */
- spin_unlock(&ptcg_lock);
+# endif
+
ia64_insn_group_barrier();
ia64_srlz_i(); /* srlz.i implies srlz.d */
ia64_insn_group_barrier();
@@ -249,11 +179,11 @@
ia64_ptce_info_t ptce_info;
ia64_get_ptce(&ptce_info);
- my_cpu_data.ptce_base = ptce_info.base;
- my_cpu_data.ptce_count[0] = ptce_info.count[0];
- my_cpu_data.ptce_count[1] = ptce_info.count[1];
- my_cpu_data.ptce_stride[0] = ptce_info.stride[0];
- my_cpu_data.ptce_stride[1] = ptce_info.stride[1];
+ local_cpu_data->ptce_base = ptce_info.base;
+ local_cpu_data->ptce_count[0] = ptce_info.count[0];
+ local_cpu_data->ptce_count[1] = ptce_info.count[1];
+ local_cpu_data->ptce_stride[0] = ptce_info.stride[0];
+ local_cpu_data->ptce_stride[1] = ptce_info.stride[1];
__flush_tlb_all(); /* nuke left overs from bootstrapping... */
}
|
