|
From: Itsuro O. <od...@us...> - 2006-11-22 01:30:35
|
Update of /cvsroot/mkdump/mkexecpatch/2.0/linus/2.6.17/arch/i386/kernel In directory sc8-pr-cvs7.sourceforge.net:/tmp/cvs-serv22290/arch/i386/kernel Added Files: Makefile apic.c doublefault.c io_apic.c irq.c minik_dump.c smp.c traps.c Log Message: support 2.6.17 (caution: only i386 done) --- NEW FILE: apic.c --- /* * Local APIC handling, local APIC timers * * (c) 1999, 2000 Ingo Molnar <mi...@re...> * * Fixes * Maciej W. Rozycki : Bits for genuine 82489DX APICs; * thanks to Eric Gilmore * and Rolf G. Tews * for testing these extensively. * Maciej W. Rozycki : Various updates and fixes. * Mikael Pettersson : Power Management for UP-APIC. * Pavel Machek and * Mikael Pettersson : PM converted to driver model. */ #include <linux/config.h> #include <linux/init.h> [...1375 lines suppressed...] * Hack: In case of kdump, after a crash, kernel might be booting * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid * might be zero if read from MP tables. Get it from LAPIC. */ #ifdef CONFIG_CRASH_DUMP boot_cpu_physical_apicid = GET_APIC_ID(apic_read(APIC_ID)); #endif phys_cpu_present_map = physid_mask_of_physid(boot_cpu_physical_apicid); setup_local_APIC(); #ifdef CONFIG_X86_IO_APIC if (smp_found_config) if (!skip_ioapic_setup && nr_ioapics) setup_IO_APIC(); #endif setup_boot_APIC_clock(); return 0; } --- NEW FILE: Makefile --- # # Makefile for the linux kernel. # extra-y := head.o init_task.o vmlinux.lds obj-y := process.o semaphore.o signal.o entry.o traps.o irq.o \ ptrace.o time.o ioport.o ldt.o setup.o i8259.o sys_i386.o \ pci-dma.o i386_ksyms.o i387.o bootflag.o \ quirks.o i8237.o topology.o alternative.o obj-y += cpu/ obj-y += timers/ obj-y += acpi/ obj-$(CONFIG_X86_BIOS_REBOOT) += reboot.o obj-$(CONFIG_MCA) += mca.o obj-$(CONFIG_X86_MSR) += msr.o obj-$(CONFIG_X86_CPUID) += cpuid.o obj-$(CONFIG_MICROCODE) += microcode.o obj-$(CONFIG_APM) += apm.o obj-$(CONFIG_X86_SMP) += smp.o smpboot.o obj-$(CONFIG_X86_TRAMPOLINE) += trampoline.o obj-$(CONFIG_X86_MPPARSE) += mpparse.o obj-$(CONFIG_X86_LOCAL_APIC) += apic.o nmi.o obj-$(CONFIG_X86_IO_APIC) += io_apic.o obj-$(CONFIG_X86_REBOOTFIXUPS) += reboot_fixups.o obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o crash.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_X86_NUMAQ) += numaq.o obj-$(CONFIG_X86_SUMMIT_NUMA) += summit.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_MODULES) += module.o obj-y += sysenter.o vsyscall.o obj-$(CONFIG_ACPI_SRAT) += srat.o obj-$(CONFIG_HPET_TIMER) += time_hpet.o obj-$(CONFIG_EFI) += efi.o efi_stub.o obj-$(CONFIG_DOUBLEFAULT) += doublefault.o obj-$(CONFIG_VM86) += vm86.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_DUMP_MKEXEC) += minik_dump.o EXTRA_AFLAGS := -traditional obj-$(CONFIG_SCx200) += scx200.o # vsyscall.o contains the vsyscall DSO images as __initdata. # We must build both images before we can assemble it. # Note: kbuild does not track this dependency due to usage of .incbin $(obj)/vsyscall.o: $(obj)/vsyscall-int80.so $(obj)/vsyscall-sysenter.so targets += $(foreach F,int80 sysenter,vsyscall-$F.o vsyscall-$F.so) targets += vsyscall-note.o vsyscall.lds # The DSO images are built using a special linker script. quiet_cmd_syscall = SYSCALL $@ cmd_syscall = $(CC) -m elf_i386 -nostdlib $(SYSCFLAGS_$(@F)) \ -Wl,-T,$(filter-out FORCE,$^) -o $@ export CPPFLAGS_vsyscall.lds += -P -C -U$(ARCH) vsyscall-flags = -shared -s -Wl,-soname=linux-gate.so.1 SYSCFLAGS_vsyscall-sysenter.so = $(vsyscall-flags) SYSCFLAGS_vsyscall-int80.so = $(vsyscall-flags) $(obj)/vsyscall-int80.so $(obj)/vsyscall-sysenter.so: \ $(obj)/vsyscall-%.so: $(src)/vsyscall.lds \ $(obj)/vsyscall-%.o $(obj)/vsyscall-note.o FORCE $(call if_changed,syscall) # We also create a special relocatable object that should mirror the symbol # table and layout of the linked DSO. With ld -R we can then refer to # these symbols in the kernel code rather than hand-coded addresses. extra-y += vsyscall-syms.o $(obj)/built-in.o: $(obj)/vsyscall-syms.o $(obj)/built-in.o: ld_flags += -R $(obj)/vsyscall-syms.o SYSCFLAGS_vsyscall-syms.o = -r $(obj)/vsyscall-syms.o: $(src)/vsyscall.lds \ $(obj)/vsyscall-sysenter.o $(obj)/vsyscall-note.o FORCE $(call if_changed,syscall) --- NEW FILE: traps.c --- /* * linux/arch/i386/traps.c * * Copyright (C) 1991, 1992 Linus Torvalds * * Pentium III FXSR, SSE support * Gareth Hughes <ga...@va...>, May 2000 */ /* * 'Traps.c' handles hardware traps and faults after we have saved some * state in 'asm.s'. */ #include <linux/config.h> #include <linux/sched.h> #include <linux/kernel.h> #include <linux/string.h> #include <linux/errno.h> #include <linux/timer.h> [...1180 lines suppressed...] set_in_cr4(X86_CR4_OSXMMEXCPT); printk("done.\n"); } set_system_gate(SYSCALL_VECTOR,&system_call); /* * Should be a barrier for any external CPU state. */ cpu_init(); trap_init_hook(); } static int __init kstack_setup(char *s) { kstack_depth_to_print = simple_strtoul(s, NULL, 0); return 1; } __setup("kstack=", kstack_setup); --- NEW FILE: io_apic.c --- /* * Intel IO-APIC support for multi-Pentium hosts. * * Copyright (C) 1997, 1998, 1999, 2000 Ingo Molnar, Hajnalka Szabo * * Many thanks to Stig Venaas for trying out countless experimental * patches and reporting/debugging problems patiently! * * (c) 1999, Multiple IO-APIC support, developed by * Ken-ichi Yaku <ya...@cs...> and * Hidemi Kishimoto <kis...@cs...>, * further tested and cleaned up by Zach Brown <za...@re...> * and Ingo Molnar <mi...@re...> * * Fixes * Maciej W. Rozycki : Bits for genuine 82489DX APICs; * thanks to Eric Gilmore * and Rolf G. Tews * for testing these extensively [...2719 lines suppressed...] apic_printk(APIC_DEBUG, KERN_DEBUG "IOAPIC[%d]: Set PCI routing entry " "(%d-%d -> 0x%x -> IRQ %d Mode:%i Active:%i)\n", ioapic, mp_ioapics[ioapic].mpc_apicid, pin, entry.vector, irq, edge_level, active_high_low); ioapic_register_intr(irq, entry.vector, edge_level); if (!ioapic && (irq < 16)) disable_8259A_irq(irq); spin_lock_irqsave(&ioapic_lock, flags); io_apic_write(ioapic, 0x11+2*pin, *(((int *)&entry)+1)); io_apic_write(ioapic, 0x10+2*pin, *(((int *)&entry)+0)); set_native_irq_info(use_pci_vector() ? entry.vector : irq, TARGET_CPUS); spin_unlock_irqrestore(&ioapic_lock, flags); return 0; } #endif /* CONFIG_ACPI */ --- NEW FILE: minik_dump.c --- /* * arch/i386/kernel/minik_dump.c * * $Id: minik_dump.c,v 1.1 2006/11/22 01:30:31 odaodab Exp $ * * Portions Copyright (C) 2004 NTT DATA CORPORATION. * Portions Copyright (C) 2004 VA Linux Systems Japan K.K. * * This file is part of Mkdump. * * Mkdump is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation (version 2 of the License). * * Mkdump is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with Mkdump; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include <linux/config.h> #include <linux/stddef.h> #include <linux/mm.h> #include <linux/swap.h> #include <linux/interrupt.h> #include <linux/pagemap.h> #include <linux/bootmem.h> #include <linux/compiler.h> #include <linux/module.h> #include <linux/suspend.h> #include <linux/pagevec.h> #include <linux/blkdev.h> #include <linux/slab.h> #include <linux/notifier.h> #include <linux/topology.h> #include <linux/sysctl.h> #include <linux/cpu.h> #include <linux/kernel.h> #include <linux/sort.h> #include <asm/e820.h> #include <asm/apic.h> #include <asm/io_apic.h> #include <asm/setup.h> #include <asm/mkexec_export.h> /* Prevent parameters to be called "minik_dump_i386_ex.crashmem" etc. */ #undef MODULE_PARAM_PREFIX #define MODULE_PARAM_PREFIX "" struct crashmem *crashmem_dma, *crashmem_main; struct crashmem *crashmem_get(int isdma) { return isdma ? crashmem_dma : crashmem_main; } EXPORT_SYMBOL(crashmem_get); static void __init crashmem_free(struct crashmem *crashmem) { if (!crashmem) return; if (crashmem->base_pa) free_bootmem(crashmem->base_pa, crashmem->size_bytes); free_bootmem(__pa(crashmem),sizeof(*crashmem)); } /* Allocated one memory type for minik. */ static int __init crashmem_set(const char *val_stringc, struct kernel_param *kp) { struct crashmem **crashmem_pointer = kp->arg; struct crashmem *crashmem; unsigned long long val_ull; unsigned long val_ul; char *val_string = (char *)val_stringc; const char *name = kp->name; void *base_va; int is_dma = (crashmem_pointer == &crashmem_dma); val_ull = memparse(val_string, &val_string); if (!val_ull || !val_string || *val_string || val_ull != (val_ul=val_ull)) { printk("mkexec: Error parsing '%s' memory size: %s\n", name, val_stringc); return 0; } val_ul = PFN_UP(val_ul) << PAGE_SHIFT; crashmem_free(*crashmem_pointer); *crashmem_pointer = NULL; crashmem = alloc_bootmem_low(sizeof(*crashmem)); if (!crashmem) { printk("mkexec: Error allocating memory!\n"); return 0; } crashmem->base_pa = 0; /* '0x400000' to get PSE targettable pages to prevent their split for R/O settings. * Current 'PMD_SIZE' may be just 2MB if PAE. */ base_va = __alloc_bootmem(val_ul, 0x400000, (is_dma ? 0 : __pa(MAX_DMA_ADDRESS))); if (!base_va) { printk("mkexec: Error allocating chunk of 0x%lx bytes of memory %s!\n", val_ul, name); crashmem_free(crashmem); return 0; } crashmem->base_pa = __pa(base_va); crashmem->size_bytes = val_ul; *crashmem_pointer = crashmem; printk("mkexec: Allocated 0x%lx of %s memory @0x%lx\n", val_ul, name, crashmem->base_pa); return 0; } /* FIXME: Make it readable/writable by: driverfs */ module_param_call(crashdma, crashmem_set, NULL, &crashmem_dma, 0); module_param_call(crashmain, crashmem_set, NULL, &crashmem_main, 0); /* * Export Symbols for mkexec (i386 specific) */ #ifdef CONFIG_X86_LOCAL_APIC EXPORT_SYMBOL(lapic_mkexec_restore); #endif #ifdef CONFIG_X86_IO_APIC EXPORT_SYMBOL(ioapic_mkexec_restore); EXPORT_SYMBOL(ioapic_lock); #endif EXPORT_SYMBOL(e820); EXPORT_SYMBOL(__PAGE_KERNEL_EXEC); EXPORT_SYMBOL(saved_command_line); #ifdef CONFIG_X86_PAE EXPORT_SYMBOL(__supported_pte_mask); #endif #ifdef CONFIG_SMP EXPORT_SYMBOL(mkdump_send_nmi); #endif #ifdef CONFIG_X86_GENERICARCH EXPORT_SYMBOL(genapic); #endif --- NEW FILE: doublefault.c --- #include <linux/mm.h> #include <linux/sched.h> #include <linux/init.h> #include <linux/init_task.h> #include <linux/fs.h> #include <asm/uaccess.h> #include <asm/pgtable.h> #include <asm/processor.h> #include <asm/desc.h> #define DOUBLEFAULT_STACKSIZE (1024) static unsigned long doublefault_stack[DOUBLEFAULT_STACKSIZE]; #define STACK_START (unsigned long)(doublefault_stack+DOUBLEFAULT_STACKSIZE) #define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + 0x1000000) static void doublefault_fn(void) { struct Xgt_desc_struct gdt_desc = {0, 0}; unsigned long gdt, tss; struct tss_struct *t = NULL; store_gdt(&gdt_desc); gdt = gdt_desc.address; printk("double fault, gdt at %08lx [%d bytes]\n", gdt, gdt_desc.size); if (ptr_ok(gdt)) { gdt += GDT_ENTRY_TSS << 3; tss = *(u16 *)(gdt+2); tss += *(u8 *)(gdt+4) << 16; tss += *(u8 *)(gdt+7) << 24; printk("double fault, tss at %08lx\n", tss); if (ptr_ok(tss)) { t = (struct tss_struct *)tss; printk("eip = %08lx, esp = %08lx\n", t->eip, t->esp); printk("eax = %08lx, ebx = %08lx, ecx = %08lx, edx = %08lx\n", t->eax, t->ebx, t->ecx, t->edx); printk("esi = %08lx, edi = %08lx\n", t->esi, t->edi); } } #ifdef CONFIG_DUMP_MKEXEC /* FIXME: breakpoint() of KGDB hangs here. */ { struct pt_regs pt_regs; if (t) { pt_regs.ebx=t->ebx; pt_regs.ecx=t->ecx; pt_regs.edx=t->edx; pt_regs.esi=t->esi; pt_regs.edi=t->edi; pt_regs.ebp=t->eax; pt_regs.xds=(t->__dsh<<16) | t->ds; pt_regs.xes=(t->__esh<<16) | t->es; /* FIXME: Meaning? Not set by minik/LKCD get_current_regs(). */ pt_regs.orig_eax=0; pt_regs.eip=t->eip; pt_regs.xcs=(t->__csh<<16) | t->cs; pt_regs.eflags=t->eflags; pt_regs.esp=t->esp; pt_regs.xss=(t->__ssh<<16) | t->ss; } do_dump("double fault - kernel stack overflow?", (!t ? NULL : &pt_regs)); } #endif /* CONFIG_DUMP_MKEXEC */ for (;;) /* nothing */; } struct tss_struct doublefault_tss __cacheline_aligned = { .esp0 = STACK_START, .ss0 = __KERNEL_DS, .ldt = 0, .io_bitmap_base = INVALID_IO_BITMAP_OFFSET, .eip = (unsigned long) doublefault_fn, .eflags = X86_EFLAGS_SF | 0x2, /* 0x2 bit is always set */ .esp = STACK_START, .es = __USER_DS, .cs = __KERNEL_CS, .ss = __KERNEL_DS, .ds = __USER_DS, .__cr3 = __pa(swapper_pg_dir) }; --- NEW FILE: irq.c --- /* * linux/arch/i386/kernel/irq.c * * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar * * This file contains the lowest level x86-specific interrupt * entry, irq-stacks and irq statistics code. All the remaining * irq logic is done by the generic kernel/irq/ code and * by the x86-specific irq controller code. (e.g. i8259.c and * io_apic.c.) */ #include <asm/uaccess.h> #include <linux/module.h> #include <linux/seq_file.h> #include <linux/interrupt.h> #include <linux/kernel_stat.h> #include <linux/notifier.h> #include <linux/cpu.h> #include <linux/delay.h> #include <linux/mkexec_dump.h> DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_internodealigned_in_smp; EXPORT_PER_CPU_SYMBOL(irq_stat); #ifndef CONFIG_X86_LOCAL_APIC /* * 'what should we do if we get a hw irq event on an illegal vector'. * each architecture has to answer this themselves. */ void ack_bad_irq(unsigned int irq) { printk("unexpected IRQ trap at vector %02x\n", irq); } #endif #ifdef CONFIG_4KSTACKS /* * per-CPU IRQ handling contexts (thread information and stack) */ union irq_ctx { struct thread_info tinfo; u32 stack[THREAD_SIZE/sizeof(u32)]; }; static union irq_ctx *hardirq_ctx[NR_CPUS]; static union irq_ctx *softirq_ctx[NR_CPUS]; #endif /* * do_IRQ handles all normal device IRQ's (the special * SMP cross-CPU interrupts have their own specific * handlers). */ fastcall unsigned int do_IRQ(struct pt_regs *regs) { /* high bits used in ret_from_ code */ int irq = regs->orig_eax & 0xff; #ifdef CONFIG_4KSTACKS union irq_ctx *curctx, *irqctx; u32 *isp; #endif irq_enter(); #ifdef CONFIG_DEBUG_STACKOVERFLOW /* Debugging check for stack overflow: is there less than 1KB free? */ { long esp; __asm__ __volatile__("andl %%esp,%0" : "=r" (esp) : "0" (THREAD_SIZE - 1)); if (unlikely(esp < (sizeof(struct thread_info) + STACK_WARN))) { printk("do_IRQ: stack overflow: %ld\n", esp - sizeof(struct thread_info)); do_dump("do_IRQ: stack overflow", regs); dump_stack(); } } #endif #ifdef CONFIG_4KSTACKS curctx = (union irq_ctx *) current_thread_info(); irqctx = hardirq_ctx[smp_processor_id()]; /* * this is where we switch to the IRQ stack. However, if we are * already using the IRQ stack (because we interrupted a hardirq * handler) we can't do that and just have to keep using the * current stack (which is the irq stack already after all) */ if (curctx != irqctx) { int arg1, arg2, ebx; /* build the stack frame on the IRQ stack */ isp = (u32*) ((char*)irqctx + sizeof(*irqctx)); irqctx->tinfo.task = curctx->tinfo.task; irqctx->tinfo.previous_esp = current_stack_pointer; asm volatile( " xchgl %%ebx,%%esp \n" " call __do_IRQ \n" " movl %%ebx,%%esp \n" : "=a" (arg1), "=d" (arg2), "=b" (ebx) : "0" (irq), "1" (regs), "2" (isp) : "memory", "cc", "ecx" ); } else #endif __do_IRQ(irq, regs); irq_exit(); return 1; } #ifdef CONFIG_4KSTACKS /* * These should really be __section__(".bss.page_aligned") as well, but * gcc's 3.0 and earlier don't handle that correctly. */ static char softirq_stack[NR_CPUS * THREAD_SIZE] __attribute__((__aligned__(THREAD_SIZE))); static char hardirq_stack[NR_CPUS * THREAD_SIZE] __attribute__((__aligned__(THREAD_SIZE))); /* * allocate per-cpu stacks for hardirq and for softirq processing */ void irq_ctx_init(int cpu) { union irq_ctx *irqctx; if (hardirq_ctx[cpu]) return; irqctx = (union irq_ctx*) &hardirq_stack[cpu*THREAD_SIZE]; irqctx->tinfo.task = NULL; irqctx->tinfo.exec_domain = NULL; irqctx->tinfo.cpu = cpu; irqctx->tinfo.preempt_count = HARDIRQ_OFFSET; irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); hardirq_ctx[cpu] = irqctx; irqctx = (union irq_ctx*) &softirq_stack[cpu*THREAD_SIZE]; irqctx->tinfo.task = NULL; irqctx->tinfo.exec_domain = NULL; irqctx->tinfo.cpu = cpu; irqctx->tinfo.preempt_count = SOFTIRQ_OFFSET; irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); softirq_ctx[cpu] = irqctx; printk("CPU %u irqstacks, hard=%p soft=%p\n", cpu,hardirq_ctx[cpu],softirq_ctx[cpu]); } void irq_ctx_exit(int cpu) { hardirq_ctx[cpu] = NULL; } extern asmlinkage void __do_softirq(void); asmlinkage void do_softirq(void) { unsigned long flags; struct thread_info *curctx; union irq_ctx *irqctx; u32 *isp; if (in_interrupt()) return; local_irq_save(flags); if (local_softirq_pending()) { curctx = current_thread_info(); irqctx = softirq_ctx[smp_processor_id()]; irqctx->tinfo.task = curctx->task; irqctx->tinfo.previous_esp = current_stack_pointer; /* build the stack frame on the softirq stack */ isp = (u32*) ((char*)irqctx + sizeof(*irqctx)); asm volatile( " xchgl %%ebx,%%esp \n" " call __do_softirq \n" " movl %%ebx,%%esp \n" : "=b"(isp) : "0"(isp) : "memory", "cc", "edx", "ecx", "eax" ); } local_irq_restore(flags); } EXPORT_SYMBOL(do_softirq); #endif /* * Interrupt statistics: */ atomic_t irq_err_count; /* * /proc/interrupts printing: */ int show_interrupts(struct seq_file *p, void *v) { int i = *(loff_t *) v, j; struct irqaction * action; unsigned long flags; if (i == 0) { seq_printf(p, " "); for_each_online_cpu(j) seq_printf(p, "CPU%d ",j); seq_putc(p, '\n'); } if (i < NR_IRQS) { spin_lock_irqsave(&irq_desc[i].lock, flags); action = irq_desc[i].action; if (!action) goto skip; seq_printf(p, "%3d: ",i); #ifndef CONFIG_SMP seq_printf(p, "%10u ", kstat_irqs(i)); #else for_each_online_cpu(j) seq_printf(p, "%10u ", kstat_cpu(j).irqs[i]); #endif seq_printf(p, " %14s", irq_desc[i].handler->typename); seq_printf(p, " %s", action->name); for (action=action->next; action; action = action->next) seq_printf(p, ", %s", action->name); seq_putc(p, '\n'); skip: spin_unlock_irqrestore(&irq_desc[i].lock, flags); } else if (i == NR_IRQS) { seq_printf(p, "NMI: "); for_each_online_cpu(j) seq_printf(p, "%10u ", nmi_count(j)); seq_putc(p, '\n'); #ifdef CONFIG_X86_LOCAL_APIC seq_printf(p, "LOC: "); for_each_online_cpu(j) seq_printf(p, "%10u ", per_cpu(irq_stat,j).apic_timer_irqs); seq_putc(p, '\n'); #endif seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count)); #if defined(CONFIG_X86_IO_APIC) seq_printf(p, "MIS: %10u\n", atomic_read(&irq_mis_count)); #endif } return 0; } #ifdef CONFIG_HOTPLUG_CPU #include <mach_apic.h> void fixup_irqs(cpumask_t map) { unsigned int irq; static int warned; for (irq = 0; irq < NR_IRQS; irq++) { cpumask_t mask; if (irq == 2) continue; cpus_and(mask, irq_affinity[irq], map); if (any_online_cpu(mask) == NR_CPUS) { printk("Breaking affinity for irq %i\n", irq); mask = map; } if (irq_desc[irq].handler->set_affinity) irq_desc[irq].handler->set_affinity(irq, mask); else if (irq_desc[irq].action && !(warned++)) printk("Cannot set affinity for irq %i\n", irq); } #if 0 barrier(); /* Ingo Molnar says: "after the IO-APIC masks have been redirected [note the nop - the interrupt-enable boundary on x86 is two instructions from sti] - to flush out pending hardirqs and IPIs. After this point nothing is supposed to reach this CPU." */ __asm__ __volatile__("sti; nop; cli"); barrier(); #else /* That doesn't seem sufficient. Give it 1ms. */ local_irq_enable(); mdelay(1); local_irq_disable(); #endif } #endif --- NEW FILE: smp.c --- /* * Intel SMP support routines. * * (c) 1995 Alan Cox, Building #3 <al...@re...> * (c) 1998-99, 2000 Ingo Molnar <mi...@re...> * * This code is released under the GNU General Public License version 2 or * later. */ #include <linux/init.h> #include <linux/mm.h> #include <linux/delay.h> #include <linux/spinlock.h> #include <linux/smp_lock.h> #include <linux/kernel_stat.h> #include <linux/mc146818rtc.h> #include <linux/cache.h> #include <linux/interrupt.h> #include <linux/cpu.h> #include <linux/module.h> #include <asm/mtrr.h> #include <asm/tlbflush.h> #include <mach_apic.h> /* * Some notes on x86 processor bugs affecting SMP operation: * * Pentium, Pentium Pro, II, III (and all CPUs) have bugs. * The Linux implications for SMP are handled as follows: * * Pentium III / [Xeon] * None of the E1AP-E3AP errata are visible to the user. * * E1AP. see PII A1AP * E2AP. see PII A2AP * E3AP. see PII A3AP * * Pentium II / [Xeon] * None of the A1AP-A3AP errata are visible to the user. * * A1AP. see PPro 1AP * A2AP. see PPro 2AP * A3AP. see PPro 7AP * * Pentium Pro * None of 1AP-9AP errata are visible to the normal user, * except occasional delivery of 'spurious interrupt' as trap #15. * This is very rare and a non-problem. * * 1AP. Linux maps APIC as non-cacheable * 2AP. worked around in hardware * 3AP. fixed in C0 and above steppings microcode update. * Linux does not use excessive STARTUP_IPIs. * 4AP. worked around in hardware * 5AP. symmetric IO mode (normal Linux operation) not affected. * 'noapic' mode has vector 0xf filled out properly. * 6AP. 'noapic' mode might be affected - fixed in later steppings * 7AP. We do not assume writes to the LVT deassering IRQs * 8AP. We do not enable low power mode (deep sleep) during MP bootup * 9AP. We do not use mixed mode * * Pentium * There is a marginal case where REP MOVS on 100MHz SMP * machines with B stepping processors can fail. XXX should provide * an L1cache=Writethrough or L1cache=off option. * * B stepping CPUs may hang. There are hardware work arounds * for this. We warn about it in case your board doesn't have the work * arounds. Basically thats so I can tell anyone with a B stepping * CPU and SMP problems "tough". * * Specific items [From Pentium Processor Specification Update] * * 1AP. Linux doesn't use remote read * 2AP. Linux doesn't trust APIC errors * 3AP. We work around this * 4AP. Linux never generated 3 interrupts of the same priority * to cause a lost local interrupt. * 5AP. Remote read is never used * 6AP. not affected - worked around in hardware * 7AP. not affected - worked around in hardware * 8AP. worked around in hardware - we get explicit CS errors if not * 9AP. only 'noapic' mode affected. Might generate spurious * interrupts, we log only the first one and count the * rest silently. * 10AP. not affected - worked around in hardware * 11AP. Linux reads the APIC between writes to avoid this, as per * the documentation. Make sure you preserve this as it affects * the C stepping chips too. * 12AP. not affected - worked around in hardware * 13AP. not affected - worked around in hardware * 14AP. we always deassert INIT during bootup * 15AP. not affected - worked around in hardware * 16AP. not affected - worked around in hardware * 17AP. not affected - worked around in hardware * 18AP. not affected - worked around in hardware * 19AP. not affected - worked around in BIOS * * If this sounds worrying believe me these bugs are either ___RARE___, * or are signal timing bugs worked around in hardware and there's * about nothing of note with C stepping upwards. */ DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate) ____cacheline_aligned = { &init_mm, 0, }; /* * the following functions deal with sending IPIs between CPUs. * * We use 'broadcast', CPU->CPU IPIs and self-IPIs too. */ static inline int __prepare_ICR (unsigned int shortcut, int vector) { return APIC_DM_FIXED | shortcut | vector | APIC_DEST_LOGICAL; } static inline int __prepare_ICR2 (unsigned int mask) { return SET_APIC_DEST_FIELD(mask); } void __send_IPI_shortcut(unsigned int shortcut, int vector) { /* * Subtle. In the case of the 'never do double writes' workaround * we have to lock out interrupts to be safe. As we don't care * of the value read we use an atomic rmw access to avoid costly * cli/sti. Otherwise we use an even cheaper single atomic write * to the APIC. */ unsigned int cfg; /* * Wait for idle. */ apic_wait_icr_idle(); /* * No need to touch the target chip field */ cfg = __prepare_ICR(shortcut, vector); /* * Send the IPI. The write to APIC_ICR fires this off. */ apic_write_around(APIC_ICR, cfg); } void fastcall send_IPI_self(int vector) { __send_IPI_shortcut(APIC_DEST_SELF, vector); } /* * This is only used on smaller machines. */ void send_IPI_mask_bitmask(cpumask_t cpumask, int vector) { unsigned long mask = cpus_addr(cpumask)[0]; unsigned long cfg; unsigned long flags; local_irq_save(flags); WARN_ON(mask & ~cpus_addr(cpu_online_map)[0]); /* * Wait for idle. */ apic_wait_icr_idle(); /* * prepare target chip field */ cfg = __prepare_ICR2(mask); apic_write_around(APIC_ICR2, cfg); /* * program the ICR */ cfg = __prepare_ICR(0, vector); /* * Send the IPI. The write to APIC_ICR fires this off. */ apic_write_around(APIC_ICR, cfg); local_irq_restore(flags); } void send_IPI_mask_sequence(cpumask_t mask, int vector) { unsigned long cfg, flags; unsigned int query_cpu; /* * Hack. The clustered APIC addressing mode doesn't allow us to send * to an arbitrary mask, so I do a unicasts to each CPU instead. This * should be modified to do 1 message per cluster ID - mbligh */ local_irq_save(flags); for (query_cpu = 0; query_cpu < NR_CPUS; ++query_cpu) { if (cpu_isset(query_cpu, mask)) { /* * Wait for idle. */ apic_wait_icr_idle(); /* * prepare target chip field */ cfg = __prepare_ICR2(cpu_to_logical_apicid(query_cpu)); apic_write_around(APIC_ICR2, cfg); /* * program the ICR */ cfg = __prepare_ICR(0, vector); /* * Send the IPI. The write to APIC_ICR fires this off. */ apic_write_around(APIC_ICR, cfg); } } local_irq_restore(flags); } #include <mach_ipi.h> /* must come after the send_IPI functions above for inlining */ /* * Smarter SMP flushing macros. * c/o Linus Torvalds. * * These mean you can really definitely utterly forget about * writing to user space from interrupts. (Its not allowed anyway). * * Optimizations Manfred Spraul <ma...@co...> */ static cpumask_t flush_cpumask; static struct mm_struct * flush_mm; static unsigned long flush_va; static DEFINE_SPINLOCK(tlbstate_lock); #define FLUSH_ALL 0xffffffff /* * We cannot call mmdrop() because we are in interrupt context, * instead update mm->cpu_vm_mask. * * We need to reload %cr3 since the page tables may be going * away from under us.. */ static inline void leave_mm (unsigned long cpu) { if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) BUG(); cpu_clear(cpu, per_cpu(cpu_tlbstate, cpu).active_mm->cpu_vm_mask); load_cr3(swapper_pg_dir); } /* * * The flush IPI assumes that a thread switch happens in this order: * [cpu0: the cpu that switches] * 1) switch_mm() either 1a) or 1b) * 1a) thread switch to a different mm * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask); * Stop ipi delivery for the old mm. This is not synchronized with * the other cpus, but smp_invalidate_interrupt ignore flush ipis * for the wrong mm, and in the worst case we perform a superflous * tlb flush. * 1a2) set cpu_tlbstate to TLBSTATE_OK * Now the smp_invalidate_interrupt won't call leave_mm if cpu0 * was in lazy tlb mode. * 1a3) update cpu_tlbstate[].active_mm * Now cpu0 accepts tlb flushes for the new mm. * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask); * Now the other cpus will send tlb flush ipis. * 1a4) change cr3. * 1b) thread switch without mm change * cpu_tlbstate[].active_mm is correct, cpu0 already handles * flush ipis. * 1b1) set cpu_tlbstate to TLBSTATE_OK * 1b2) test_and_set the cpu bit in cpu_vm_mask. * Atomically set the bit [other cpus will start sending flush ipis], * and test the bit. * 1b3) if the bit was 0: leave_mm was called, flush the tlb. * 2) switch %%esp, ie current * * The interrupt must handle 2 special cases: * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm. * - the cpu performs speculative tlb reads, i.e. even if the cpu only * runs in kernel space, the cpu could load tlb entries for user space * pages. * * The good news is that cpu_tlbstate is local to each cpu, no * write/read ordering problems. */ /* * TLB flush IPI: * * 1) Flush the tlb entries if the cpu uses the mm that's being flushed. * 2) Leave the mm if we are in the lazy tlb mode. */ fastcall void smp_invalidate_interrupt(struct pt_regs *regs) { unsigned long cpu; cpu = get_cpu(); if (!cpu_isset(cpu, flush_cpumask)) goto out; /* * This was a BUG() but until someone can quote me the * line from the intel manual that guarantees an IPI to * multiple CPUs is retried _only_ on the erroring CPUs * its staying as a return * * BUG(); */ if (flush_mm == per_cpu(cpu_tlbstate, cpu).active_mm) { if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) { if (flush_va == FLUSH_ALL) local_flush_tlb(); else __flush_tlb_one(flush_va); } else leave_mm(cpu); } ack_APIC_irq(); smp_mb__before_clear_bit(); cpu_clear(cpu, flush_cpumask); smp_mb__after_clear_bit(); out: put_cpu_no_resched(); } static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, unsigned long va) { /* * A couple of (to be removed) sanity checks: * * - current CPU must not be in mask * - mask must exist :) */ BUG_ON(cpus_empty(cpumask)); BUG_ON(cpu_isset(smp_processor_id(), cpumask)); BUG_ON(!mm); /* If a CPU which we ran on has gone down, OK. */ cpus_and(cpumask, cpumask, cpu_online_map); if (cpus_empty(cpumask)) return; /* * i'm not happy about this global shared spinlock in the * MM hot path, but we'll see how contended it is. * Temporarily this turns IRQs off, so that lockups are * detected by the NMI watchdog. */ spin_lock(&tlbstate_lock); flush_mm = mm; flush_va = va; #if NR_CPUS <= BITS_PER_LONG atomic_set_mask(cpumask, &flush_cpumask); #else { int k; unsigned long *flush_mask = (unsigned long *)&flush_cpumask; unsigned long *cpu_mask = (unsigned long *)&cpumask; for (k = 0; k < BITS_TO_LONGS(NR_CPUS); ++k) atomic_set_mask(cpu_mask[k], &flush_mask[k]); } #endif /* * We have to send the IPI only to * CPUs affected. */ send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR); while (!cpus_empty(flush_cpumask)) /* nothing. lockup detection does not belong here */ mb(); flush_mm = NULL; flush_va = 0; spin_unlock(&tlbstate_lock); } void flush_tlb_current_task(void) { struct mm_struct *mm = current->mm; cpumask_t cpu_mask; preempt_disable(); cpu_mask = mm->cpu_vm_mask; cpu_clear(smp_processor_id(), cpu_mask); local_flush_tlb(); if (!cpus_empty(cpu_mask)) flush_tlb_others(cpu_mask, mm, FLUSH_ALL); preempt_enable(); } void flush_tlb_mm (struct mm_struct * mm) { cpumask_t cpu_mask; preempt_disable(); cpu_mask = mm->cpu_vm_mask; cpu_clear(smp_processor_id(), cpu_mask); if (current->active_mm == mm) { if (current->mm) local_flush_tlb(); else leave_mm(smp_processor_id()); } if (!cpus_empty(cpu_mask)) flush_tlb_others(cpu_mask, mm, FLUSH_ALL); preempt_enable(); } void flush_tlb_page(struct vm_area_struct * vma, unsigned long va) { struct mm_struct *mm = vma->vm_mm; cpumask_t cpu_mask; preempt_disable(); cpu_mask = mm->cpu_vm_mask; cpu_clear(smp_processor_id(), cpu_mask); if (current->active_mm == mm) { if(current->mm) __flush_tlb_one(va); else leave_mm(smp_processor_id()); } if (!cpus_empty(cpu_mask)) flush_tlb_others(cpu_mask, mm, va); preempt_enable(); } EXPORT_SYMBOL(flush_tlb_page); static void do_flush_tlb_all(void* info) { unsigned long cpu = smp_processor_id(); __flush_tlb_all(); if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_LAZY) leave_mm(cpu); } void flush_tlb_all(void) { on_each_cpu(do_flush_tlb_all, NULL, 1, 1); } /* * this function sends a 'reschedule' IPI to another CPU. * it goes straight through and wastes no time serializing * anything. Worst case is that we lose a reschedule ... */ void smp_send_reschedule(int cpu) { WARN_ON(cpu_is_offline(cpu)); send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR); } /* * Structure and data for smp_call_function(). This is designed to minimise * static memory requirements. It also looks cleaner. */ static DEFINE_SPINLOCK(call_lock); struct call_data_struct { void (*func) (void *info); void *info; atomic_t started; atomic_t finished; int wait; }; void lock_ipi_call_lock(void) { spin_lock_irq(&call_lock); } void unlock_ipi_call_lock(void) { spin_unlock_irq(&call_lock); } static struct call_data_struct *call_data; /** * smp_call_function(): Run a function on all other CPUs. * @func: The function to run. This must be fast and non-blocking. * @info: An arbitrary pointer to pass to the function. * @nonatomic: currently unused. * @wait: If true, wait (atomically) until function has completed on other CPUs. * * Returns 0 on success, else a negative status code. Does not return until * remote CPUs are nearly ready to execute <<func>> or are or have executed. * * You must not call this function with disabled interrupts or from a * hardware interrupt handler or from a bottom half handler. */ int smp_call_function (void (*func) (void *info), void *info, int nonatomic, int wait) { struct call_data_struct data; int cpus; /* Holding any lock stops cpus from going down. */ spin_lock(&call_lock); cpus = num_online_cpus() - 1; if (!cpus) { spin_unlock(&call_lock); return 0; } /* Can deadlock when called with interrupts disabled */ WARN_ON(irqs_disabled()); data.func = func; data.info = info; atomic_set(&data.started, 0); data.wait = wait; if (wait) atomic_set(&data.finished, 0); call_data = &data; mb(); /* Send a message to all other CPUs and wait for them to respond */ send_IPI_allbutself(CALL_FUNCTION_VECTOR); /* Wait for response */ while (atomic_read(&data.started) != cpus) cpu_relax(); if (wait) while (atomic_read(&data.finished) != cpus) cpu_relax(); spin_unlock(&call_lock); return 0; } EXPORT_SYMBOL(smp_call_function); void stop_this_cpu (void * dummy) { /* * Remove this CPU: */ cpu_clear(smp_processor_id(), cpu_online_map); local_irq_disable(); disable_local_APIC(); if (cpu_data[smp_processor_id()].hlt_works_ok) for(;;) halt(); for (;;); } /* * this function calls the 'stop' function on all other CPUs in the system. */ void smp_send_stop(void) { smp_call_function(stop_this_cpu, NULL, 1, 0); local_irq_disable(); disable_local_APIC(); local_irq_enable(); } /* * Reschedule call back. Nothing to do, * all the work is done automatically when * we return from the interrupt. */ fastcall void smp_reschedule_interrupt(struct pt_regs *regs) { ack_APIC_irq(); } fastcall void smp_call_function_interrupt(struct pt_regs *regs) { void (*func) (void *info) = call_data->func; void *info = call_data->info; int wait = call_data->wait; ack_APIC_irq(); /* * Notify initiating CPU that I've grabbed the data and am * about to execute the function */ mb(); atomic_inc(&call_data->started); /* * At this point the info structure may be out of scope unless wait==1 */ irq_enter(); (*func)(info); irq_exit(); if (wait) { mb(); atomic_inc(&call_data->finished); } } #ifdef CONFIG_DUMP_MKEXEC /* Save the state of all the other CPUs. */ /* * Do not move the function to mkexec-module as send_IPI_allbutself() * requires machine type specific SYMBOL_EXPORT()s. */ void mkdump_send_nmi(void) { send_IPI_allbutself(APIC_DM_NMI); } EXPORT_SYMBOL(stop_this_cpu); #endif /* CONFIG_DUMP_MKEXEC */ |