diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index f65b9b59b7..6c94982f37 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -275,13 +275,15 @@ uint32 x86_read_cr4(); void x86_write_cr4(uint32 value); uint64 x86_read_msr(uint32 registerNumber); void x86_write_msr(uint32 registerNumber, uint64 value); -void x86_set_task_gate(int32 n, int32 segment); +void x86_set_task_gate(int32 cpu, int32 n, int32 segment); +void* x86_get_idt(int32 cpu); uint32 x86_count_mtrrs(void); void x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type); status_t x86_get_mtrr(uint32 index, uint64 *_base, uint64 *_length, uint8 *_type); bool x86_check_feature(uint32 feature, enum x86_feature_type type); void* x86_get_double_fault_stack(int32 cpu, size_t* _size); -int x86_double_fault_get_cpu(); +int32 x86_double_fault_get_cpu(void); +void x86_double_fault_exception(struct iframe* frame); #define read_cr3(value) \ diff --git a/headers/private/kernel/debug.h b/headers/private/kernel/debug.h index 1b748de624..83b807e6aa 100644 --- a/headers/private/kernel/debug.h +++ b/headers/private/kernel/debug.h @@ -120,7 +120,8 @@ extern void debug_stop_screen_debug_output(void); extern void debug_set_page_fault_info(addr_t faultAddress, addr_t pc, uint32 flags); extern debug_page_fault_info* debug_get_page_fault_info(); -extern void debug_trap_cpu_in_kdl(bool returnIfHandedOver); +extern void debug_trap_cpu_in_kdl(int32 cpu, bool returnIfHandedOver); +extern void debug_double_fault(int32 cpu); extern bool debug_emergency_key_pressed(char key); extern char kgetc(void); diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 255e61cfad..ea5dd6771f 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -52,12 +52,14 @@ void smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, uint32 data, uint32 data2, uint32 data3, void *data_ptr, uint32 flags); void smp_send_broadcast_ici(int32 message, uint32 data, uint32 data2, uint32 data3, void *data_ptr, uint32 flags); +void smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, + uint32 data, uint32 data2, uint32 data3, void *data_ptr, uint32 flags); int32 smp_get_num_cpus(void); void smp_set_num_cpus(int32 numCPUs); int32 smp_get_current_cpu(void); -int smp_intercpu_int_handler(void); +int smp_intercpu_int_handler(int32 cpu); #ifdef __cplusplus } diff --git a/src/system/kernel/arch/x86/Jamfile b/src/system/kernel/arch/x86/Jamfile index cc705b95e7..50c05f700c 100644 --- a/src/system/kernel/arch/x86/Jamfile +++ b/src/system/kernel/arch/x86/Jamfile @@ -16,7 +16,7 @@ KernelMergeObject kernel_arch_x86.o : arch_debug.cpp arch_debug_console.c arch_elf.c - arch_int.c + arch_int.cpp arch_platform.c # arch_selector.c arch_real_time_clock.c diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index ca18cb683a..d9048eaa9f 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -269,8 +269,11 @@ init_double_fault(int cpuNum) tss->io_map_base = sizeof(struct tss); // add TSS descriptor for this new TSS - set_tss_descriptor(&gGDT[DOUBLE_FAULT_TSS_BASE_SEGMENT + cpuNum], + uint16 tssSegmentDescriptorIndex = DOUBLE_FAULT_TSS_BASE_SEGMENT + cpuNum; + set_tss_descriptor(&gGDT[tssSegmentDescriptorIndex], (addr_t)tss, sizeof(struct tss)); + + x86_set_task_gate(cpuNum, 8, tssSegmentDescriptorIndex << 3); } @@ -511,8 +514,8 @@ x86_get_double_fault_stack(int32 cpu, size_t* _size) /*! Returns the index of the current CPU. Can only be called from the double fault handler. */ -int -x86_double_fault_get_cpu() +int32 +x86_double_fault_get_cpu(void) { uint32 stack = x86_read_ebp(); return (stack - (uint32)sDoubleFaultStacks) / kDoubleFaultStackSize; @@ -561,9 +564,21 @@ arch_cpu_init_percpu(kernel_args *args, int cpu) // load the TSS for this cpu // note the main cpu gets initialized in arch_cpu_init_post_vm() - if (cpu != 0) + if (cpu != 0) { load_tss(cpu); + // set the IDT + struct { + uint16 limit; + void* address; + } _PACKED descriptor = { + 256 * 8 - 1, // 256 descriptors, 8 bytes each (-1 for "limit") + x86_get_idt(cpu) + }; + + asm volatile("lidt %0" : : "m"(descriptor)); + } + return 0; } @@ -621,8 +636,6 @@ arch_cpu_init_post_vm(kernel_args *args) // set the current hardware task on cpu 0 load_tss(0); - x86_set_task_gate(8, DOUBLE_FAULT_TSS_BASE_SEGMENT << 3); - // setup TLS descriptors (one for every CPU) for (i = 0; i < args->num_cpus; i++) { diff --git a/src/system/kernel/arch/x86/arch_int.c b/src/system/kernel/arch/x86/arch_int.cpp similarity index 85% rename from src/system/kernel/arch/x86/arch_int.c rename to src/system/kernel/arch/x86/arch_int.cpp index 5bc57cd660..607f1a8a0d 100644 --- a/src/system/kernel/arch/x86/arch_int.c +++ b/src/system/kernel/arch/x86/arch_int.cpp @@ -1,4 +1,5 @@ /* + * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de. * Copyright 2002-2008, Axel Dörfler, axeld@pinc-software.de. * Distributed under the terms of the MIT License. * @@ -174,7 +175,7 @@ static const int kInterruptNameCount = 20; typedef struct { uint32 a, b; } desc_table; -static desc_table *sIDT = NULL; +static desc_table* sIDTs[B_MAX_CPU_COUNT]; static uint32 sLevelTriggeredInterrupts = 0; // binary mask: 1 level, 0 edge @@ -186,6 +187,8 @@ interrupt_handler_function* gInterruptHandlerTable[ INTERRUPT_HANDLER_TABLE_SIZE]; +/*! Initializes a descriptor in an IDT. +*/ static void set_gate(desc_table *gate_addr, addr_t addr, int type, int dpl) { @@ -200,25 +203,45 @@ set_gate(desc_table *gate_addr, addr_t addr, int type, int dpl) } +/*! Initializes the descriptor for interrupt vector \a n in the IDT of the + boot CPU to an interrupt-gate descriptor with the given procedure address. +*/ static void -set_intr_gate(int n, void *addr) +set_interrupt_gate(int n, void (*addr)()) { - set_gate(&sIDT[n], (addr_t)addr, 14, DPL_KERNEL); + set_gate(&sIDTs[0][n], (addr_t)addr, 14, DPL_KERNEL); } +/*! Initializes the descriptor for interrupt vector \a n in the IDT of the + boot CPU to an trap-gate descriptor with the given procedure address. +*/ static void -set_system_gate(int n, void *addr) +set_trap_gate(int n, void (*addr)()) { - set_gate(&sIDT[n], (unsigned int)addr, 15, DPL_USER); + set_gate(&sIDTs[0][n], (unsigned int)addr, 15, DPL_USER); } +/*! Initializes the descriptor for interrupt vector \a n in the IDT of CPU + \a cpu to a task-gate descripter referring to the TSS segment identified + by TSS segment selector \a segment. + For CPUs other than the boot CPU it must not be called before + arch_int_init_post_vm() (arch_cpu_init_post_vm() is fine). +*/ void -x86_set_task_gate(int32 n, int32 segment) +x86_set_task_gate(int32 cpu, int32 n, int32 segment) { - sIDT[n].a = (segment << 16); - sIDT[n].b = 0x8000 | (0 << 13) | (0x5 << 8); // present, dpl 0, type 5 + sIDTs[cpu][n].a = (segment << 16); + sIDTs[cpu][n].b = 0x8000 | (0 << 13) | (0x5 << 8); // present, dpl 0, type 5 +} + + +/*! Returns the virtual IDT address for CPU \a cpu. */ +void* +x86_get_idt(int32 cpu) +{ + return sIDTs[cpu]; } @@ -446,7 +469,7 @@ ioapic_enable_io_interrupt(int32 num) { uint64 entry; int32 pin = sIRQToIOAPICPin[num]; - if (pin < 0 || pin > sIOAPICMaxRedirectionEntry) + if (pin < 0 || pin > (int32)sIOAPICMaxRedirectionEntry) return; TRACE(("ioapic_enable_io_interrupt: IRQ %ld -> pin %ld\n", num, pin)); @@ -463,7 +486,7 @@ ioapic_disable_io_interrupt(int32 num) { uint64 entry; int32 pin = sIRQToIOAPICPin[num]; - if (pin < 0 || pin > sIOAPICMaxRedirectionEntry) + if (pin < 0 || pin > (int32)sIOAPICMaxRedirectionEntry) return; TRACE(("ioapic_disable_io_interrupt: IRQ %ld -> pin %ld\n", num, pin)); @@ -480,7 +503,7 @@ ioapic_configure_io_interrupt(int32 num, uint32 config) { uint64 entry; int32 pin = sIRQToIOAPICPin[num]; - if (pin < 0 || pin > sIOAPICMaxRedirectionEntry) + if (pin < 0 || pin > (int32)sIOAPICMaxRedirectionEntry) return; TRACE(("ioapic_configure_io_interrupt: IRQ %ld -> pin %ld; config 0x%08lx\n", @@ -799,16 +822,18 @@ unexpected_exception(struct iframe* frame) } -static void -double_fault_exception(struct iframe* frame) +void +x86_double_fault_exception(struct iframe* frame) { + int cpu = x86_double_fault_get_cpu(); + // The double fault iframe contains no useful information (as // per Intel's architecture spec). Thus we simply save the - // information from the (unhandable) exception which caused the + // information from the (unhandlable) exception which caused the // double in our iframe. This will result even in useful stack // traces. Only problem is that we trust that at least the // TSS is still accessible. - struct tss *tss = &gCPU[smp_get_current_cpu()].arch.tss; + struct tss *tss = &gCPU[cpu].arch.tss; frame->cs = tss->cs; frame->es = tss->es; @@ -826,7 +851,7 @@ double_fault_exception(struct iframe* frame) frame->edi = tss->edi; frame->flags = tss->eflags; - panic("double fault!\n"); + debug_double_fault(cpu); } @@ -941,65 +966,65 @@ arch_int_init(struct kernel_args *args) interrupt_handler_function** table; // set the global sIDT variable - sIDT = (desc_table *)args->arch_args.vir_idt; + sIDTs[0] = (desc_table *)args->arch_args.vir_idt; // setup the standard programmable interrupt controller pic_init(); - set_intr_gate(0, &trap0); - set_intr_gate(1, &trap1); - set_intr_gate(2, &trap2); - set_system_gate(3, &trap3); - set_intr_gate(4, &trap4); - set_intr_gate(5, &trap5); - set_intr_gate(6, &trap6); - set_intr_gate(7, &trap7); + set_interrupt_gate(0, &trap0); + set_interrupt_gate(1, &trap1); + set_interrupt_gate(2, &trap2); + set_trap_gate(3, &trap3); + set_interrupt_gate(4, &trap4); + set_interrupt_gate(5, &trap5); + set_interrupt_gate(6, &trap6); + set_interrupt_gate(7, &trap7); // trap8 (double fault) is set in arch_cpu.c - set_intr_gate(9, &trap9); - set_intr_gate(10, &trap10); - set_intr_gate(11, &trap11); - set_intr_gate(12, &trap12); - set_intr_gate(13, &trap13); - set_intr_gate(14, &trap14); -// set_intr_gate(15, &trap15); - set_intr_gate(16, &trap16); - set_intr_gate(17, &trap17); - set_intr_gate(18, &trap18); - set_intr_gate(19, &trap19); + set_interrupt_gate(9, &trap9); + set_interrupt_gate(10, &trap10); + set_interrupt_gate(11, &trap11); + set_interrupt_gate(12, &trap12); + set_interrupt_gate(13, &trap13); + set_interrupt_gate(14, &trap14); +// set_interrupt_gate(15, &trap15); + set_interrupt_gate(16, &trap16); + set_interrupt_gate(17, &trap17); + set_interrupt_gate(18, &trap18); + set_interrupt_gate(19, &trap19); - set_intr_gate(32, &trap32); - set_intr_gate(33, &trap33); - set_intr_gate(34, &trap34); - set_intr_gate(35, &trap35); - set_intr_gate(36, &trap36); - set_intr_gate(37, &trap37); - set_intr_gate(38, &trap38); - set_intr_gate(39, &trap39); - set_intr_gate(40, &trap40); - set_intr_gate(41, &trap41); - set_intr_gate(42, &trap42); - set_intr_gate(43, &trap43); - set_intr_gate(44, &trap44); - set_intr_gate(45, &trap45); - set_intr_gate(46, &trap46); - set_intr_gate(47, &trap47); - set_intr_gate(48, &trap48); - set_intr_gate(49, &trap49); - set_intr_gate(50, &trap50); - set_intr_gate(51, &trap51); - set_intr_gate(52, &trap52); - set_intr_gate(53, &trap53); - set_intr_gate(54, &trap54); - set_intr_gate(55, &trap55); + set_interrupt_gate(32, &trap32); + set_interrupt_gate(33, &trap33); + set_interrupt_gate(34, &trap34); + set_interrupt_gate(35, &trap35); + set_interrupt_gate(36, &trap36); + set_interrupt_gate(37, &trap37); + set_interrupt_gate(38, &trap38); + set_interrupt_gate(39, &trap39); + set_interrupt_gate(40, &trap40); + set_interrupt_gate(41, &trap41); + set_interrupt_gate(42, &trap42); + set_interrupt_gate(43, &trap43); + set_interrupt_gate(44, &trap44); + set_interrupt_gate(45, &trap45); + set_interrupt_gate(46, &trap46); + set_interrupt_gate(47, &trap47); + set_interrupt_gate(48, &trap48); + set_interrupt_gate(49, &trap49); + set_interrupt_gate(50, &trap50); + set_interrupt_gate(51, &trap51); + set_interrupt_gate(52, &trap52); + set_interrupt_gate(53, &trap53); + set_interrupt_gate(54, &trap54); + set_interrupt_gate(55, &trap55); - set_system_gate(98, &trap98); // for performance testing only - set_system_gate(99, &trap99); + set_trap_gate(98, &trap98); // for performance testing only + set_trap_gate(99, &trap99); - set_intr_gate(251, &trap251); - set_intr_gate(252, &trap252); - set_intr_gate(253, &trap253); - set_intr_gate(254, &trap254); - set_intr_gate(255, &trap255); + set_interrupt_gate(251, &trap251); + set_interrupt_gate(252, &trap252); + set_interrupt_gate(253, &trap253); + set_interrupt_gate(254, &trap254); + set_interrupt_gate(255, &trap255); // init interrupt handler table table = gInterruptHandlerTable; @@ -1018,7 +1043,7 @@ arch_int_init(struct kernel_args *args) table[5] = unexpected_exception; // BOUND Range Exceeded Exception (#BR) table[6] = unexpected_exception; // Invalid Opcode Exception (#UD) table[7] = fatal_exception; // Device Not Available Exception (#NM) - table[8] = double_fault_exception; // Double Fault Exception (#DF) + table[8] = x86_double_fault_exception; // Double Fault Exception (#DF) table[9] = fatal_exception; // Coprocessor Segment Overrun table[10] = fatal_exception; // Invalid TSS Exception (#TS) table[11] = fatal_exception; // Segment Not Present (#NP) @@ -1037,13 +1062,33 @@ arch_int_init(struct kernel_args *args) status_t arch_int_init_post_vm(struct kernel_args *args) { - area_id area; - ioapic_init(args); - sIDT = (desc_table *)args->arch_args.vir_idt; - area = create_area("idt", (void *)&sIDT, B_EXACT_ADDRESS, B_PAGE_SIZE, B_ALREADY_WIRED, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + // create IDT area for the boot CPU + area_id area = create_area("idt", (void**)&sIDTs[0], B_EXACT_ADDRESS, + B_PAGE_SIZE, B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + if (area < 0) + return area; + + // create IDTs for the off-boot CPU + size_t idtSize = 256 * 8; + // 256 8 bytes-sized descriptors + int32 cpuCount = smp_get_num_cpus(); + if (cpuCount > 0) { + size_t areaSize = ROUNDUP(cpuCount * idtSize, B_PAGE_SIZE); + desc_table* idt; + area = create_area("idt", (void**)&idt, B_ANY_KERNEL_ADDRESS, + areaSize, B_CONTIGUOUS, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + if (area < 0) + return area; + + for (int32 i = 1; i < cpuCount; i++) { + sIDTs[i] = idt; + memcpy(idt, sIDTs[0], idtSize); + idt += 256; + // The CPU's IDTR will be set in arch_cpu_init_percpu(). + } + } return area >= B_OK ? B_OK : area; } diff --git a/src/system/kernel/arch/x86/arch_interrupts.S b/src/system/kernel/arch/x86/arch_interrupts.S index 09ccc88ff5..59f6869928 100644 --- a/src/system/kernel/arch/x86/arch_interrupts.S +++ b/src/system/kernel/arch/x86/arch_interrupts.S @@ -200,7 +200,17 @@ FUNCTION(double_fault): pushl $8; pushl $-1; pushl $-1; - jmp int_bottom + + PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER) + + movl %esp, %ebp // frame pointer is the iframe + + pushl %ebp + call x86_double_fault_exception + + // Well, there's no returning from a double fault, but maybe a real hacker + // can repair things in KDL. + POP_IFRAME_AND_RETURN() FUNCTION_END(double_fault) TRAP(trap9, 9) diff --git a/src/system/kernel/arch/x86/arch_smp.c b/src/system/kernel/arch/x86/arch_smp.c index 3b9f47cde2..48e1d24ff8 100644 --- a/src/system/kernel/arch/x86/arch_smp.c +++ b/src/system/kernel/arch/x86/arch_smp.c @@ -122,14 +122,15 @@ static int32 i386_ici_interrupt(void *data) { // genuine inter-cpu interrupt - TRACE(("inter-cpu interrupt on cpu %ld\n", smp_get_current_cpu())); + int cpu = smp_get_current_cpu(); + TRACE(("inter-cpu interrupt on cpu %ld\n", cpu)); // if we are not using the IO APIC we need to acknowledge the // interrupt ourselfs if (!gUsingIOAPIC) apic_write(APIC_EOI, 0); - return smp_intercpu_int_handler(); + return smp_intercpu_int_handler(cpu); } diff --git a/src/system/kernel/debug/debug.cpp b/src/system/kernel/debug/debug.cpp index 30f799508d..be9fb96414 100644 --- a/src/system/kernel/debug/debug.cpp +++ b/src/system/kernel/debug/debug.cpp @@ -7,8 +7,10 @@ * Distributed under the terms of the NewOS License. */ + /*! This file contains the debugger and debug output facilities */ + #include "blue_screen.h" #include @@ -662,13 +664,18 @@ kgets(char* buffer, int length) static void -kernel_debugger_loop(void) +kernel_debugger_loop(const char* message, int32 cpu) { int32 previousCPU = sDebuggerOnCPU; - sDebuggerOnCPU = smp_get_current_cpu(); + sDebuggerOnCPU = cpu; DebugAllocPool* allocPool = create_debug_alloc_pool(); + sCurrentKernelDebuggerMessage = message; + + if (message) + kprintf("PANIC: %s\n", message); + kprintf("Welcome to Kernel Debugging Land...\n"); if (struct thread* thread = thread_get_current_thread()) { @@ -741,11 +748,11 @@ kernel_debugger_loop(void) static void -enter_kernel_debugger(const char* message) +enter_kernel_debugger(int32 cpu) { while (atomic_add(&sInDebugger, 1) > 0) { // The debugger is already running, find out where... - if (sDebuggerOnCPU == smp_get_current_cpu()) { + if (sDebuggerOnCPU == cpu) { // We are re-entering the debugger on the same CPU. break; } @@ -755,19 +762,18 @@ enter_kernel_debugger(const char* message) // blocking there until everyone leaves the debugger and we can // try to enter it again. atomic_add(&sInDebugger, -1); - smp_intercpu_int_handler(); + smp_intercpu_int_handler(cpu); } - arch_debug_save_registers(&dbg_register_file[smp_get_current_cpu()][0]); + arch_debug_save_registers(&dbg_register_file[cpu][0]); sPreviousDprintfState = set_dprintf_enabled(true); - if (!gKernelStartup && sDebuggerOnCPU != smp_get_current_cpu() - && smp_get_num_cpus() > 1) { + if (!gKernelStartup && sDebuggerOnCPU != cpu && smp_get_num_cpus() > 1) { // First entry on a MP system, send a halt request to all of the other // CPUs. Should they try to enter the debugger they will be cought in // the loop above. - smp_send_broadcast_ici(SMP_MSG_CPU_HALT, 0, 0, 0, NULL, - SMP_MSG_FLAG_SYNC); + smp_send_broadcast_ici_interrupts_disabled(cpu, SMP_MSG_CPU_HALT, 0, 0, + 0, NULL, SMP_MSG_FLAG_SYNC); } if (sBlueScreenOutput) { @@ -779,11 +785,6 @@ enter_kernel_debugger(const char* message) sDebuggedThread = NULL; - if (message) - kprintf("PANIC: %s\n", message); - - sCurrentKernelDebuggerMessage = message; - // sort the commands sort_debugger_commands(); @@ -821,6 +822,33 @@ hand_over_kernel_debugger() } +static void +kernel_debugger_internal(const char* message, int32 cpu) +{ + while (true) { + if (sHandOverKDLToCPU == cpu) { + sHandOverKDLToCPU = -1; + sHandOverKDL = false; + } else + enter_kernel_debugger(cpu); + + kernel_debugger_loop(message, cpu); + + if (sHandOverKDLToCPU < 0) { + exit_kernel_debugger(); + break; + } + + hand_over_kernel_debugger(); + + debug_trap_cpu_in_kdl(cpu, true); + + if (sHandOverKDLToCPU != cpu) + break; + } +} + + static int cmd_dump_kdl_message(int argc, char** argv) { @@ -1326,12 +1354,10 @@ debug_get_page_fault_info() void -debug_trap_cpu_in_kdl(bool returnIfHandedOver) +debug_trap_cpu_in_kdl(int32 cpu, bool returnIfHandedOver) { InterruptsLocker locker; - int cpu = smp_get_current_cpu(); - // return, if we've been called recursively (we call // smp_intercpu_int_handler() below) if (sCPUTrapped[cpu]) @@ -1344,15 +1370,22 @@ debug_trap_cpu_in_kdl(bool returnIfHandedOver) if (returnIfHandedOver) break; - kernel_debugger(NULL); + kernel_debugger_internal(NULL, cpu); } else - smp_intercpu_int_handler(); + smp_intercpu_int_handler(cpu); } sCPUTrapped[cpu] = false; } +void +debug_double_fault(int32 cpu) +{ + kernel_debugger_internal("Double Fault!\n", cpu); +} + + bool debug_emergency_key_pressed(char key) { @@ -1407,27 +1440,7 @@ kernel_debugger(const char* message) { cpu_status state = disable_interrupts(); - while (true) { - if (sHandOverKDLToCPU == smp_get_current_cpu()) { - sHandOverKDLToCPU = -1; - sHandOverKDL = false; - } else - enter_kernel_debugger(message); - - kernel_debugger_loop(); - - if (sHandOverKDLToCPU < 0) { - exit_kernel_debugger(); - break; - } - - hand_over_kernel_debugger(); - - debug_trap_cpu_in_kdl(true); - - if (sHandOverKDLToCPU != smp_get_current_cpu()) - break; - } + kernel_debugger_internal(message, smp_get_current_cpu()); restore_interrupts(state); } diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index 22e9eb0215..16b817d055 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2008-2009, Ingo Weinhold, ingo_weinhold@gmx.de. * Copyright 2002-2008, Axel Dörfler, axeld@pinc-software.de. * Distributed under the terms of the MIT License. * @@ -315,6 +315,58 @@ acquire_spinlock_nocheck(spinlock *lock) } +/*! Equivalent to acquire_spinlock(), save for currentCPU parameter. */ +static void +acquire_spinlock_cpu(int32 currentCPU, spinlock *lock) +{ +#if DEBUG_SPINLOCKS + if (are_interrupts_enabled()) { + panic("acquire_spinlock_cpu: attempt to acquire lock %p with " + "interrupts enabled", lock); + } +#endif + + if (sNumCPUs > 1) { +#if B_DEBUG_SPINLOCK_CONTENTION + while (atomic_add(&lock->lock, 1) != 0) + process_all_pending_ici(currentCPU); +#else + while (1) { + uint32 count = 0; + while (*lock != 0) { + if (++count == SPINLOCK_DEADLOCK_COUNT) { + panic("acquire_spinlock_cpu(): Failed to acquire spinlock " + "%p for a long time!", lock); + count = 0; + } + + process_all_pending_ici(currentCPU); + PAUSE(); + } + if (atomic_set((int32 *)lock, 1) == 0) + break; + } + +#if DEBUG_SPINLOCKS + push_lock_caller(arch_debug_get_caller(), lock); +#endif +#endif + } else { +#if DEBUG_SPINLOCKS + int32 oldValue; + oldValue = atomic_set((int32 *)lock, 1); + if (oldValue != 0) { + panic("acquire_spinlock_cpu(): attempt to acquire lock %p twice on " + "non-SMP system (last caller: %p, value %ld)", lock, + find_lock_caller(lock), oldValue); + } + + push_lock_caller(arch_debug_get_caller(), lock); +#endif + } +} + + void release_spinlock(spinlock *lock) { @@ -391,6 +443,34 @@ retry: } +/*! Similar to find_free_message(), but expects the interrupts to be disabled + already. +*/ +static void +find_free_message_interrupts_disabled(int32 currentCPU, + struct smp_msg** _message) +{ + TRACE(("find_free_message_interrupts_disabled: entry\n")); + + acquire_spinlock_cpu(currentCPU, &sFreeMessageSpinlock); + while (sFreeMessageCount <= 0) { + release_spinlock(&sFreeMessageSpinlock); + process_all_pending_ici(currentCPU); + PAUSE(); + acquire_spinlock_cpu(currentCPU, &sFreeMessageSpinlock); + } + + *_message = sFreeMessages; + sFreeMessages = (*_message)->next; + sFreeMessageCount--; + + release_spinlock(&sFreeMessageSpinlock); + + TRACE(("find_free_message_interrupts_disabled: returning msg %p\n", + *_message)); +} + + static void return_free_message(struct smp_msg *msg) { @@ -553,6 +633,8 @@ process_pending_ici(int32 currentCPU) } case SMP_MSG_RESCHEDULE_IF_IDLE: { + // TODO: We must not dereference the thread when entering the kernel + // debugger from a double fault. struct thread* thread = thread_get_current_thread(); if (thread->priority == B_IDLE_PRIORITY) thread->cpu->invoke_scheduler = true; @@ -568,7 +650,7 @@ process_pending_ici(int32 currentCPU) // special case for the halt message if (haltCPU) - debug_trap_cpu_in_kdl(false); + debug_trap_cpu_in_kdl(currentCPU, false); return retval; } @@ -620,13 +702,11 @@ spinlock_contention_syscall(const char* subsystem, uint32 function, int -smp_intercpu_int_handler(void) +smp_intercpu_int_handler(int32 cpu) { - int currentCPU = smp_get_current_cpu(); + TRACE(("smp_intercpu_int_handler: entry on cpu %d\n", cpu)); - TRACE(("smp_intercpu_int_handler: entry on cpu %d\n", currentCPU)); - - process_all_pending_ici(currentCPU); + process_all_pending_ici(cpu); TRACE(("smp_intercpu_int_handler: done\n")); @@ -822,6 +902,68 @@ smp_send_broadcast_ici(int32 message, uint32 data, uint32 data2, uint32 data3, } +void +smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, + uint32 data, uint32 data2, uint32 data3, void *data_ptr, uint32 flags) +{ + if (!sICIEnabled) + return; + + TRACE(("smp_send_broadcast_ici_interrupts_disabled: cpu %ld mess 0x%lx, " + "data 0x%lx, data2 0x%lx, data3 0x%lx, ptr %p, flags 0x%lx\n", + currentCPU, message, data, data2, data3, data_ptr, flags)); + + struct smp_msg *msg; + find_free_message_interrupts_disabled(currentCPU, &msg); + + msg->message = message; + msg->data = data; + msg->data2 = data2; + msg->data3 = data3; + msg->data_ptr = data_ptr; + msg->ref_count = sNumCPUs - 1; + msg->flags = flags; + msg->proc_bitmap = SET_BIT(0, currentCPU); + msg->done = false; + + TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: inserting msg %p " + "into broadcast mbox\n", currentCPU, msg)); + + // stick it in the appropriate cpu's mailbox + acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); + msg->next = sBroadcastMessages; + sBroadcastMessages = msg; + release_spinlock(&sBroadcastMessageSpinlock); + + arch_smp_send_broadcast_ici(); + + TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: sent interrupt\n", + currentCPU)); + + if (flags & SMP_MSG_FLAG_SYNC) { + // wait for the other cpus to finish processing it + // the interrupt handler will ref count it to <0 + // if the message is sync after it has removed it from the mailbox + TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: waiting for " + "ack\n", currentCPU)); + + while (msg->done == false) { + process_all_pending_ici(currentCPU); + PAUSE(); + } + + TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: returning " + "message to free list\n", currentCPU)); + + // for SYNC messages, it's our responsibility to put it + // back into the free list + return_free_message(msg); + } + + TRACE(("smp_send_broadcast_ici_interrupts_disabled: done\n")); +} + + bool smp_trap_non_boot_cpus(int32 cpu) {