diff --git a/headers/private/kernel/arch/smp.h b/headers/private/kernel/arch/smp.h index f85fcf136f..440617bd1a 100644 --- a/headers/private/kernel/arch/smp.h +++ b/headers/private/kernel/arch/smp.h @@ -10,6 +10,8 @@ struct kernel_args; +class CPUSet; + #ifdef __cplusplus extern "C" { @@ -19,6 +21,7 @@ status_t arch_smp_init(struct kernel_args *args); status_t arch_smp_per_cpu_init(struct kernel_args *args, int32 cpu); void arch_smp_send_ici(int32 target_cpu); void arch_smp_send_broadcast_ici(void); +void arch_smp_send_multicast_ici(CPUSet& cpuSet); #ifdef __cplusplus } diff --git a/headers/private/kernel/arch/x86/apic.h b/headers/private/kernel/arch/x86/apic.h index 9c3d92284a..a9dbed7852 100644 --- a/headers/private/kernel/arch/x86/apic.h +++ b/headers/private/kernel/arch/x86/apic.h @@ -52,7 +52,7 @@ #define APIC_TRIGGER_MODE_LEVEL (1 << 15) /* Interrupt Command defines */ -#define APIC_INTR_COMMAND_1_MASK 0xfff3f000 +#define APIC_INTR_COMMAND_1_MASK 0xfff32000 #define APIC_INTR_COMMAND_2_MASK 0x00ffffff #define APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL 0 @@ -110,6 +110,7 @@ #if !_BOOT_MODE bool apic_available(); +bool x2apic_available(); uint32 apic_read(uint32 offset); void apic_write(uint32 offset, uint32 data); uint32 apic_local_id(); @@ -122,10 +123,9 @@ void apic_disable_local_ints(); uint32 apic_spurious_intr_vector(); void apic_set_spurious_intr_vector(uint32 config); -uint32 apic_intr_command_1(); -void apic_set_intr_command_1(uint32 config); -uint32 apic_intr_command_2(); -void apic_set_intr_command_2(uint32 config); + +void apic_set_interrupt_command(uint32 destination, uint32 mode); +bool apic_interrupt_delivered(void); uint32 apic_lvt_timer(); void apic_set_lvt_timer(uint32 config); diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index 516b5fc52d..f745c50c6d 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -326,6 +326,8 @@ typedef struct arch_cpu_info { int model; int extended_model; + uint32 logical_apic_id; + struct X86PagingStructures* active_paging_structures; size_t dr6; // temporary storage for debug registers (cf. diff --git a/src/system/kernel/arch/x86/apic.cpp b/src/system/kernel/arch/x86/apic.cpp index 47587063a3..c6719cc23d 100644 --- a/src/system/kernel/arch/x86/apic.cpp +++ b/src/system/kernel/arch/x86/apic.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include "timers/apic_timer.h" @@ -30,6 +31,12 @@ apic_available() } +bool x2apic_available() +{ + return sX2APIC; +} + + uint32 apic_read(uint32 offset) { @@ -94,6 +101,16 @@ apic_end_of_interrupt() } +uint32 +apic_logical_apic_id() +{ + if (sX2APIC) + return x86_read_msr(IA32_MSR_APIC_LOGICAL_DEST); + else + return apic_read(APIC_LOGICAL_DEST); +} + + void apic_disable_local_ints() { @@ -128,49 +145,36 @@ apic_set_spurious_intr_vector(uint32 config) } -uint32 -apic_intr_command_1() -{ - if (sX2APIC) - return x86_read_msr(IA32_MSR_APIC_INTR_COMMAND) & 0xffffffff; - else - return apic_read(APIC_INTR_COMMAND_1); -} - - void -apic_set_intr_command_1(uint32 config) +apic_set_interrupt_command(uint32 destination, uint32 mode) { if (sX2APIC) { - uint64 value = x86_read_msr(IA32_MSR_APIC_INTR_COMMAND); - arch_cpu_memory_read_write_barrier(); - x86_write_msr(IA32_MSR_APIC_INTR_COMMAND, - (value & 0xffffffff00000000LL) | config); - } else - apic_write(APIC_INTR_COMMAND_1, config); + uint64 command = x86_read_msr(IA32_MSR_APIC_INTR_COMMAND); + command &= APIC_INTR_COMMAND_1_MASK; + command |= (uint64)destination << 32; + command |= mode; + x86_write_msr(IA32_MSR_APIC_INTR_COMMAND, command); + } else { + uint32 command2 = apic_read(APIC_INTR_COMMAND_2) + & APIC_INTR_COMMAND_2_MASK; + command2 |= destination << 24; + apic_write(APIC_INTR_COMMAND_2, command2); + + uint32 command1 = apic_read(APIC_INTR_COMMAND_1) + & APIC_INTR_COMMAND_1_MASK; + command1 |= mode; + apic_write(APIC_INTR_COMMAND_1, command1); + } } -uint32 -apic_intr_command_2() +bool +apic_interrupt_delivered(void) { if (sX2APIC) - return x86_read_msr(IA32_MSR_APIC_INTR_COMMAND) >> 32; + return true; else - return apic_read(APIC_INTR_COMMAND_2); -} - - -void -apic_set_intr_command_2(uint32 config) -{ - if (sX2APIC) { - uint64 value = x86_read_msr(IA32_MSR_APIC_INTR_COMMAND); - arch_cpu_memory_read_write_barrier(); - x86_write_msr(IA32_MSR_APIC_INTR_COMMAND, - (value & 0xffffffff) | ((uint64)config << 32)); - } else - apic_write(APIC_INTR_COMMAND_2, config); + return (apic_read(APIC_INTR_COMMAND_1) & APIC_DELIVERY_STATUS) == 0; } @@ -264,11 +268,6 @@ apic_init(kernel_args *args) dprintf("found x2apic\n"); #if 0 if (!get_safemode_boolean(B_SAFEMODE_DISABLE_X2APIC, false)) { - uint64 apic_base = x86_read_msr(IA32_MSR_APIC_BASE); - if ((apic_base & IA32_MSR_APIC_BASE_X2APIC) == 0) { - x86_write_msr(IA32_MSR_APIC_BASE, apic_base - | IA32_MSR_APIC_BASE_X2APIC); - } sX2APIC = true; return B_OK; } @@ -276,11 +275,6 @@ apic_init(kernel_args *args) dprintf("x2apic disabled per safemode setting\n"); #else if (get_safemode_boolean(B_SAFEMODE_ENABLE_X2APIC, false)) { - uint64 apic_base = x86_read_msr(IA32_MSR_APIC_BASE); - if ((apic_base & IA32_MSR_APIC_BASE_X2APIC) == 0) { - x86_write_msr(IA32_MSR_APIC_BASE, apic_base - | IA32_MSR_APIC_BASE_X2APIC); - } sX2APIC = true; dprintf("x2apic enabled per safemode setting\n"); @@ -306,8 +300,33 @@ apic_init(kernel_args *args) status_t apic_per_cpu_init(kernel_args *args, int32 cpu) { - dprintf("setting up apic for CPU %" B_PRId32 ": apic id %" B_PRIu32 ", " - "version %" B_PRIu32 "\n", cpu, apic_local_id(), apic_version()); + if (sX2APIC) { + uint64 apic_base = x86_read_msr(IA32_MSR_APIC_BASE); + if ((apic_base & IA32_MSR_APIC_BASE_X2APIC) == 0) { + x86_write_msr(IA32_MSR_APIC_BASE, apic_base + | IA32_MSR_APIC_BASE_X2APIC); + } + } + + dprintf("setting up %sapic for CPU %" B_PRId32 ": apic id %" B_PRIu32 ", " + "version %" B_PRIu32 "\n", sX2APIC ? "x2" : "", cpu, apic_local_id(), + apic_version()); + + if (!sX2APIC && cpu < 8) { + apic_write(APIC_DEST_FORMAT, uint32(-1)); + + uint8 logical_apic_id = 1 << cpu; + uint32 value = apic_read(APIC_LOGICAL_DEST); + value &= 0xffffff; + apic_write(APIC_LOGICAL_DEST, value | (logical_apic_id << 24)); + } + + // get logical APIC ID + gCPU[cpu].arch.logical_apic_id = apic_logical_apic_id(); + if (!sX2APIC) + gCPU[cpu].arch.logical_apic_id >>= 24; + dprintf("CPU %" B_PRId32 ": logical apic id: %#" B_PRIx32 "\n", cpu, + gCPU[cpu].arch.logical_apic_id); /* set spurious interrupt vector to 0xff */ uint32 config = apic_spurious_intr_vector() & 0xffffff00; diff --git a/src/system/kernel/arch/x86/arch_smp.cpp b/src/system/kernel/arch/x86/arch_smp.cpp index adf89a247c..bb95501f38 100644 --- a/src/system/kernel/arch/x86/arch_smp.cpp +++ b/src/system/kernel/arch/x86/arch_smp.cpp @@ -1,4 +1,5 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2002-2005, Axel Dörfler, axeld@pinc-software.de. All rights reserved. * Distributed under the terms of the MIT License. * @@ -25,6 +26,8 @@ #include #include +#include + //#define TRACE_ARCH_SMP #ifdef TRACE_ARCH_SMP @@ -34,6 +37,9 @@ #endif +#define ICI_VECTOR 0xfd + + static uint32 sCPUAPICIds[SMP_MAX_CPUS]; static uint32 sAPICVersions[SMP_MAX_CPUS]; @@ -124,47 +130,95 @@ arch_smp_per_cpu_init(kernel_args *args, int32 cpu) } +void +arch_smp_send_multicast_ici(CPUSet& cpuSet) +{ +#if KDEBUG + if (are_interrupts_enabled()) + panic("arch_smp_send_multicast_ici: called with interrupts enabled"); +#endif + + arch_cpu_memory_write_barrier(); + + int32 i = 0; + int32 cpuCount = smp_get_num_cpus(); + + int32 logicalModeCPUs; + if (x2apic_available()) + logicalModeCPUs = cpuCount; + else + logicalModeCPUs = std::min(cpuCount, int32(8)); + + uint32 destination = 0; + for (; i < logicalModeCPUs; i++) { + if (cpuSet.GetBit(i) && i != smp_get_current_cpu()) + destination |= gCPU[i].arch.logical_apic_id; + } + + uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED + | APIC_INTR_COMMAND_1_ASSERT + | APIC_INTR_COMMAND_1_DEST_MODE_LOGICAL + | APIC_INTR_COMMAND_1_DEST_FIELD; + + while (!apic_interrupt_delivered()) + cpu_pause(); + apic_set_interrupt_command(destination, mode); + + for (; i < cpuCount; i++) { + if (cpuSet.GetBit(i)) { + uint32 destination = sCPUAPICIds[i]; + uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED + | APIC_INTR_COMMAND_1_ASSERT + | APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL + | APIC_INTR_COMMAND_1_DEST_FIELD; + + while (!apic_interrupt_delivered()) + cpu_pause(); + apic_set_interrupt_command(destination, mode); + } + } +} + + void arch_smp_send_broadcast_ici(void) { - uint32 config; - cpu_status state = disable_interrupts(); +#if KDEBUG + if (are_interrupts_enabled()) + panic("arch_smp_send_broadcast_ici: called with interrupts enabled"); +#endif - config = apic_intr_command_1() & APIC_INTR_COMMAND_1_MASK; - apic_set_intr_command_1(config | 0xfd | APIC_DELIVERY_MODE_FIXED - | APIC_INTR_COMMAND_1_ASSERT - | APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL - | APIC_INTR_COMMAND_1_DEST_ALL_BUT_SELF); + arch_cpu_memory_write_barrier(); - restore_interrupts(state); + uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED + | APIC_INTR_COMMAND_1_ASSERT + | APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL + | APIC_INTR_COMMAND_1_DEST_ALL_BUT_SELF; + + while (!apic_interrupt_delivered()) + cpu_pause(); + apic_set_interrupt_command(0, mode); } void arch_smp_send_ici(int32 target_cpu) { - uint32 config; - uint32 timeout; - cpu_status state; +#if KDEBUG + if (are_interrupts_enabled()) + panic("arch_smp_send_ici: called with interrupts enabled"); +#endif - state = disable_interrupts(); + arch_cpu_memory_write_barrier(); - config = apic_intr_command_2() & APIC_INTR_COMMAND_2_MASK; - apic_set_intr_command_2(config | sCPUAPICIds[target_cpu] << 24); + uint32 destination = sCPUAPICIds[target_cpu]; + uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED + | APIC_INTR_COMMAND_1_ASSERT + | APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL + | APIC_INTR_COMMAND_1_DEST_FIELD; - config = apic_intr_command_1() & APIC_INTR_COMMAND_1_MASK; - apic_set_intr_command_1(config | 0xfd | APIC_DELIVERY_MODE_FIXED - | APIC_INTR_COMMAND_1_ASSERT - | APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL - | APIC_INTR_COMMAND_1_DEST_FIELD); - - timeout = 100000000; - // wait for message to be sent - while ((apic_intr_command_1() & APIC_DELIVERY_STATUS) != 0 && --timeout != 0) - asm volatile ("pause;"); - - if (timeout == 0) - panic("arch_smp_send_ici: timeout, target_cpu %" B_PRId32, target_cpu); - - restore_interrupts(state); + while (!apic_interrupt_delivered()) + cpu_pause(); + apic_set_interrupt_command(destination, mode); } + diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index bc7f58323b..780912abc8 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -1112,11 +1112,16 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, if (!sICIEnabled) return; + int currentCPU = smp_get_current_cpu(); + bool broadcast = true; + // count target CPUs int32 targetCPUs = 0; for (int32 i = 0; i < sNumCPUs; i++) { if (cpuMask.GetBit(i)) targetCPUs++; + else if (i != currentCPU) + broadcast = false; } // find_free_message leaves interrupts disabled @@ -1132,7 +1137,6 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, msg->flags = flags; msg->done = 0; - int currentCPU = smp_get_current_cpu(); msg->proc_bitmap = cpuMask; msg->proc_bitmap.ClearBit(currentCPU); if (msg->proc_bitmap.IsEmpty()) { @@ -1153,8 +1157,10 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, atomic_add(&gCPU[i].ici_counter, 1); } - arch_smp_send_broadcast_ici(); - // TODO: Introduce a call that only bothers the target CPUs! + if (broadcast) + arch_smp_send_broadcast_ici(); + else + arch_smp_send_multicast_ici(cpuMask); if ((flags & SMP_MSG_FLAG_SYNC) != 0) { // wait for the other cpus to finish processing it