diff --git a/headers/private/kernel/arch/int.h b/headers/private/kernel/arch/int.h index 9ebe6b0717..6d4afc6f67 100644 --- a/headers/private/kernel/arch/int.h +++ b/headers/private/kernel/arch/int.h @@ -34,6 +34,7 @@ void arch_int_enable_io_interrupt(int irq); void arch_int_disable_io_interrupt(int irq); void arch_int_configure_io_interrupt(int irq, uint32 config); bool arch_int_are_interrupts_enabled(void); +void arch_int_assign_to_cpu(int32 irq, int32 cpu); #ifdef __cplusplus } diff --git a/headers/private/kernel/arch/x86/arch_int.h b/headers/private/kernel/arch/x86/arch_int.h index da4943726d..5de0dcb9d1 100644 --- a/headers/private/kernel/arch/x86/arch_int.h +++ b/headers/private/kernel/arch/x86/arch_int.h @@ -68,6 +68,7 @@ typedef struct interrupt_controller_s { bool (*is_spurious_interrupt)(int32 num); bool (*is_level_triggered_interrupt)(int32 num); bool (*end_of_interrupt)(int32 num); + void (*assign_interrupt_to_cpu)(int32 num, int32 cpu); } interrupt_controller; diff --git a/headers/private/kernel/arch/x86/arch_smp.h b/headers/private/kernel/arch/x86/arch_smp.h index 5b2d654efc..4eef8eb00f 100644 --- a/headers/private/kernel/arch/x86/arch_smp.h +++ b/headers/private/kernel/arch/x86/arch_smp.h @@ -99,4 +99,18 @@ enum { MP_INTR_TYPE_ExtINT, }; + +#ifdef __cplusplus +extern "C" { +#endif + + +uint32 x86_get_cpu_apic_id(int32 cpu); + + +#ifdef __cplusplus +} +#endif + + #endif /* _KERNEL_ARCH_x86_ARCH_SMP_H */ diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index 7d1dcd365a..4a70428350 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -11,6 +11,7 @@ #include +#include #include #include #include @@ -79,8 +80,12 @@ typedef struct cpu_ent { int topology_id[CPU_TOPOLOGY_LEVELS]; int cache_id[CPU_MAX_CACHE_LEVEL]; + // IRQs assigned to this CPU + struct list irqs; + spinlock irqs_lock; + // arch-specific stuff - arch_cpu_info arch; + arch_cpu_info arch; } cpu_ent __attribute__((aligned(64))); diff --git a/headers/private/kernel/int.h b/headers/private/kernel/int.h index 215deb03dc..6922aa0573 100644 --- a/headers/private/kernel/int.h +++ b/headers/private/kernel/int.h @@ -12,6 +12,8 @@ #include #include +#include + // private install_io_interrupt_handler() flags #define B_NO_LOCK_VECTOR 0x100 #define B_NO_HANDLED_INFO 0x200 @@ -28,6 +30,18 @@ enum interrupt_type { INTERRUPT_TYPE_UNKNOWN }; +struct irq_assignment { + list_link link; + uint32 irq; + + spinlock load_lock; + bigtime_t last_measure_time; + bigtime_t last_measure_active; + int32 load; + + int32 cpu; +}; + #ifdef __cplusplus extern "C" { @@ -68,4 +82,6 @@ status_t reserve_io_interrupt_vectors(long count, long startVector, status_t allocate_io_interrupt_vectors(long count, long *startVector); void free_io_interrupt_vectors(long count, long startVector); +void assign_io_interrupt_to_cpu(long vector, int32 cpu); + #endif /* _KERNEL_INT_H */ diff --git a/src/system/kernel/arch/x86/arch_int.cpp b/src/system/kernel/arch/x86/arch_int.cpp index 3591f909e8..abcaf5fa08 100644 --- a/src/system/kernel/arch/x86/arch_int.cpp +++ b/src/system/kernel/arch/x86/arch_int.cpp @@ -389,6 +389,14 @@ arch_int_are_interrupts_enabled(void) } +void +arch_int_assign_to_cpu(int32 irq, int32 cpu) +{ + if (sCurrentPIC->assign_interrupt_to_cpu != NULL) + sCurrentPIC->assign_interrupt_to_cpu(irq, cpu); +} + + status_t arch_int_init(kernel_args* args) { diff --git a/src/system/kernel/arch/x86/arch_smp.cpp b/src/system/kernel/arch/x86/arch_smp.cpp index f9a56ba6c3..340854bb79 100644 --- a/src/system/kernel/arch/x86/arch_smp.cpp +++ b/src/system/kernel/arch/x86/arch_smp.cpp @@ -70,6 +70,14 @@ x86_smp_error_interrupt(void *data) } +uint32 +x86_get_cpu_apic_id(int32 cpu) +{ + ASSERT(cpu >= 0 && cpu < B_MAX_CPU_COUNT); + return sCPUAPICIds[cpu]; +} + + status_t arch_smp_init(kernel_args *args) { diff --git a/src/system/kernel/arch/x86/ioapic.cpp b/src/system/kernel/arch/x86/ioapic.cpp index 87dd94c48c..3f6c5448f6 100644 --- a/src/system/kernel/arch/x86/ioapic.cpp +++ b/src/system/kernel/arch/x86/ioapic.cpp @@ -18,6 +18,7 @@ #include #include +#include #include // to gain access to the ACPICA types @@ -26,9 +27,9 @@ //#define TRACE_IOAPIC #ifdef TRACE_IOAPIC -# define TRACE(x) dprintf x +# define TRACE(...) dprintf(__VA_ARGS__) #else -# define TRACE(x) ; +# define TRACE(...) (void)0 #endif @@ -242,6 +243,30 @@ ioapic_end_of_interrupt(int32 num) } +static void +ioapic_assign_interrupt_to_cpu(int32 gsi, int32 cpu) +{ + if (gsi < ISA_INTERRUPT_COUNT && sSourceOverrides[gsi] != 0) + gsi = sSourceOverrides[gsi]; + + struct ioapic* ioapic = find_ioapic(gsi); + if (ioapic == NULL) + return; + + uint32 apicid = x86_get_cpu_apic_id(cpu); + + uint8 pin = gsi - ioapic->global_interrupt_base; + TRACE("ioapic_assign_interrupt_to_cpu: gsi %ld (io-apic %u pin %u) to" + " cpu %ld (apic_id %lu)\n", gsi, ioapic->number, pin, cpu, apicid); + + uint64 entry = ioapic_read_64(*ioapic, IO_APIC_REDIRECTION_TABLE + pin * 2); + entry &= ~(uint64(IO_APIC_DESTINATION_FIELD_MASK) + << IO_APIC_DESTINATION_FIELD_SHIFT); + entry |= uint64(apicid) << IO_APIC_DESTINATION_FIELD_SHIFT; + ioapic_write_64(*ioapic, IO_APIC_REDIRECTION_TABLE + pin * 2, entry, false); +} + + static void ioapic_enable_io_interrupt(int32 gsi) { @@ -256,8 +281,8 @@ ioapic_enable_io_interrupt(int32 gsi) return; uint8 pin = gsi - ioapic->global_interrupt_base; - TRACE(("ioapic_enable_io_interrupt: gsi %ld -> io-apic %u pin %u\n", - gsi, ioapic->number, pin)); + TRACE("ioapic_enable_io_interrupt: gsi %ld -> io-apic %u pin %u\n", + gsi, ioapic->number, pin); uint64 entry = ioapic_read_64(*ioapic, IO_APIC_REDIRECTION_TABLE + pin * 2); entry &= ~IO_APIC_INTERRUPT_MASKED; @@ -273,8 +298,8 @@ ioapic_disable_io_interrupt(int32 gsi) return; uint8 pin = gsi - ioapic->global_interrupt_base; - TRACE(("ioapic_disable_io_interrupt: gsi %ld -> io-apic %u pin %u\n", - gsi, ioapic->number, pin)); + TRACE("ioapic_disable_io_interrupt: gsi %ld -> io-apic %u pin %u\n", + gsi, ioapic->number, pin); uint64 entry = ioapic_read_64(*ioapic, IO_APIC_REDIRECTION_TABLE + pin * 2); entry |= IO_APIC_INTERRUPT_MASKED; @@ -290,8 +315,8 @@ ioapic_configure_io_interrupt(int32 gsi, uint32 config) return; uint8 pin = gsi - ioapic->global_interrupt_base; - TRACE(("ioapic_configure_io_interrupt: gsi %ld -> io-apic %u pin %u; " - "config 0x%08lx\n", gsi, ioapic->number, pin, config)); + TRACE("ioapic_configure_io_interrupt: gsi %ld -> io-apic %u pin %u; " + "config 0x%08lx\n", gsi, ioapic->number, pin, config); ioapic_configure_pin(*ioapic, pin, gsi, config, IO_APIC_DELIVERY_MODE_FIXED); @@ -310,7 +335,7 @@ ioapic_map_ioapic(struct ioapic& ioapic, phys_addr_t physicalAddress) return ioapic.register_area; } - TRACE(("mapped io-apic %u to %p\n", ioapic.number, ioapic.registers)); + TRACE("mapped io-apic %u to %p\n", ioapic.number, ioapic.registers); ioapic.version = ioapic_read_32(ioapic, IO_APIC_VERSION); if (ioapic.version == 0xffffffff) { @@ -650,7 +675,8 @@ ioapic_init(kernel_args* args) &ioapic_configure_io_interrupt, &ioapic_is_spurious_interrupt, &ioapic_is_level_triggered_interrupt, - &ioapic_end_of_interrupt + &ioapic_end_of_interrupt, + &ioapic_assign_interrupt_to_cpu, }; if (args->arch_args.apic == NULL) diff --git a/src/system/kernel/arch/x86/pic.cpp b/src/system/kernel/arch/x86/pic.cpp index 96cd171d15..a802ff32d9 100644 --- a/src/system/kernel/arch/x86/pic.cpp +++ b/src/system/kernel/arch/x86/pic.cpp @@ -196,7 +196,8 @@ pic_init() &pic_configure_io_interrupt, &pic_is_spurious_interrupt, &pic_is_level_triggered_interrupt, - &pic_end_of_interrupt + &pic_end_of_interrupt, + NULL }; // Start initialization sequence for the master and slave PICs diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index b98ad77bdd..433d567c75 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -99,6 +99,9 @@ cpu_preboot_init_percpu(kernel_args *args, int curr_cpu) memset(&gCPU[curr_cpu], 0, sizeof(gCPU[curr_cpu])); gCPU[curr_cpu].cpu_num = curr_cpu; + list_init(&gCPU[curr_cpu].irqs); + B_INITIALIZE_SPINLOCK(&gCPU[curr_cpu].irqs_lock); + return arch_cpu_preboot_init_percpu(args, curr_cpu); } diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index 6e1b1fd446..cfea6fe33e 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -1,4 +1,7 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + * Copyright 2011, Michael Lotz, mmlr@mlotz.ch. * Distributed under the terms of the MIT License. * @@ -54,10 +57,7 @@ struct io_vector { bool no_lock_vector; interrupt_type type; - spinlock load_lock; - bigtime_t last_measure_time; - bigtime_t last_measure_active; - int32 load; + irq_assignment assigned_cpu; #if DEBUG_INTERRUPTS int64 handled_count; @@ -67,6 +67,7 @@ struct io_vector { #endif }; +static uint32 sLastCPU; static struct io_vector sVectors[NUM_IO_VECTORS]; static bool sAllocatedIOInterruptVectors[NUM_IO_VECTORS]; static mutex sIOInterruptVectorAllocationLock @@ -137,10 +138,20 @@ dump_int_load(int argc, char** argv) continue; kprintf("int %3d, type %s, enabled %" B_PRId32 ", load %" B_PRId32 - "%%%s\n", i, typeNames[min_c(sVectors[i].type, + "%%", i, typeNames[min_c(sVectors[i].type, INTERRUPT_TYPE_UNKNOWN)], - sVectors[i].enable_count, sVectors[i].load / 10, - B_SPINLOCK_IS_LOCKED(&sVectors[i].vector_lock) ? ", ACTIVE" : ""); + sVectors[i].enable_count, sVectors[i].assigned_cpu.load / 10); + + if (sVectors[i].type == INTERRUPT_TYPE_IRQ) { + if (sVectors[i].assigned_cpu.cpu != -1) + kprintf(", cpu %" B_PRId32, sVectors[i].assigned_cpu.cpu); + else + kprintf(", cpu -"); + } + + if (B_SPINLOCK_IS_LOCKED(&sVectors[i].vector_lock)) + kprintf(", ACTIVE"); + kprintf("\n"); } return 0; @@ -178,10 +189,15 @@ int_init_post_vm(kernel_args* args) sVectors[i].no_lock_vector = false; sVectors[i].type = INTERRUPT_TYPE_UNKNOWN; - B_INITIALIZE_SPINLOCK(&sVectors[i].load_lock); - sVectors[i].last_measure_time = 0; - sVectors[i].last_measure_active = 0; - sVectors[i].load = 0; + irq_assignment* assigned_cpu = &sVectors[i].assigned_cpu; + assigned_cpu->irq = i; + + B_INITIALIZE_SPINLOCK(&assigned_cpu->load_lock); + assigned_cpu->last_measure_time = 0; + assigned_cpu->last_measure_active = 0; + assigned_cpu->load = 0; + + assigned_cpu->cpu = -1; #if DEBUG_INTERRUPTS sVectors[i].handled_count = 0; @@ -223,11 +239,14 @@ int_init_post_device_manager(kernel_args* args) static void update_int_load(int i) { - if (!try_acquire_spinlock(&sVectors[i].load_lock)) + if (!try_acquire_spinlock(&sVectors[i].assigned_cpu.load_lock)) return; - compute_load(sVectors[i].last_measure_time, sVectors[i].last_measure_active, - sVectors[i].load); - release_spinlock(&sVectors[i].load_lock); + + compute_load(sVectors[i].assigned_cpu.last_measure_time, + sVectors[i].assigned_cpu.last_measure_active, + sVectors[i].assigned_cpu.load); + + release_spinlock(&sVectors[i].assigned_cpu.load_lock); } @@ -325,9 +344,9 @@ int_io_interrupt_handler(int vector, bool levelTriggered) if (!sVectors[vector].no_lock_vector) release_spinlock(&sVectors[vector].vector_lock); - SpinLocker locker(sVectors[vector].load_lock); + SpinLocker locker(sVectors[vector].assigned_cpu.load_lock); bigtime_t deltaTime = system_time() - start; - sVectors[vector].last_measure_active += deltaTime; + sVectors[vector].assigned_cpu.last_measure_active += deltaTime; locker.Unlock(); atomic_add64(&get_cpu_struct()->interrupt_time, deltaTime); @@ -402,6 +421,23 @@ install_io_interrupt_handler(long vector, interrupt_handler handler, void *data, state = disable_interrupts(); acquire_spinlock(&sVectors[vector].vector_lock); + // Initial attempt to balance IRQs, the scheduler will correct this + // if some cores end up being overloaded. + if (sVectors[vector].type == INTERRUPT_TYPE_IRQ + && sVectors[vector].handler_list == NULL) { + + int32 cpuID = atomic_add(&sLastCPU, 1) % smp_get_num_cpus(); + arch_int_assign_to_cpu(vector, cpuID); + + ASSERT(sVectors[vector].assigned_cpu.cpu == -1); + + sVectors[vector].assigned_cpu.cpu = cpuID; + + cpu_ent* cpu = &gCPU[cpuID]; + SpinLocker _(cpu->irqs_lock); + list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + } + if ((flags & B_NO_HANDLED_INFO) != 0 && sVectors[vector].handler_list != NULL) { // The driver registering this interrupt handler doesn't know @@ -441,6 +477,7 @@ install_io_interrupt_handler(long vector, interrupt_handler handler, void *data, sVectors[vector].no_lock_vector = true; release_spinlock(&sVectors[vector].vector_lock); + restore_interrupts(state); return B_OK; @@ -486,6 +523,28 @@ remove_io_interrupt_handler(long vector, interrupt_handler handler, void *data) last = io; } + if (sVectors[vector].handler_list == NULL + && sVectors[vector].type == INTERRUPT_TYPE_IRQ) { + + int32 oldCPU; + SpinLocker locker; + cpu_ent* cpu; + + do { + locker.Unlock(); + + oldCPU = sVectors[vector].assigned_cpu.cpu; + + ASSERT(oldCPU != -1); + cpu = &gCPU[oldCPU]; + + locker.SetTo(cpu->irqs_lock, false); + } while (sVectors[vector].assigned_cpu.cpu != oldCPU); + + sVectors[vector].assigned_cpu.cpu = -1; + list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + } + release_spinlock(&sVectors[vector].vector_lock); restore_interrupts(state); @@ -596,3 +655,27 @@ free_io_interrupt_vectors(long count, long startVector) sAllocatedIOInterruptVectors[startVector + i] = false; } } + + +void assign_io_interrupt_to_cpu(long vector, int32 newCPU) +{ + ASSERT(sVectors[vector].type == INTERRUPT_TYPE_IRQ); + + int32 oldCPU = sVectors[vector].assigned_cpu.cpu; + + ASSERT(oldCPU != -1); + cpu_ent* cpu = &gCPU[oldCPU]; + + SpinLocker locker(cpu->irqs_lock); + sVectors[vector].assigned_cpu.cpu = -1; + list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + locker.Unlock(); + + cpu = &gCPU[newCPU]; + locker.SetTo(cpu->irqs_lock, false); + sVectors[vector].assigned_cpu.cpu = newCPU; + arch_int_assign_to_cpu(vector, newCPU); + list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + locker.Unlock(); +} + diff --git a/src/system/kernel/system_info.cpp b/src/system/kernel/system_info.cpp index d6a9fab733..d6713b00c0 100644 --- a/src/system/kernel/system_info.cpp +++ b/src/system/kernel/system_info.cpp @@ -53,8 +53,8 @@ dump_info(int argc, char **argv) kprintf("cpu count: %" B_PRId32 "\n", smp_get_num_cpus()); for (int32 i = 0; i < smp_get_num_cpus(); i++) - kprintf(" [%" B_PRId32 "] active time: %12" B_PRId64 ", interrupt" - " time: %12" B_PRId64 ", irq time: %12" B_PRId64 "\n", i + 1, + kprintf(" [%" B_PRId32 "] active time: %10" B_PRId64 ", interrupt" + " time: %10" B_PRId64 ", irq time: %10" B_PRId64 "\n", i + 1, gCPU[i].active_time, gCPU[i].interrupt_time, gCPU[i].irq_time); // ToDo: Add page_faults