diff --git a/headers/private/kernel/arch/x86/arch_int.h b/headers/private/kernel/arch/x86/arch_int.h index 5de0dcb9d1..bfbe401061 100644 --- a/headers/private/kernel/arch/x86/arch_int.h +++ b/headers/private/kernel/arch/x86/arch_int.h @@ -11,6 +11,13 @@ #define NUM_IO_VECTORS (256 - ARCH_INTERRUPT_BASE) +enum irq_source { + IRQ_SOURCE_INVALID, + IRQ_SOURCE_IOAPIC, + IRQ_SOURCE_MSI, +}; + + static inline void arch_int_enable_interrupts_inline(void) { @@ -72,6 +79,8 @@ typedef struct interrupt_controller_s { } interrupt_controller; +void x86_set_irq_source(int irq, irq_source source); + void arch_int_set_interrupt_controller(const interrupt_controller &controller); #endif // __cplusplus diff --git a/headers/private/kernel/arch/x86/msi.h b/headers/private/kernel/arch/x86/msi.h index 18685cb021..04e0cd9674 100644 --- a/headers/private/kernel/arch/x86/msi.h +++ b/headers/private/kernel/arch/x86/msi.h @@ -28,5 +28,6 @@ bool msi_supported(); status_t msi_allocate_vectors(uint8 count, uint8 *startVector, uint64 *address, uint16 *data); void msi_free_vectors(uint8 count, uint8 startVector); +void msi_assign_interrupt_to_cpu(uint8 irq, int32 cpu); #endif // _KERNEL_ARCH_x86_MSI_H diff --git a/headers/private/kernel/int.h b/headers/private/kernel/int.h index 6922aa0573..91608a4782 100644 --- a/headers/private/kernel/int.h +++ b/headers/private/kernel/int.h @@ -32,13 +32,13 @@ enum interrupt_type { struct irq_assignment { list_link link; + uint32 irq; + uint32 count; + + int32 handlers_count; - spinlock load_lock; - bigtime_t last_measure_time; - bigtime_t last_measure_active; int32 load; - int32 cpu; }; @@ -79,7 +79,8 @@ are_interrupts_enabled(void) status_t reserve_io_interrupt_vectors(long count, long startVector, enum interrupt_type type); -status_t allocate_io_interrupt_vectors(long count, long *startVector); +status_t allocate_io_interrupt_vectors(long count, long *startVector, + enum interrupt_type type); void free_io_interrupt_vectors(long count, long startVector); void assign_io_interrupt_to_cpu(long vector, int32 cpu); diff --git a/src/system/kernel/arch/x86/arch_int.cpp b/src/system/kernel/arch/x86/arch_int.cpp index 41176d850f..0ff467a37c 100644 --- a/src/system/kernel/arch/x86/arch_int.cpp +++ b/src/system/kernel/arch/x86/arch_int.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include @@ -41,6 +42,8 @@ #endif +static irq_source sVectorSources[NUM_IO_VECTORS]; + static const char *kInterruptNames[] = { /* 0 */ "Divide Error Exception", /* 1 */ "Debug Exception", @@ -331,6 +334,13 @@ x86_page_fault_exception(struct iframe* frame) } +void +x86_set_irq_source(int irq, irq_source source) +{ + sVectorSources[irq] = source; +} + + // #pragma mark - @@ -392,8 +402,20 @@ arch_int_are_interrupts_enabled(void) void arch_int_assign_to_cpu(int32 irq, int32 cpu) { - if (sCurrentPIC->assign_interrupt_to_cpu != NULL) - sCurrentPIC->assign_interrupt_to_cpu(irq, cpu); + dprintf("ASSIGN IRQ TO CPU\n"); + switch (sVectorSources[irq]) { + case IRQ_SOURCE_IOAPIC: + if (sCurrentPIC->assign_interrupt_to_cpu != NULL) + sCurrentPIC->assign_interrupt_to_cpu(irq, cpu); + break; + + case IRQ_SOURCE_MSI: + msi_assign_interrupt_to_cpu(irq, cpu); + break; + + default: + break; + } } diff --git a/src/system/kernel/arch/x86/ioapic.cpp b/src/system/kernel/arch/x86/ioapic.cpp index 3f6c5448f6..bbc8d72ff9 100644 --- a/src/system/kernel/arch/x86/ioapic.cpp +++ b/src/system/kernel/arch/x86/ioapic.cpp @@ -280,6 +280,8 @@ ioapic_enable_io_interrupt(int32 gsi) if (ioapic == NULL) return; + x86_set_irq_source(gsi, IRQ_SOURCE_IOAPIC); + uint8 pin = gsi - ioapic->global_interrupt_base; TRACE("ioapic_enable_io_interrupt: gsi %ld -> io-apic %u pin %u\n", gsi, ioapic->number, pin); @@ -801,6 +803,12 @@ ioapic_init(kernel_args* args) while (current != NULL) { reserve_io_interrupt_vectors(current->max_redirection_entry + 1, current->global_interrupt_base, INTERRUPT_TYPE_IRQ); + + for (int32 i = 0; i < current->max_redirection_entry + 1; i++) { + x86_set_irq_source(current->global_interrupt_base + i, + IRQ_SOURCE_IOAPIC); + } + current = current->next; } diff --git a/src/system/kernel/arch/x86/msi.cpp b/src/system/kernel/arch/x86/msi.cpp index 54f4cffc28..4578e14d19 100644 --- a/src/system/kernel/arch/x86/msi.cpp +++ b/src/system/kernel/arch/x86/msi.cpp @@ -5,12 +5,20 @@ #include #include +#include #include #include #include +struct MSIConfiguration { + uint64* fAddress; + uint16* fData; +}; + +static MSIConfiguration sMSIConfigurations[NUM_IO_VECTORS]; + static bool sMSISupported = false; static uint32 sBootCPUAPICId = 0; @@ -44,7 +52,8 @@ msi_allocate_vectors(uint8 count, uint8 *startVector, uint64 *address, return B_UNSUPPORTED; long vector; - status_t result = allocate_io_interrupt_vectors(count, &vector); + status_t result = allocate_io_interrupt_vectors(count, &vector, + INTERRUPT_TYPE_IRQ); if (result != B_OK) return result; @@ -53,6 +62,10 @@ msi_allocate_vectors(uint8 count, uint8 *startVector, uint64 *address, return B_NO_MEMORY; } + sMSIConfigurations[vector].fAddress = address; + sMSIConfigurations[vector].fData = data; + x86_set_irq_source(vector, IRQ_SOURCE_MSI); + *startVector = (uint8)vector; *address = MSI_ADDRESS_BASE | (sBootCPUAPICId << MSI_DESTINATION_ID_SHIFT) | MSI_NO_REDIRECTION | MSI_DESTINATION_MODE_PHYSICAL; @@ -78,3 +91,15 @@ msi_free_vectors(uint8 count, uint8 startVector) free_io_interrupt_vectors(count, startVector); } + + +void +msi_assign_interrupt_to_cpu(uint8 irq, int32 cpu) +{ + uint32 apic_id = x86_get_cpu_apic_id(cpu); + + uint64* address = sMSIConfigurations[irq].fAddress; + *address = MSI_ADDRESS_BASE | (apic_id << MSI_DESTINATION_ID_SHIFT) + | MSI_NO_REDIRECTION | MSI_DESTINATION_MODE_PHYSICAL; +} + diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index 6601f3ace2..fea008b845 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -57,7 +57,12 @@ struct io_vector { bool no_lock_vector; interrupt_type type; - irq_assignment assigned_cpu; + spinlock load_lock; + bigtime_t last_measure_time; + bigtime_t last_measure_active; + int32 load; + + irq_assignment* assigned_cpu; #if DEBUG_INTERRUPTS int64 handled_count; @@ -69,8 +74,9 @@ struct io_vector { static int32 sLastCPU; -static struct io_vector sVectors[NUM_IO_VECTORS]; +static io_vector sVectors[NUM_IO_VECTORS]; static bool sAllocatedIOInterruptVectors[NUM_IO_VECTORS]; +static irq_assignment sVectorCPUAssignments[NUM_IO_VECTORS]; static mutex sIOInterruptVectorAllocationLock = MUTEX_INITIALIZER("io_interrupt_vector_allocation"); @@ -141,11 +147,15 @@ dump_int_load(int argc, char** argv) kprintf("int %3d, type %s, enabled %" B_PRId32 ", load %" B_PRId32 "%%", i, typeNames[min_c(sVectors[i].type, INTERRUPT_TYPE_UNKNOWN)], - sVectors[i].enable_count, sVectors[i].assigned_cpu.load / 10); + sVectors[i].enable_count, + sVectors[i].assigned_cpu != NULL + ? sVectors[i].assigned_cpu->load / 10 : 0); if (sVectors[i].type == INTERRUPT_TYPE_IRQ) { - if (sVectors[i].assigned_cpu.cpu != -1) - kprintf(", cpu %" B_PRId32, sVectors[i].assigned_cpu.cpu); + ASSERT(sVectors[i].assigned_cpu != NULL); + + if (sVectors[i].assigned_cpu->cpu != -1) + kprintf(", cpu %" B_PRId32, sVectors[i].assigned_cpu->cpu); else kprintf(", cpu -"); } @@ -190,15 +200,10 @@ int_init_post_vm(kernel_args* args) sVectors[i].no_lock_vector = false; sVectors[i].type = INTERRUPT_TYPE_UNKNOWN; - irq_assignment* assigned_cpu = &sVectors[i].assigned_cpu; - assigned_cpu->irq = i; - - B_INITIALIZE_SPINLOCK(&assigned_cpu->load_lock); - assigned_cpu->last_measure_time = 0; - assigned_cpu->last_measure_active = 0; - assigned_cpu->load = 0; - - assigned_cpu->cpu = -1; + B_INITIALIZE_SPINLOCK(&sVectors[i].load_lock); + sVectors[i].last_measure_time = 0; + sVectors[i].last_measure_active = 0; + sVectors[i].load = 0; #if DEBUG_INTERRUPTS sVectors[i].handled_count = 0; @@ -207,6 +212,12 @@ int_init_post_vm(kernel_args* args) sVectors[i].ignored_count = 0; #endif sVectors[i].handler_list = NULL; + + sVectorCPUAssignments[i].irq = i; + sVectorCPUAssignments[i].count = 1; + sVectorCPUAssignments[i].handlers_count = 0; + sVectorCPUAssignments[i].load = 0; + sVectorCPUAssignments[i].cpu = -1; } #if DEBUG_INTERRUPTS @@ -240,14 +251,17 @@ int_init_post_device_manager(kernel_args* args) static void update_int_load(int i) { - if (!try_acquire_spinlock(&sVectors[i].assigned_cpu.load_lock)) + if (!try_acquire_spinlock(&sVectors[i].load_lock)) return; - compute_load(sVectors[i].assigned_cpu.last_measure_time, - sVectors[i].assigned_cpu.last_measure_active, - sVectors[i].assigned_cpu.load); + int32 oldLoad = sVectors[i].load; + compute_load(sVectors[i].last_measure_time, sVectors[i].last_measure_active, + sVectors[i].load); - release_spinlock(&sVectors[i].assigned_cpu.load_lock); + if (oldLoad != sVectors[i].load) + atomic_add(&sVectors[i].assigned_cpu->load, sVectors[i].load - oldLoad); + + release_spinlock(&sVectors[i].load_lock); } @@ -345,9 +359,9 @@ int_io_interrupt_handler(int vector, bool levelTriggered) if (!sVectors[vector].no_lock_vector) release_spinlock(&sVectors[vector].vector_lock); - SpinLocker locker(sVectors[vector].assigned_cpu.load_lock); + SpinLocker locker(sVectors[vector].load_lock); bigtime_t deltaTime = system_time() - start; - sVectors[vector].assigned_cpu.last_measure_active += deltaTime; + sVectors[vector].last_measure_active += deltaTime; locker.Unlock(); atomic_add64(&get_cpu_struct()->interrupt_time, deltaTime); @@ -444,18 +458,17 @@ install_io_interrupt_handler(long vector, interrupt_handler handler, void *data, // Initial attempt to balance IRQs, the scheduler will correct this // if some cores end up being overloaded. if (sVectors[vector].type == INTERRUPT_TYPE_IRQ - && sVectors[vector].handler_list == NULL) { + && sVectors[vector].handler_list == NULL + && sVectors[vector].assigned_cpu->cpu == -1) { int32 cpuID = assign_cpu(); arch_int_assign_to_cpu(vector, cpuID); - - ASSERT(sVectors[vector].assigned_cpu.cpu == -1); - - sVectors[vector].assigned_cpu.cpu = cpuID; + sVectors[vector].assigned_cpu->cpu = cpuID; cpu_ent* cpu = &gCPU[cpuID]; SpinLocker _(cpu->irqs_lock); - list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + atomic_add(&sVectors[vector].assigned_cpu->handlers_count, 1); + list_add_item(&cpu->irqs, sVectors[vector].assigned_cpu); } if ((flags & B_NO_HANDLED_INFO) != 0 @@ -544,25 +557,32 @@ remove_io_interrupt_handler(long vector, interrupt_handler handler, void *data) } if (sVectors[vector].handler_list == NULL - && sVectors[vector].type == INTERRUPT_TYPE_IRQ) { + && sVectors[vector].type == INTERRUPT_TYPE_IRQ + && sVectors[vector].assigned_cpu != NULL + && sVectors[vector].assigned_cpu->handlers_count > 0) { - int32 oldCPU; - SpinLocker locker; - cpu_ent* cpu; + int32 oldHandlersCount + = atomic_add(&sVectors[vector].assigned_cpu->handlers_count, -1); - do { - locker.Unlock(); + if (oldHandlersCount == 1) { + int32 oldCPU; + SpinLocker locker; + cpu_ent* cpu; - oldCPU = sVectors[vector].assigned_cpu.cpu; + do { + locker.Unlock(); - ASSERT(oldCPU != -1); - cpu = &gCPU[oldCPU]; + oldCPU = sVectors[vector].assigned_cpu->cpu; - locker.SetTo(cpu->irqs_lock, false); - } while (sVectors[vector].assigned_cpu.cpu != oldCPU); + ASSERT(oldCPU != -1); + cpu = &gCPU[oldCPU]; - sVectors[vector].assigned_cpu.cpu = -1; - list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + locker.SetTo(cpu->irqs_lock, false); + } while (sVectors[vector].assigned_cpu->cpu != oldCPU); + + sVectors[vector].assigned_cpu->cpu = -1; + list_remove_item(&cpu->irqs, sVectors[vector].assigned_cpu); + } } release_spinlock(&sVectors[vector].vector_lock); @@ -596,6 +616,9 @@ reserve_io_interrupt_vectors(long count, long startVector, interrupt_type type) } sVectors[startVector + i].type = type; + sVectors[startVector + i].assigned_cpu + = &sVectorCPUAssignments[startVector + i]; + sVectorCPUAssignments[startVector + i].count = 1; sAllocatedIOInterruptVectors[startVector + i] = true; } @@ -610,7 +633,8 @@ reserve_io_interrupt_vectors(long count, long startVector, interrupt_type type) The first vector to be used is returned in \a startVector on success. */ status_t -allocate_io_interrupt_vectors(long count, long *startVector) +allocate_io_interrupt_vectors(long count, long *startVector, + interrupt_type type) { MutexLocker locker(&sIOInterruptVectorAllocationLock); @@ -639,8 +663,14 @@ allocate_io_interrupt_vectors(long count, long *startVector) return B_NO_MEMORY; } - for (long i = 0; i < count; i++) + for (long i = 0; i < count; i++) { + sVectors[vector + i].type = type; + sVectors[vector + i].assigned_cpu = &sVectorCPUAssignments[vector]; sAllocatedIOInterruptVectors[vector + i] = true; + } + + sVectorCPUAssignments[vector].irq = vector; + sVectorCPUAssignments[vector].count = count; *startVector = vector; dprintf("allocate_io_interrupt_vectors: allocated %ld vectors starting " @@ -672,6 +702,7 @@ free_io_interrupt_vectors(long count, long startVector) startVector + i); } + sVectors[startVector + i].assigned_cpu = NULL; sAllocatedIOInterruptVectors[startVector + i] = false; } } @@ -681,7 +712,7 @@ void assign_io_interrupt_to_cpu(long vector, int32 newCPU) { ASSERT(sVectors[vector].type == INTERRUPT_TYPE_IRQ); - int32 oldCPU = sVectors[vector].assigned_cpu.cpu; + int32 oldCPU = sVectors[vector].assigned_cpu->cpu; if (newCPU == -1) newCPU = assign_cpu(); @@ -693,14 +724,14 @@ void assign_io_interrupt_to_cpu(long vector, int32 newCPU) cpu_ent* cpu = &gCPU[oldCPU]; SpinLocker locker(cpu->irqs_lock); - sVectors[vector].assigned_cpu.cpu = -1; - list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + sVectors[vector].assigned_cpu->cpu = -1; + list_remove_item(&cpu->irqs, sVectors[vector].assigned_cpu); locker.Unlock(); cpu = &gCPU[newCPU]; locker.SetTo(cpu->irqs_lock, false); - sVectors[vector].assigned_cpu.cpu = newCPU; + sVectors[vector].assigned_cpu->cpu = newCPU; arch_int_assign_to_cpu(vector, newCPU); - list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + list_add_item(&cpu->irqs, sVectors[vector].assigned_cpu); }