x86[_64]: Support assigning MSI IRQs to arbitrary CPU

headers/private/kernel/arch/x86/arch_int.h

@@ -11,6 +11,13 @@
 #define NUM_IO_VECTORS		(256 - ARCH_INTERRUPT_BASE)
 
 
+enum irq_source {
+	IRQ_SOURCE_INVALID,
+	IRQ_SOURCE_IOAPIC,
+	IRQ_SOURCE_MSI,
+};
+
+
 static inline void
 arch_int_enable_interrupts_inline(void)
 {
@@ -72,6 +79,8 @@ typedef struct interrupt_controller_s {
 } interrupt_controller;
 
 
+void x86_set_irq_source(int irq, irq_source source);
+
 void arch_int_set_interrupt_controller(const interrupt_controller &controller);
 
 #endif // __cplusplus

headers/private/kernel/arch/x86/msi.h

@@ -28,5 +28,6 @@ bool		msi_supported();
 status_t	msi_allocate_vectors(uint8 count, uint8 *startVector,
 				uint64 *address, uint16 *data);
 void		msi_free_vectors(uint8 count, uint8 startVector);
+void		msi_assign_interrupt_to_cpu(uint8 irq, int32 cpu);
 
 #endif // _KERNEL_ARCH_x86_MSI_H

headers/private/kernel/int.h

@@ -32,13 +32,13 @@ enum interrupt_type {
 
 struct irq_assignment {
 	list_link	link;
+
 	uint32		irq;
+	uint32		count;
+
+	int32		handlers_count;
 
-	spinlock	load_lock;
-	bigtime_t	last_measure_time;
-	bigtime_t	last_measure_active;
 	int32		load;
-
 	int32		cpu;
 };
 
@@ -79,7 +79,8 @@ are_interrupts_enabled(void)
 
 status_t reserve_io_interrupt_vectors(long count, long startVector,
 	enum interrupt_type type);
-status_t allocate_io_interrupt_vectors(long count, long *startVector);
+status_t allocate_io_interrupt_vectors(long count, long *startVector,
+	enum interrupt_type type);
 void free_io_interrupt_vectors(long count, long startVector);
 
 void assign_io_interrupt_to_cpu(long vector, int32 cpu);

src/system/kernel/arch/x86/arch_int.cpp

@@ -24,6 +24,7 @@
 
 #include <arch/x86/apic.h>
 #include <arch/x86/descriptors.h>
+#include <arch/x86/msi.h>
 #include <arch/x86/msi_priv.h>
 
 #include <stdio.h>
@@ -41,6 +42,8 @@
 #endif
 
 
+static irq_source sVectorSources[NUM_IO_VECTORS];
+
 static const char *kInterruptNames[] = {
 	/*  0 */ "Divide Error Exception",
 	/*  1 */ "Debug Exception",
@@ -331,6 +334,13 @@ x86_page_fault_exception(struct iframe* frame)
 }
 
 
+void
+x86_set_irq_source(int irq, irq_source source)
+{
+	sVectorSources[irq] = source;
+}
+
+
 // #pragma mark -
 
 
@@ -392,8 +402,20 @@ arch_int_are_interrupts_enabled(void)
 void
 arch_int_assign_to_cpu(int32 irq, int32 cpu)
 {
-	if (sCurrentPIC->assign_interrupt_to_cpu != NULL)
+	dprintf("ASSIGN IRQ TO CPU\n");
-		sCurrentPIC->assign_interrupt_to_cpu(irq, cpu);
+	switch (sVectorSources[irq]) {
+		case IRQ_SOURCE_IOAPIC:
+			if (sCurrentPIC->assign_interrupt_to_cpu != NULL)
+				sCurrentPIC->assign_interrupt_to_cpu(irq, cpu);
+			break;
+
+		case IRQ_SOURCE_MSI:
+			msi_assign_interrupt_to_cpu(irq, cpu);
+			break;
+
+		default:
+			break;
+	}
 }
 
 

src/system/kernel/arch/x86/ioapic.cpp

@@ -280,6 +280,8 @@ ioapic_enable_io_interrupt(int32 gsi)
 	if (ioapic == NULL)
 		return;
 
+	x86_set_irq_source(gsi, IRQ_SOURCE_IOAPIC);
+
 	uint8 pin = gsi - ioapic->global_interrupt_base;
 	TRACE("ioapic_enable_io_interrupt: gsi %ld -> io-apic %u pin %u\n",
 		gsi, ioapic->number, pin);
@@ -801,6 +803,12 @@ ioapic_init(kernel_args* args)
 	while (current != NULL) {
 		reserve_io_interrupt_vectors(current->max_redirection_entry + 1,
 			current->global_interrupt_base, INTERRUPT_TYPE_IRQ);
+
+		for (int32 i = 0; i < current->max_redirection_entry + 1; i++) {
+			x86_set_irq_source(current->global_interrupt_base + i,
+				IRQ_SOURCE_IOAPIC);
+		}
+
 		current = current->next;
 	}
 

src/system/kernel/arch/x86/msi.cpp

@@ -5,12 +5,20 @@
 
 #include <arch/x86/apic.h>
 #include <arch/x86/msi.h>
+#include <arch/x86/arch_smp.h>
 
 #include <debug.h>
 #include <int.h>
 #include <lock.h>
 
 
+struct MSIConfiguration {
+	uint64*	fAddress;
+	uint16* fData;
+};
+
+static MSIConfiguration sMSIConfigurations[NUM_IO_VECTORS];
+
 static bool sMSISupported = false;
 static uint32 sBootCPUAPICId = 0;
 
@@ -44,7 +52,8 @@ msi_allocate_vectors(uint8 count, uint8 *startVector, uint64 *address,
 		return B_UNSUPPORTED;
 
 	long vector;
-	status_t result = allocate_io_interrupt_vectors(count, &vector);
+	status_t result = allocate_io_interrupt_vectors(count, &vector,
+		INTERRUPT_TYPE_IRQ);
 	if (result != B_OK)
 		return result;
 
@@ -53,6 +62,10 @@ msi_allocate_vectors(uint8 count, uint8 *startVector, uint64 *address,
 		return B_NO_MEMORY;
 	}
 
+	sMSIConfigurations[vector].fAddress = address;
+	sMSIConfigurations[vector].fData = data;
+	x86_set_irq_source(vector, IRQ_SOURCE_MSI);
+
 	*startVector = (uint8)vector;
 	*address = MSI_ADDRESS_BASE | (sBootCPUAPICId << MSI_DESTINATION_ID_SHIFT)
 		| MSI_NO_REDIRECTION | MSI_DESTINATION_MODE_PHYSICAL;
@@ -78,3 +91,15 @@ msi_free_vectors(uint8 count, uint8 startVector)
 
 	free_io_interrupt_vectors(count, startVector);
 }
+
+
+void
+msi_assign_interrupt_to_cpu(uint8 irq, int32 cpu)
+{
+	uint32 apic_id = x86_get_cpu_apic_id(cpu);
+
+	uint64* address = sMSIConfigurations[irq].fAddress;
+	*address = MSI_ADDRESS_BASE | (apic_id << MSI_DESTINATION_ID_SHIFT)
+		| MSI_NO_REDIRECTION | MSI_DESTINATION_MODE_PHYSICAL;
+}
+

src/system/kernel/int.cpp

@@ -57,7 +57,12 @@ struct io_vector {
 	bool				no_lock_vector;
 	interrupt_type		type;
 
-	irq_assignment		assigned_cpu;
+	spinlock			load_lock;
+	bigtime_t			last_measure_time;
+	bigtime_t			last_measure_active;
+	int32				load;
+
+	irq_assignment*		assigned_cpu;
 
 #if DEBUG_INTERRUPTS
 	int64				handled_count;
@@ -69,8 +74,9 @@ struct io_vector {
 
 static int32 sLastCPU;
 
-static struct io_vector sVectors[NUM_IO_VECTORS];
+static io_vector sVectors[NUM_IO_VECTORS];
 static bool sAllocatedIOInterruptVectors[NUM_IO_VECTORS];
+static irq_assignment sVectorCPUAssignments[NUM_IO_VECTORS];
 static mutex sIOInterruptVectorAllocationLock
 	= MUTEX_INITIALIZER("io_interrupt_vector_allocation");
 
@@ -141,11 +147,15 @@ dump_int_load(int argc, char** argv)
 		kprintf("int %3d, type %s, enabled %" B_PRId32 ", load %" B_PRId32
 			"%%", i, typeNames[min_c(sVectors[i].type,
 					INTERRUPT_TYPE_UNKNOWN)],
-			sVectors[i].enable_count, sVectors[i].assigned_cpu.load / 10);
+			sVectors[i].enable_count,
+			sVectors[i].assigned_cpu != NULL
+				? sVectors[i].assigned_cpu->load / 10 : 0);
 
 		if (sVectors[i].type == INTERRUPT_TYPE_IRQ) {
-			if (sVectors[i].assigned_cpu.cpu != -1)
+			ASSERT(sVectors[i].assigned_cpu != NULL);
-				kprintf(", cpu %" B_PRId32, sVectors[i].assigned_cpu.cpu);
+
+			if (sVectors[i].assigned_cpu->cpu != -1)
+				kprintf(", cpu %" B_PRId32, sVectors[i].assigned_cpu->cpu);
 			else
 				kprintf(", cpu -");
 		}
@@ -190,15 +200,10 @@ int_init_post_vm(kernel_args* args)
 		sVectors[i].no_lock_vector = false;
 		sVectors[i].type = INTERRUPT_TYPE_UNKNOWN;
 
-		irq_assignment* assigned_cpu = &sVectors[i].assigned_cpu;
+		B_INITIALIZE_SPINLOCK(&sVectors[i].load_lock);
-		assigned_cpu->irq = i;
+		sVectors[i].last_measure_time = 0;
-
+		sVectors[i].last_measure_active = 0;
-		B_INITIALIZE_SPINLOCK(&assigned_cpu->load_lock);
+		sVectors[i].load = 0;
-		assigned_cpu->last_measure_time = 0;
-		assigned_cpu->last_measure_active = 0;
-		assigned_cpu->load = 0;
-
-		assigned_cpu->cpu = -1;
 
 #if DEBUG_INTERRUPTS
 		sVectors[i].handled_count = 0;
@@ -207,6 +212,12 @@ int_init_post_vm(kernel_args* args)
 		sVectors[i].ignored_count = 0;
 #endif
 		sVectors[i].handler_list = NULL;
+
+		sVectorCPUAssignments[i].irq = i;
+		sVectorCPUAssignments[i].count = 1;
+		sVectorCPUAssignments[i].handlers_count = 0;
+		sVectorCPUAssignments[i].load = 0;
+		sVectorCPUAssignments[i].cpu = -1;
 	}
 
 #if DEBUG_INTERRUPTS
@@ -240,14 +251,17 @@ int_init_post_device_manager(kernel_args* args)
 static void
 update_int_load(int i)
 {
-	if (!try_acquire_spinlock(&sVectors[i].assigned_cpu.load_lock))
+	if (!try_acquire_spinlock(&sVectors[i].load_lock))
 		return;
 
-	compute_load(sVectors[i].assigned_cpu.last_measure_time,
+	int32 oldLoad = sVectors[i].load;
-		sVectors[i].assigned_cpu.last_measure_active,
+	compute_load(sVectors[i].last_measure_time, sVectors[i].last_measure_active,
-		sVectors[i].assigned_cpu.load);
+		sVectors[i].load);
 
-	release_spinlock(&sVectors[i].assigned_cpu.load_lock);
+	if (oldLoad != sVectors[i].load)
+		atomic_add(&sVectors[i].assigned_cpu->load, sVectors[i].load - oldLoad);
+
+	release_spinlock(&sVectors[i].load_lock);
 }
 
 
@@ -345,9 +359,9 @@ int_io_interrupt_handler(int vector, bool levelTriggered)
 	if (!sVectors[vector].no_lock_vector)
 		release_spinlock(&sVectors[vector].vector_lock);
 
-	SpinLocker locker(sVectors[vector].assigned_cpu.load_lock);
+	SpinLocker locker(sVectors[vector].load_lock);
 	bigtime_t deltaTime = system_time() - start;
-	sVectors[vector].assigned_cpu.last_measure_active += deltaTime;
+	sVectors[vector].last_measure_active += deltaTime;
 	locker.Unlock();
 
 	atomic_add64(&get_cpu_struct()->interrupt_time, deltaTime);
@@ -444,18 +458,17 @@ install_io_interrupt_handler(long vector, interrupt_handler handler, void *data,
 	// Initial attempt to balance IRQs, the scheduler will correct this
 	// if some cores end up being overloaded.
 	if (sVectors[vector].type == INTERRUPT_TYPE_IRQ
-		&& sVectors[vector].handler_list == NULL) {
+		&& sVectors[vector].handler_list == NULL
+		&& sVectors[vector].assigned_cpu->cpu == -1) {
 
 		int32 cpuID = assign_cpu();
 		arch_int_assign_to_cpu(vector, cpuID);
-
+		sVectors[vector].assigned_cpu->cpu = cpuID;
-		ASSERT(sVectors[vector].assigned_cpu.cpu == -1);
-
-		sVectors[vector].assigned_cpu.cpu = cpuID;
 
 		cpu_ent* cpu = &gCPU[cpuID];
 		SpinLocker _(cpu->irqs_lock);
-		list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu);
+		atomic_add(&sVectors[vector].assigned_cpu->handlers_count, 1);
+		list_add_item(&cpu->irqs, sVectors[vector].assigned_cpu);
 	}
 
 	if ((flags & B_NO_HANDLED_INFO) != 0
@@ -544,25 +557,32 @@ remove_io_interrupt_handler(long vector, interrupt_handler handler, void *data)
 	}
 
 	if (sVectors[vector].handler_list == NULL
-		&& sVectors[vector].type == INTERRUPT_TYPE_IRQ) {
+		&& sVectors[vector].type == INTERRUPT_TYPE_IRQ
+		&& sVectors[vector].assigned_cpu != NULL
+		&& sVectors[vector].assigned_cpu->handlers_count > 0) {
 
-		int32 oldCPU;
+		int32 oldHandlersCount
-		SpinLocker locker;
+			= atomic_add(&sVectors[vector].assigned_cpu->handlers_count, -1);
-		cpu_ent* cpu;
 
-		do {
+		if (oldHandlersCount == 1) {
-			locker.Unlock();
+			int32 oldCPU;
+			SpinLocker locker;
+			cpu_ent* cpu;
 
-			oldCPU = sVectors[vector].assigned_cpu.cpu;
+			do {
+				locker.Unlock();
 
-			ASSERT(oldCPU != -1);
+				oldCPU = sVectors[vector].assigned_cpu->cpu;
-			cpu = &gCPU[oldCPU];
 
-			locker.SetTo(cpu->irqs_lock, false);
+				ASSERT(oldCPU != -1);
-		} while (sVectors[vector].assigned_cpu.cpu != oldCPU);
+				cpu = &gCPU[oldCPU];
 
-		sVectors[vector].assigned_cpu.cpu = -1;
+				locker.SetTo(cpu->irqs_lock, false);
-		list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu);
+			} while (sVectors[vector].assigned_cpu->cpu != oldCPU);
+
+			sVectors[vector].assigned_cpu->cpu = -1;
+			list_remove_item(&cpu->irqs, sVectors[vector].assigned_cpu);
+		}
 	}
 
 	release_spinlock(&sVectors[vector].vector_lock);
@@ -596,6 +616,9 @@ reserve_io_interrupt_vectors(long count, long startVector, interrupt_type type)
 		}
 
 		sVectors[startVector + i].type = type;
+		sVectors[startVector + i].assigned_cpu
+			= &sVectorCPUAssignments[startVector + i];
+		sVectorCPUAssignments[startVector + i].count = 1;
 		sAllocatedIOInterruptVectors[startVector + i] = true;
 	}
 
@@ -610,7 +633,8 @@ reserve_io_interrupt_vectors(long count, long startVector, interrupt_type type)
 	The first vector to be used is returned in \a startVector on success.
 */
 status_t
-allocate_io_interrupt_vectors(long count, long *startVector)
+allocate_io_interrupt_vectors(long count, long *startVector,
+	interrupt_type type)
 {
 	MutexLocker locker(&sIOInterruptVectorAllocationLock);
 
@@ -639,8 +663,14 @@ allocate_io_interrupt_vectors(long count, long *startVector)
 		return B_NO_MEMORY;
 	}
 
-	for (long i = 0; i < count; i++)
+	for (long i = 0; i < count; i++) {
+		sVectors[vector + i].type = type;
+		sVectors[vector + i].assigned_cpu = &sVectorCPUAssignments[vector];
 		sAllocatedIOInterruptVectors[vector + i] = true;
+	}
+
+	sVectorCPUAssignments[vector].irq = vector;
+	sVectorCPUAssignments[vector].count = count;
 
 	*startVector = vector;
 	dprintf("allocate_io_interrupt_vectors: allocated %ld vectors starting "
@@ -672,6 +702,7 @@ free_io_interrupt_vectors(long count, long startVector)
 				startVector + i);
 		}
 
+		sVectors[startVector + i].assigned_cpu = NULL;
 		sAllocatedIOInterruptVectors[startVector + i] = false;
 	}
 }
@@ -681,7 +712,7 @@ void assign_io_interrupt_to_cpu(long vector, int32 newCPU)
 {
 	ASSERT(sVectors[vector].type == INTERRUPT_TYPE_IRQ);
 
-	int32 oldCPU = sVectors[vector].assigned_cpu.cpu;
+	int32 oldCPU = sVectors[vector].assigned_cpu->cpu;
 
 	if (newCPU == -1)
 		newCPU = assign_cpu();
@@ -693,14 +724,14 @@ void assign_io_interrupt_to_cpu(long vector, int32 newCPU)
 	cpu_ent* cpu = &gCPU[oldCPU];
 
 	SpinLocker locker(cpu->irqs_lock);
-	sVectors[vector].assigned_cpu.cpu = -1;
+	sVectors[vector].assigned_cpu->cpu = -1;
-	list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu);
+	list_remove_item(&cpu->irqs, sVectors[vector].assigned_cpu);
 	locker.Unlock();
 
 	cpu = &gCPU[newCPU];
 	locker.SetTo(cpu->irqs_lock, false);
-	sVectors[vector].assigned_cpu.cpu = newCPU;
+	sVectors[vector].assigned_cpu->cpu = newCPU;
 	arch_int_assign_to_cpu(vector, newCPU);
-	list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu);
+	list_add_item(&cpu->irqs, sVectors[vector].assigned_cpu);
 }