kernel: Allow reassigning IRQs to logical processors

headers/private/kernel/arch/int.h

@@ -34,6 +34,7 @@ void arch_int_enable_io_interrupt(int irq);
 void arch_int_disable_io_interrupt(int irq);
 void arch_int_configure_io_interrupt(int irq, uint32 config);
 bool arch_int_are_interrupts_enabled(void);
+void arch_int_assign_to_cpu(int32 irq, int32 cpu);
 
 #ifdef __cplusplus
 }

headers/private/kernel/arch/x86/arch_int.h

@@ -68,6 +68,7 @@ typedef struct interrupt_controller_s {
 	bool	(*is_spurious_interrupt)(int32 num);
 	bool	(*is_level_triggered_interrupt)(int32 num);
 	bool	(*end_of_interrupt)(int32 num);
+	void	(*assign_interrupt_to_cpu)(int32 num, int32 cpu);
 } interrupt_controller;
 
 

headers/private/kernel/arch/x86/arch_smp.h

@@ -99,4 +99,18 @@ enum {
 	MP_INTR_TYPE_ExtINT,
 };
 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+uint32 x86_get_cpu_apic_id(int32 cpu);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+
 #endif	/* _KERNEL_ARCH_x86_ARCH_SMP_H */

headers/private/kernel/cpu.h

@@ -11,6 +11,7 @@
 
 #include <setjmp.h>
 
+#include <int.h>
 #include <smp.h>
 #include <timer.h>
 #include <arch/cpu.h>
@@ -79,8 +80,12 @@ typedef struct cpu_ent {
 	int				topology_id[CPU_TOPOLOGY_LEVELS];
 	int				cache_id[CPU_MAX_CACHE_LEVEL];
 
+	// IRQs assigned to this CPU
+	struct list		irqs;
+	spinlock		irqs_lock;
+
 	// arch-specific stuff
-	arch_cpu_info arch;
+	arch_cpu_info 	arch;
 } cpu_ent __attribute__((aligned(64)));
 
 

headers/private/kernel/int.h

@@ -12,6 +12,8 @@
 #include <KernelExport.h>
 #include <arch/int.h>
 
+#include <util/list.h>
+
 // private install_io_interrupt_handler() flags
 #define B_NO_LOCK_VECTOR	0x100
 #define B_NO_HANDLED_INFO	0x200
@@ -28,6 +30,18 @@ enum interrupt_type {
 	INTERRUPT_TYPE_UNKNOWN
 };
 
+struct irq_assignment {
+	list_link	link;
+	uint32		irq;
+
+	spinlock	load_lock;
+	bigtime_t	last_measure_time;
+	bigtime_t	last_measure_active;
+	int32		load;
+
+	int32		cpu;
+};
+
 
 #ifdef __cplusplus
 extern "C" {
@@ -68,4 +82,6 @@ status_t reserve_io_interrupt_vectors(long count, long startVector,
 status_t allocate_io_interrupt_vectors(long count, long *startVector);
 void free_io_interrupt_vectors(long count, long startVector);
 
+void assign_io_interrupt_to_cpu(long vector, int32 cpu);
+
 #endif /* _KERNEL_INT_H */

src/system/kernel/arch/x86/arch_int.cpp

@@ -389,6 +389,14 @@ arch_int_are_interrupts_enabled(void)
 }
 
 
+void
+arch_int_assign_to_cpu(int32 irq, int32 cpu)
+{
+	if (sCurrentPIC->assign_interrupt_to_cpu != NULL)
+		sCurrentPIC->assign_interrupt_to_cpu(irq, cpu);
+}
+
+
 status_t
 arch_int_init(kernel_args* args)
 {

src/system/kernel/arch/x86/arch_smp.cpp

@@ -70,6 +70,14 @@ x86_smp_error_interrupt(void *data)
 }
 
 
+uint32
+x86_get_cpu_apic_id(int32 cpu)
+{
+	ASSERT(cpu >= 0 && cpu < B_MAX_CPU_COUNT);
+	return sCPUAPICIds[cpu];
+}
+
+
 status_t
 arch_smp_init(kernel_args *args)
 {

src/system/kernel/arch/x86/ioapic.cpp

@@ -18,6 +18,7 @@
 
 #include <arch/x86/apic.h>
 #include <arch/x86/arch_int.h>
+#include <arch/x86/arch_smp.h>
 #include <arch/x86/pic.h>
 
 // to gain access to the ACPICA types
@@ -26,9 +27,9 @@
 
 //#define TRACE_IOAPIC
 #ifdef TRACE_IOAPIC
-#	define TRACE(x) dprintf x
+#	define TRACE(...) dprintf(__VA_ARGS__)
 #else
-#	define TRACE(x) ;
+#	define TRACE(...) (void)0
 #endif
 
 
@@ -242,6 +243,30 @@ ioapic_end_of_interrupt(int32 num)
 }
 
 
+static void
+ioapic_assign_interrupt_to_cpu(int32 gsi, int32 cpu)
+{
+	if (gsi < ISA_INTERRUPT_COUNT && sSourceOverrides[gsi] != 0)
+		gsi = sSourceOverrides[gsi];
+
+	struct ioapic* ioapic = find_ioapic(gsi);
+	if (ioapic == NULL)
+		return;
+
+	uint32 apicid = x86_get_cpu_apic_id(cpu);
+
+	uint8 pin = gsi - ioapic->global_interrupt_base;
+	TRACE("ioapic_assign_interrupt_to_cpu: gsi %ld (io-apic %u pin %u) to"
+		" cpu %ld (apic_id %lu)\n", gsi, ioapic->number, pin, cpu, apicid);
+
+	uint64 entry = ioapic_read_64(*ioapic, IO_APIC_REDIRECTION_TABLE + pin * 2);
+	entry &= ~(uint64(IO_APIC_DESTINATION_FIELD_MASK)
+			<< IO_APIC_DESTINATION_FIELD_SHIFT);
+	entry |= uint64(apicid) << IO_APIC_DESTINATION_FIELD_SHIFT;
+	ioapic_write_64(*ioapic, IO_APIC_REDIRECTION_TABLE + pin * 2, entry, false);
+}
+
+
 static void
 ioapic_enable_io_interrupt(int32 gsi)
 {
@@ -256,8 +281,8 @@ ioapic_enable_io_interrupt(int32 gsi)
 		return;
 
 	uint8 pin = gsi - ioapic->global_interrupt_base;
-	TRACE(("ioapic_enable_io_interrupt: gsi %ld -> io-apic %u pin %u\n",
+	TRACE("ioapic_enable_io_interrupt: gsi %ld -> io-apic %u pin %u\n",
-		gsi, ioapic->number, pin));
+		gsi, ioapic->number, pin);
 
 	uint64 entry = ioapic_read_64(*ioapic, IO_APIC_REDIRECTION_TABLE + pin * 2);
 	entry &= ~IO_APIC_INTERRUPT_MASKED;
@@ -273,8 +298,8 @@ ioapic_disable_io_interrupt(int32 gsi)
 		return;
 
 	uint8 pin = gsi - ioapic->global_interrupt_base;
-	TRACE(("ioapic_disable_io_interrupt: gsi %ld -> io-apic %u pin %u\n",
+	TRACE("ioapic_disable_io_interrupt: gsi %ld -> io-apic %u pin %u\n",
-		gsi, ioapic->number, pin));
+		gsi, ioapic->number, pin);
 
 	uint64 entry = ioapic_read_64(*ioapic, IO_APIC_REDIRECTION_TABLE + pin * 2);
 	entry |= IO_APIC_INTERRUPT_MASKED;
@@ -290,8 +315,8 @@ ioapic_configure_io_interrupt(int32 gsi, uint32 config)
 		return;
 
 	uint8 pin = gsi - ioapic->global_interrupt_base;
-	TRACE(("ioapic_configure_io_interrupt: gsi %ld -> io-apic %u pin %u; "
+	TRACE("ioapic_configure_io_interrupt: gsi %ld -> io-apic %u pin %u; "
-		"config 0x%08lx\n", gsi, ioapic->number, pin, config));
+		"config 0x%08lx\n", gsi, ioapic->number, pin, config);
 
 	ioapic_configure_pin(*ioapic, pin, gsi, config,
 		IO_APIC_DELIVERY_MODE_FIXED);
@@ -310,7 +335,7 @@ ioapic_map_ioapic(struct ioapic& ioapic, phys_addr_t physicalAddress)
 		return ioapic.register_area;
 	}
 
-	TRACE(("mapped io-apic %u to %p\n", ioapic.number, ioapic.registers));
+	TRACE("mapped io-apic %u to %p\n", ioapic.number, ioapic.registers);
 
 	ioapic.version = ioapic_read_32(ioapic, IO_APIC_VERSION);
 	if (ioapic.version == 0xffffffff) {
@@ -650,7 +675,8 @@ ioapic_init(kernel_args* args)
 		&ioapic_configure_io_interrupt,
 		&ioapic_is_spurious_interrupt,
 		&ioapic_is_level_triggered_interrupt,
-		&ioapic_end_of_interrupt
+		&ioapic_end_of_interrupt,
+		&ioapic_assign_interrupt_to_cpu,
 	};
 
 	if (args->arch_args.apic == NULL)

src/system/kernel/arch/x86/pic.cpp

@@ -196,7 +196,8 @@ pic_init()
 		&pic_configure_io_interrupt,
 		&pic_is_spurious_interrupt,
 		&pic_is_level_triggered_interrupt,
-		&pic_end_of_interrupt
+		&pic_end_of_interrupt,
+		NULL
 	};
 
 	// Start initialization sequence for the master and slave PICs

src/system/kernel/cpu.cpp

@@ -99,6 +99,9 @@ cpu_preboot_init_percpu(kernel_args *args, int curr_cpu)
 	memset(&gCPU[curr_cpu], 0, sizeof(gCPU[curr_cpu]));
 	gCPU[curr_cpu].cpu_num = curr_cpu;
 
+	list_init(&gCPU[curr_cpu].irqs);
+	B_INITIALIZE_SPINLOCK(&gCPU[curr_cpu].irqs_lock);
+
 	return arch_cpu_preboot_init_percpu(args, curr_cpu);
 }
 

src/system/kernel/int.cpp

@@ -1,4 +1,7 @@
 /*
+ * Copyright 2013, Paweł Dziepak, [email protected].
+ * Distributed under the terms of the MIT License.
+
  * Copyright 2011, Michael Lotz, [email protected].
  * Distributed under the terms of the MIT License.
  *
@@ -54,10 +57,7 @@ struct io_vector {
 	bool				no_lock_vector;
 	interrupt_type		type;
 
-	spinlock			load_lock;
+	irq_assignment		assigned_cpu;
-	bigtime_t			last_measure_time;
-	bigtime_t			last_measure_active;
-	int32				load;
 
 #if DEBUG_INTERRUPTS
 	int64				handled_count;
@@ -67,6 +67,7 @@ struct io_vector {
 #endif
 };
 
+static uint32 sLastCPU;
 static struct io_vector sVectors[NUM_IO_VECTORS];
 static bool sAllocatedIOInterruptVectors[NUM_IO_VECTORS];
 static mutex sIOInterruptVectorAllocationLock
@@ -137,10 +138,20 @@ dump_int_load(int argc, char** argv)
 			continue;
 
 		kprintf("int %3d, type %s, enabled %" B_PRId32 ", load %" B_PRId32
-			"%%%s\n", i, typeNames[min_c(sVectors[i].type,
+			"%%", i, typeNames[min_c(sVectors[i].type,
 					INTERRUPT_TYPE_UNKNOWN)],
-			sVectors[i].enable_count, sVectors[i].load / 10,
+			sVectors[i].enable_count, sVectors[i].assigned_cpu.load / 10);
-			B_SPINLOCK_IS_LOCKED(&sVectors[i].vector_lock) ? ", ACTIVE" : "");
+
+		if (sVectors[i].type == INTERRUPT_TYPE_IRQ) {
+			if (sVectors[i].assigned_cpu.cpu != -1)
+				kprintf(", cpu %" B_PRId32, sVectors[i].assigned_cpu.cpu);
+			else
+				kprintf(", cpu -");
+		}
+
+		if (B_SPINLOCK_IS_LOCKED(&sVectors[i].vector_lock))
+			kprintf(", ACTIVE");
+		kprintf("\n");
 	}
 
 	return 0;
@@ -178,10 +189,15 @@ int_init_post_vm(kernel_args* args)
 		sVectors[i].no_lock_vector = false;
 		sVectors[i].type = INTERRUPT_TYPE_UNKNOWN;
 
-		B_INITIALIZE_SPINLOCK(&sVectors[i].load_lock);
+		irq_assignment* assigned_cpu = &sVectors[i].assigned_cpu;
-		sVectors[i].last_measure_time = 0;
+		assigned_cpu->irq = i;
-		sVectors[i].last_measure_active = 0;
+
-		sVectors[i].load = 0;
+		B_INITIALIZE_SPINLOCK(&assigned_cpu->load_lock);
+		assigned_cpu->last_measure_time = 0;
+		assigned_cpu->last_measure_active = 0;
+		assigned_cpu->load = 0;
+
+		assigned_cpu->cpu = -1;
 
 #if DEBUG_INTERRUPTS
 		sVectors[i].handled_count = 0;
@@ -223,11 +239,14 @@ int_init_post_device_manager(kernel_args* args)
 static void
 update_int_load(int i)
 {
-	if (!try_acquire_spinlock(&sVectors[i].load_lock))
+	if (!try_acquire_spinlock(&sVectors[i].assigned_cpu.load_lock))
 		return;
-	compute_load(sVectors[i].last_measure_time, sVectors[i].last_measure_active,
+
-		sVectors[i].load);
+	compute_load(sVectors[i].assigned_cpu.last_measure_time,
-	release_spinlock(&sVectors[i].load_lock);
+		sVectors[i].assigned_cpu.last_measure_active,
+		sVectors[i].assigned_cpu.load);
+
+	release_spinlock(&sVectors[i].assigned_cpu.load_lock);
 }
 
 
@@ -325,9 +344,9 @@ int_io_interrupt_handler(int vector, bool levelTriggered)
 	if (!sVectors[vector].no_lock_vector)
 		release_spinlock(&sVectors[vector].vector_lock);
 
-	SpinLocker locker(sVectors[vector].load_lock);
+	SpinLocker locker(sVectors[vector].assigned_cpu.load_lock);
 	bigtime_t deltaTime = system_time() - start;
-	sVectors[vector].last_measure_active += deltaTime;
+	sVectors[vector].assigned_cpu.last_measure_active += deltaTime;
 	locker.Unlock();
 
 	atomic_add64(&get_cpu_struct()->interrupt_time, deltaTime);
@@ -402,6 +421,23 @@ install_io_interrupt_handler(long vector, interrupt_handler handler, void *data,
 	state = disable_interrupts();
 	acquire_spinlock(&sVectors[vector].vector_lock);
 
+	// Initial attempt to balance IRQs, the scheduler will correct this
+	// if some cores end up being overloaded.
+	if (sVectors[vector].type == INTERRUPT_TYPE_IRQ
+		&& sVectors[vector].handler_list == NULL) {
+
+		int32 cpuID = atomic_add(&sLastCPU, 1) % smp_get_num_cpus();
+		arch_int_assign_to_cpu(vector, cpuID);
+
+		ASSERT(sVectors[vector].assigned_cpu.cpu == -1);
+
+		sVectors[vector].assigned_cpu.cpu = cpuID;
+
+		cpu_ent* cpu = &gCPU[cpuID];
+		SpinLocker _(cpu->irqs_lock);
+		list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu);
+	}
+
 	if ((flags & B_NO_HANDLED_INFO) != 0
 		&& sVectors[vector].handler_list != NULL) {
 		// The driver registering this interrupt handler doesn't know
@@ -441,6 +477,7 @@ install_io_interrupt_handler(long vector, interrupt_handler handler, void *data,
 		sVectors[vector].no_lock_vector = true;
 
 	release_spinlock(&sVectors[vector].vector_lock);
+
 	restore_interrupts(state);
 
 	return B_OK;
@@ -486,6 +523,28 @@ remove_io_interrupt_handler(long vector, interrupt_handler handler, void *data)
 		last = io;
 	}
 
+	if (sVectors[vector].handler_list == NULL
+		&& sVectors[vector].type == INTERRUPT_TYPE_IRQ) {
+
+		int32 oldCPU;
+		SpinLocker locker;
+		cpu_ent* cpu;
+
+		do {
+			locker.Unlock();
+
+			oldCPU = sVectors[vector].assigned_cpu.cpu;
+
+			ASSERT(oldCPU != -1);
+			cpu = &gCPU[oldCPU];
+
+			locker.SetTo(cpu->irqs_lock, false);
+		} while (sVectors[vector].assigned_cpu.cpu != oldCPU);
+
+		sVectors[vector].assigned_cpu.cpu = -1;
+		list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu);
+	}
+
 	release_spinlock(&sVectors[vector].vector_lock);
 	restore_interrupts(state);
 
@@ -596,3 +655,27 @@ free_io_interrupt_vectors(long count, long startVector)
 		sAllocatedIOInterruptVectors[startVector + i] = false;
 	}
 }
+
+
+void assign_io_interrupt_to_cpu(long vector, int32 newCPU)
+{
+	ASSERT(sVectors[vector].type == INTERRUPT_TYPE_IRQ);
+
+	int32 oldCPU = sVectors[vector].assigned_cpu.cpu;
+
+	ASSERT(oldCPU != -1);
+	cpu_ent* cpu = &gCPU[oldCPU];
+
+	SpinLocker locker(cpu->irqs_lock);
+	sVectors[vector].assigned_cpu.cpu = -1;
+	list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu);
+	locker.Unlock();
+
+	cpu = &gCPU[newCPU];
+	locker.SetTo(cpu->irqs_lock, false);
+	sVectors[vector].assigned_cpu.cpu = newCPU;
+	arch_int_assign_to_cpu(vector, newCPU);
+	list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu);
+	locker.Unlock();
+}
+

src/system/kernel/system_info.cpp

@@ -53,8 +53,8 @@ dump_info(int argc, char **argv)
 	kprintf("cpu count: %" B_PRId32 "\n", smp_get_num_cpus());
 
 	for (int32 i = 0; i < smp_get_num_cpus(); i++)
-		kprintf("  [%" B_PRId32 "] active time: %12" B_PRId64 ", interrupt"
+		kprintf("  [%" B_PRId32 "] active time: %10" B_PRId64 ", interrupt"
-			" time: %12" B_PRId64 ", irq time: %12" B_PRId64 "\n", i + 1,
+			" time: %10" B_PRId64 ", irq time: %10" B_PRId64 "\n", i + 1,
 			gCPU[i].active_time, gCPU[i].interrupt_time, gCPU[i].irq_time);
 
 	// ToDo: Add page_faults