x86: Implement multicast ICIs

headers/private/kernel/arch/smp.h

@@ -10,6 +10,8 @@
 
 struct kernel_args;
 
+class CPUSet;
+
 
 #ifdef __cplusplus
 extern "C" {
@@ -19,6 +21,7 @@ status_t arch_smp_init(struct kernel_args *args);
 status_t arch_smp_per_cpu_init(struct kernel_args *args, int32 cpu);
 void arch_smp_send_ici(int32 target_cpu);
 void arch_smp_send_broadcast_ici(void);
+void arch_smp_send_multicast_ici(CPUSet& cpuSet);
 
 #ifdef __cplusplus
 }

headers/private/kernel/arch/x86/apic.h

@@ -52,7 +52,7 @@
 #define APIC_TRIGGER_MODE_LEVEL					(1 << 15)
 
 /* Interrupt Command defines */
-#define APIC_INTR_COMMAND_1_MASK				0xfff3f000
+#define APIC_INTR_COMMAND_1_MASK				0xfff32000
 #define APIC_INTR_COMMAND_2_MASK				0x00ffffff
 
 #define APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL	0
@@ -110,6 +110,7 @@
 #if !_BOOT_MODE
 
 bool		apic_available();
+bool		x2apic_available();
 uint32		apic_read(uint32 offset);
 void		apic_write(uint32 offset, uint32 data);
 uint32		apic_local_id();
@@ -122,10 +123,9 @@ void		apic_disable_local_ints();
 
 uint32		apic_spurious_intr_vector();
 void		apic_set_spurious_intr_vector(uint32 config);
-uint32		apic_intr_command_1();
+
-void		apic_set_intr_command_1(uint32 config);
+void		apic_set_interrupt_command(uint32 destination, uint32 mode);
-uint32		apic_intr_command_2();
+bool		apic_interrupt_delivered(void);
-void		apic_set_intr_command_2(uint32 config);
 
 uint32		apic_lvt_timer();
 void		apic_set_lvt_timer(uint32 config);

headers/private/kernel/arch/x86/arch_cpu.h

@@ -326,6 +326,8 @@ typedef struct arch_cpu_info {
 	int					model;
 	int					extended_model;
 
+	uint32				logical_apic_id;
+
 	struct X86PagingStructures* active_paging_structures;
 
 	size_t				dr6;	// temporary storage for debug registers (cf.

src/system/kernel/arch/x86/apic.cpp

@@ -15,6 +15,7 @@
 #include <debug.h>
 #include <safemode.h>
 #include <vm/vm.h>
+#include <util/AutoLock.h>
 
 #include "timers/apic_timer.h"
 
@@ -30,6 +31,12 @@ apic_available()
 }
 
 
+bool x2apic_available()
+{
+	return sX2APIC;
+}
+
+
 uint32
 apic_read(uint32 offset)
 {
@@ -94,6 +101,16 @@ apic_end_of_interrupt()
 }
 
 
+uint32
+apic_logical_apic_id()
+{
+	if (sX2APIC)
+		return x86_read_msr(IA32_MSR_APIC_LOGICAL_DEST);
+	else
+		return apic_read(APIC_LOGICAL_DEST);
+}
+
+
 void
 apic_disable_local_ints()
 {
@@ -128,49 +145,36 @@ apic_set_spurious_intr_vector(uint32 config)
 }
 
 
-uint32
-apic_intr_command_1()
-{
-	if (sX2APIC)
-		return x86_read_msr(IA32_MSR_APIC_INTR_COMMAND) & 0xffffffff;
-	else
-		return apic_read(APIC_INTR_COMMAND_1);
-}
-
-
 void
-apic_set_intr_command_1(uint32 config)
+apic_set_interrupt_command(uint32 destination, uint32 mode)
 {
 	if (sX2APIC) {
-		uint64 value = x86_read_msr(IA32_MSR_APIC_INTR_COMMAND);
+		uint64 command = x86_read_msr(IA32_MSR_APIC_INTR_COMMAND);
-		arch_cpu_memory_read_write_barrier();
+		command &= APIC_INTR_COMMAND_1_MASK;
-		x86_write_msr(IA32_MSR_APIC_INTR_COMMAND,
+		command |= (uint64)destination << 32;
-			(value & 0xffffffff00000000LL) | config);
+		command |= mode;
-	} else
+		x86_write_msr(IA32_MSR_APIC_INTR_COMMAND, command);
-		apic_write(APIC_INTR_COMMAND_1, config);
+	} else {
+		uint32 command2 = apic_read(APIC_INTR_COMMAND_2)
+				& APIC_INTR_COMMAND_2_MASK;
+		command2 |= destination << 24;
+		apic_write(APIC_INTR_COMMAND_2, command2);
+
+		uint32 command1 = apic_read(APIC_INTR_COMMAND_1)
+				& APIC_INTR_COMMAND_1_MASK;
+		command1 |= mode;
+		apic_write(APIC_INTR_COMMAND_1, command1);
+	}
 }
 
 
-uint32
+bool
-apic_intr_command_2()
+apic_interrupt_delivered(void)
 {
 	if (sX2APIC)
-		return x86_read_msr(IA32_MSR_APIC_INTR_COMMAND) >> 32;
+		return true;
 	else
-		return apic_read(APIC_INTR_COMMAND_2);
+		return (apic_read(APIC_INTR_COMMAND_1) & APIC_DELIVERY_STATUS) == 0;
-}
-
-
-void
-apic_set_intr_command_2(uint32 config)
-{
-	if (sX2APIC) {
-		uint64 value = x86_read_msr(IA32_MSR_APIC_INTR_COMMAND);
-		arch_cpu_memory_read_write_barrier();
-		x86_write_msr(IA32_MSR_APIC_INTR_COMMAND,
-			(value & 0xffffffff) | ((uint64)config << 32));
-	} else
-		apic_write(APIC_INTR_COMMAND_2, config);
 }
 
 
@@ -264,11 +268,6 @@ apic_init(kernel_args *args)
 		dprintf("found x2apic\n");
 #if 0
 		if (!get_safemode_boolean(B_SAFEMODE_DISABLE_X2APIC, false)) {
-			uint64 apic_base = x86_read_msr(IA32_MSR_APIC_BASE);
-			if ((apic_base & IA32_MSR_APIC_BASE_X2APIC) == 0) {
-				x86_write_msr(IA32_MSR_APIC_BASE, apic_base
-					| IA32_MSR_APIC_BASE_X2APIC);
-			}
 			sX2APIC = true;
 			return B_OK;
 		}
@@ -276,11 +275,6 @@ apic_init(kernel_args *args)
 		dprintf("x2apic disabled per safemode setting\n");
 #else
 		if (get_safemode_boolean(B_SAFEMODE_ENABLE_X2APIC, false)) {
-			uint64 apic_base = x86_read_msr(IA32_MSR_APIC_BASE);
-			if ((apic_base & IA32_MSR_APIC_BASE_X2APIC) == 0) {
-				x86_write_msr(IA32_MSR_APIC_BASE, apic_base
-					| IA32_MSR_APIC_BASE_X2APIC);
-			}
 			sX2APIC = true;
 
 			dprintf("x2apic enabled per safemode setting\n");
@@ -306,8 +300,33 @@ apic_init(kernel_args *args)
 status_t
 apic_per_cpu_init(kernel_args *args, int32 cpu)
 {
-	dprintf("setting up apic for CPU %" B_PRId32 ": apic id %" B_PRIu32 ", "
+	if (sX2APIC) {
-		"version %" B_PRIu32 "\n", cpu, apic_local_id(), apic_version());
+		uint64 apic_base = x86_read_msr(IA32_MSR_APIC_BASE);
+		if ((apic_base & IA32_MSR_APIC_BASE_X2APIC) == 0) {
+			x86_write_msr(IA32_MSR_APIC_BASE, apic_base
+				| IA32_MSR_APIC_BASE_X2APIC);
+		}
+	}
+
+	dprintf("setting up %sapic for CPU %" B_PRId32 ": apic id %" B_PRIu32 ", "
+		"version %" B_PRIu32 "\n", sX2APIC ? "x2" : "", cpu, apic_local_id(),
+		apic_version());
+
+	if (!sX2APIC && cpu < 8) {
+		apic_write(APIC_DEST_FORMAT, uint32(-1));
+
+		uint8 logical_apic_id = 1 << cpu;
+		uint32 value = apic_read(APIC_LOGICAL_DEST);
+		value &= 0xffffff;
+		apic_write(APIC_LOGICAL_DEST, value | (logical_apic_id << 24));
+	}
+
+	// get logical APIC ID
+	gCPU[cpu].arch.logical_apic_id = apic_logical_apic_id();
+	if (!sX2APIC)
+		gCPU[cpu].arch.logical_apic_id >>= 24;
+	dprintf("CPU %" B_PRId32 ": logical apic id: %#" B_PRIx32 "\n", cpu,
+		gCPU[cpu].arch.logical_apic_id);
 
 	/* set spurious interrupt vector to 0xff */
 	uint32 config = apic_spurious_intr_vector() & 0xffffff00;

src/system/kernel/arch/x86/arch_smp.cpp

@@ -1,4 +1,5 @@
 /*
+ * Copyright 2013, Paweł Dziepak, [email protected].
  * Copyright 2002-2005, Axel Dörfler, [email protected]. All rights reserved.
  * Distributed under the terms of the MIT License.
  *
@@ -25,6 +26,8 @@
 #include <string.h>
 #include <stdio.h>
 
+#include <algorithm>
+
 
 //#define TRACE_ARCH_SMP
 #ifdef TRACE_ARCH_SMP
@@ -34,6 +37,9 @@
 #endif
 
 
+#define	ICI_VECTOR		0xfd
+
+
 static uint32 sCPUAPICIds[SMP_MAX_CPUS];
 static uint32 sAPICVersions[SMP_MAX_CPUS];
 
@@ -124,47 +130,95 @@ arch_smp_per_cpu_init(kernel_args *args, int32 cpu)
 }
 
 
+void
+arch_smp_send_multicast_ici(CPUSet& cpuSet)
+{
+#if KDEBUG
+	if (are_interrupts_enabled())
+		panic("arch_smp_send_multicast_ici: called with interrupts enabled");
+#endif
+
+	arch_cpu_memory_write_barrier();
+
+	int32 i = 0;
+	int32 cpuCount = smp_get_num_cpus();
+
+	int32 logicalModeCPUs;
+	if (x2apic_available())
+		logicalModeCPUs = cpuCount;
+	else
+		logicalModeCPUs = std::min(cpuCount, int32(8));
+
+	uint32 destination = 0;
+	for (; i < logicalModeCPUs; i++) {
+		if (cpuSet.GetBit(i) && i != smp_get_current_cpu())
+			destination |= gCPU[i].arch.logical_apic_id;
+	}
+
+	uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED
+			| APIC_INTR_COMMAND_1_ASSERT
+			| APIC_INTR_COMMAND_1_DEST_MODE_LOGICAL
+			| APIC_INTR_COMMAND_1_DEST_FIELD;
+
+	while (!apic_interrupt_delivered())
+		cpu_pause();
+	apic_set_interrupt_command(destination, mode);
+
+	for (; i < cpuCount; i++) {
+		if (cpuSet.GetBit(i)) {
+			uint32 destination = sCPUAPICIds[i];
+			uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED
+					| APIC_INTR_COMMAND_1_ASSERT
+					| APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL
+					| APIC_INTR_COMMAND_1_DEST_FIELD;
+
+			while (!apic_interrupt_delivered())
+				cpu_pause();
+			apic_set_interrupt_command(destination, mode);
+		}
+	}
+}
+
+
 void
 arch_smp_send_broadcast_ici(void)
 {
-	uint32 config;
+#if KDEBUG
-	cpu_status state = disable_interrupts();
+	if (are_interrupts_enabled())
+		panic("arch_smp_send_broadcast_ici: called with interrupts enabled");
+#endif
 
-	config = apic_intr_command_1() & APIC_INTR_COMMAND_1_MASK;
+	arch_cpu_memory_write_barrier();
-	apic_set_intr_command_1(config | 0xfd | APIC_DELIVERY_MODE_FIXED
-		| APIC_INTR_COMMAND_1_ASSERT
-		| APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL
-		| APIC_INTR_COMMAND_1_DEST_ALL_BUT_SELF);
 
-	restore_interrupts(state);
+	uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED
+			| APIC_INTR_COMMAND_1_ASSERT
+			| APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL
+			| APIC_INTR_COMMAND_1_DEST_ALL_BUT_SELF;
+
+	while (!apic_interrupt_delivered())
+		cpu_pause();
+	apic_set_interrupt_command(0, mode);
 }
 
 
 void
 arch_smp_send_ici(int32 target_cpu)
 {
-	uint32 config;
+#if KDEBUG
-	uint32 timeout;
+	if (are_interrupts_enabled())
-	cpu_status state;
+		panic("arch_smp_send_ici: called with interrupts enabled");
+#endif
 
-	state = disable_interrupts();
+	arch_cpu_memory_write_barrier();
 
-	config = apic_intr_command_2() & APIC_INTR_COMMAND_2_MASK;
+	uint32 destination = sCPUAPICIds[target_cpu];
-	apic_set_intr_command_2(config | sCPUAPICIds[target_cpu] << 24);
+	uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED
+			| APIC_INTR_COMMAND_1_ASSERT
+			| APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL
+			| APIC_INTR_COMMAND_1_DEST_FIELD;
 
-	config = apic_intr_command_1() & APIC_INTR_COMMAND_1_MASK;
+	while (!apic_interrupt_delivered())
-	apic_set_intr_command_1(config | 0xfd | APIC_DELIVERY_MODE_FIXED
+		cpu_pause();
-		| APIC_INTR_COMMAND_1_ASSERT
+	apic_set_interrupt_command(destination, mode);
-		| APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL
-		| APIC_INTR_COMMAND_1_DEST_FIELD);
-
-	timeout = 100000000;
-	// wait for message to be sent
-	while ((apic_intr_command_1() & APIC_DELIVERY_STATUS) != 0 && --timeout != 0)
-		asm volatile ("pause;");
-
-	if (timeout == 0)
-		panic("arch_smp_send_ici: timeout, target_cpu %" B_PRId32, target_cpu);
-
-	restore_interrupts(state);
 }
+

src/system/kernel/smp.cpp

@@ -1112,11 +1112,16 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
 	if (!sICIEnabled)
 		return;
 
+	int currentCPU = smp_get_current_cpu();
+	bool broadcast = true;
+
 	// count target CPUs
 	int32 targetCPUs = 0;
 	for (int32 i = 0; i < sNumCPUs; i++) {
 		if (cpuMask.GetBit(i))
 			targetCPUs++;
+		else if (i != currentCPU)
+			broadcast = false;
 	}
 
 	// find_free_message leaves interrupts disabled
@@ -1132,7 +1137,6 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
 	msg->flags = flags;
 	msg->done = 0;
 
-	int currentCPU = smp_get_current_cpu();
 	msg->proc_bitmap = cpuMask;
 	msg->proc_bitmap.ClearBit(currentCPU);
 	if (msg->proc_bitmap.IsEmpty()) {
@@ -1153,8 +1157,10 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
 			atomic_add(&gCPU[i].ici_counter, 1);
 	}
 
-	arch_smp_send_broadcast_ici();
+	if (broadcast)
-		// TODO: Introduce a call that only bothers the target CPUs!
+		arch_smp_send_broadcast_ici();
+	else
+		arch_smp_send_multicast_ici(cpuMask);
 
 	if ((flags & SMP_MSG_FLAG_SYNC) != 0) {
 		// wait for the other cpus to finish processing it