kernel: Use CPUSet in ICI code instead of cpu_mask_t

headers/private/kernel/arch/smp.h

@@ -11,9 +11,6 @@
 struct kernel_args;
 
 
-// must match MAX_BOOT_CPUS in platform_kernel_args.h
-#define SMP_MAX_CPUS MAX_BOOT_CPUS
-
 #ifdef __cplusplus
 extern "C" {
 #endif

headers/private/kernel/smp.h

@@ -9,8 +9,17 @@
 #define KERNEL_SMP_H
 
 
+#include <boot/kernel_args.h>
+#include <kernel.h>
+
 #include <KernelExport.h>
 
+#include <string.h>
+
+
+#define SMP_MAX_CPUS MAX_BOOT_CPUS
+
+
 struct kernel_args;
 
 
@@ -35,6 +44,26 @@ typedef uint32 cpu_mask_t;
 
 typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3);
 
+class CPUSet {
+public:
+						CPUSet() { ClearAll(); }
+
+	inline	void		ClearAll();
+	inline	void		SetAll();
+
+	inline	void		SetBit(int32 cpu);
+	inline	void		ClearBit(int32 cpu);
+
+	inline	bool		GetBit(int32 cpu) const;
+
+	inline	bool		IsEmpty() const;
+
+private:
+	static	const int	kArraySize = ROUNDUP(SMP_MAX_CPUS, 32) / 32;
+
+			uint32		fBitmap[kArraySize];
+};
+
 
 #ifdef __cplusplus
 extern "C" {
@@ -47,10 +76,10 @@ status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu);
 status_t smp_init_post_generic_syscalls(void);
 bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous);
 void smp_wake_up_non_boot_cpus(void);
-void smp_cpu_rendezvous(uint32 *var);
+void smp_cpu_rendezvous(uint32* var);
 void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3,
 		void *data_ptr, uint32 flags);
-void smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data,
+void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
 		addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
 void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
 		void *data_ptr, uint32 flags);
@@ -68,6 +97,56 @@ int smp_intercpu_int_handler(int32 cpu);
 #endif
 
 
+inline void
+CPUSet::ClearAll()
+{
+	memset(fBitmap, 0, sizeof(fBitmap));
+}
+
+
+inline void
+CPUSet::SetAll()
+{
+	memset(fBitmap, ~uint8(0), sizeof(fBitmap));
+}
+
+
+inline void
+CPUSet::SetBit(int32 cpu)
+{
+	uint32* element = &fBitmap[cpu % kArraySize];
+	atomic_or(element, 1u << (cpu / kArraySize));
+}
+
+
+inline void
+CPUSet::ClearBit(int32 cpu)
+{
+	uint32* element = &fBitmap[cpu % kArraySize];
+	atomic_and(element, ~uint32(1u << (cpu / kArraySize)));
+}
+
+
+inline bool
+CPUSet::GetBit(int32 cpu) const
+{
+	int32* element = (int32*)&fBitmap[cpu % kArraySize];
+	return ((uint32)atomic_get(element) & (1u << (cpu / kArraySize))) != 0;
+}
+
+
+inline bool
+CPUSet::IsEmpty() const
+{
+	for (int i = 0; i < kArraySize; i++) {
+		if (fBitmap[i] != 0)
+			return false;
+	}
+
+	return true;
+}
+
+
 // Unless spinlock debug features are enabled, try to inline
 // {acquire,release}_spinlock().
 #if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION

src/system/kernel/arch/x86/arch_thread.cpp

@@ -229,9 +229,8 @@ arch_thread_context_switch(Thread* from, Thread* to)
 					!= activePagingStructures) {
 		// update on which CPUs the address space is used
 		int cpu = cpuData->cpu_num;
-		atomic_and(&activePagingStructures->active_on_cpus,
+		activePagingStructures->active_on_cpus.ClearBit(cpu);
-			~((uint32)1 << cpu));
+		toPagingStructures->active_on_cpus.SetBit(cpu);
-		atomic_or(&toPagingStructures->active_on_cpus, (uint32)1 << cpu);
 
 		// assign the new paging structures to the CPU
 		toPagingStructures->AddReference();

src/system/kernel/arch/x86/paging/X86PagingStructures.cpp

@@ -9,8 +9,7 @@
 
 X86PagingStructures::X86PagingStructures()
 	:
-	ref_count(1),
+	ref_count(1)
-	active_on_cpus(0)
 {
 }
 

src/system/kernel/arch/x86/paging/X86PagingStructures.h

@@ -14,11 +14,13 @@
 
 #include <heap.h>
 
+#include <smp.h>
+
 
 struct X86PagingStructures : DeferredDeletable {
 	phys_addr_t					pgdir_phys;
 	int32						ref_count;
-	int32						active_on_cpus;
+	CPUSet						active_on_cpus;
 		// mask indicating on which CPUs the map is currently used
 
 								X86PagingStructures();

src/system/kernel/arch/x86/paging/X86VMTranslationMap.cpp

@@ -113,9 +113,10 @@ X86VMTranslationMap::Flush()
 			restore_interrupts(state);
 
 			int cpu = smp_get_current_cpu();
-			uint32 cpuMask = PagingStructures()->active_on_cpus
+			CPUSet cpuMask = PagingStructures()->active_on_cpus;
-				& ~((uint32)1 << cpu);
+			cpuMask.ClearBit(cpu);
-			if (cpuMask != 0) {
+
+			if (!cpuMask.IsEmpty()) {
 				smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
 					0, 0, 0, NULL, SMP_MSG_FLAG_SYNC);
 			}
@@ -132,9 +133,10 @@ X86VMTranslationMap::Flush()
 				SMP_MSG_FLAG_SYNC);
 		} else {
 			int cpu = smp_get_current_cpu();
-			uint32 cpuMask = PagingStructures()->active_on_cpus
+			CPUSet cpuMask = PagingStructures()->active_on_cpus;
-				& ~((uint32)1 << cpu);
+			cpuMask.ClearBit(cpu);
-			if (cpuMask != 0) {
+
+			if (!cpuMask.IsEmpty()) {
 				smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
 					(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL,
 					SMP_MSG_FLAG_SYNC);

src/system/kernel/smp.cpp

@@ -80,7 +80,7 @@ struct smp_msg {
 	uint32			flags;
 	int32			ref_count;
 	int32			done;
-	uint32			proc_bitmap;
+	CPUSet			proc_bitmap;
 };
 
 enum mailbox_source {
@@ -90,7 +90,8 @@ enum mailbox_source {
 
 static int32 sBootCPUSpin = 0;
 
-static int32 sEarlyCPUCall = 0;
+static int32 sEarlyCPUCallCount;
+static CPUSet sEarlyCPUCallSet;
 static void (*sEarlyCPUCallFunction)(void*, int);
 void* sEarlyCPUCallCookie;
 
@@ -294,7 +295,13 @@ dump_ici_message(int argc, char** argv)
 	kprintf("  flags:       %" B_PRIx32 "\n", message->flags);
 	kprintf("  ref_count:   %" B_PRIx32 "\n", message->ref_count);
 	kprintf("  done:        %s\n", message->done == 1 ? "true" : "false");
-	kprintf("  proc_bitmap: %" B_PRIx32 "\n", message->proc_bitmap);
+
+	kprintf("  proc_bitmap: ");
+	for (int32 i = 0; i < sNumCPUs; i++) {
+		if (message->proc_bitmap.GetBit(i))
+			kprintf("%s%" B_PRId32, i != 0 ? ", " : "", i);
+	}
+	kprintf("\n");
 
 	return 0;
 }
@@ -809,14 +816,14 @@ check_for_message(int currentCPU, mailbox_source& sourceMailbox)
 
 		msg = sBroadcastMessages;
 		while (msg != NULL) {
-			if (CHECK_BIT(msg->proc_bitmap, currentCPU) != 0) {
+			if (!msg->proc_bitmap.GetBit(currentCPU)) {
 				// we have handled this one already
 				msg = msg->next;
 				continue;
 			}
 
 			// mark it so we wont try to process this one again
-			msg->proc_bitmap = SET_BIT(msg->proc_bitmap, currentCPU);
+			msg->proc_bitmap.ClearBit(currentCPU);
 			atomic_add(&gCPU[currentCPU].ici_counter, 1);
 
 			sourceMailbox = MAILBOX_BCAST;
@@ -994,7 +1001,8 @@ static void
 process_early_cpu_call(int32 cpu)
 {
 	sEarlyCPUCallFunction(sEarlyCPUCallCookie, cpu);
-	atomic_and(&sEarlyCPUCall, ~(uint32)(1 << cpu));
+	sEarlyCPUCallSet.ClearBit(cpu);
+	atomic_add(&sEarlyCPUCallCount, 1);
 }
 
 
@@ -1005,14 +1013,13 @@ call_all_cpus_early(void (*function)(void*, int), void* cookie)
 		sEarlyCPUCallFunction = function;
 		sEarlyCPUCallCookie = cookie;
 
-		uint32 cpuMask = (1 << sNumCPUs) - 2;
+		atomic_set(&sEarlyCPUCallCount, 1);
-			// all CPUs but the boot cpu
+		sEarlyCPUCallSet.SetAll();
-
+		sEarlyCPUCallSet.ClearBit(0);
-		atomic_set(&sEarlyCPUCall, cpuMask);
 
 		// wait for all CPUs to finish
-		while ((atomic_get(&sEarlyCPUCall) & cpuMask) != 0)
+		while (atomic_get(&sEarlyCPUCallCount) < sNumCPUs)
-			cpu_pause();
+			cpu_wait(&sEarlyCPUCallCount, sNumCPUs);
 	}
 
 	function(cookie, 0);
@@ -1099,24 +1106,16 @@ smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2,
 
 
 void
-smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data,
+smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
 	addr_t data2, addr_t data3, void *dataPointer, uint32 flags)
 {
 	if (!sICIEnabled)
 		return;
 
-	int currentCPU = smp_get_current_cpu();
-	cpuMask &= ~((cpu_mask_t)1 << currentCPU)
-		& (((cpu_mask_t)1 << sNumCPUs) - 1);
-	if (cpuMask == 0) {
-		panic("smp_send_multicast_ici(): 0 CPU mask");
-		return;
-	}
-
 	// count target CPUs
 	int32 targetCPUs = 0;
 	for (int32 i = 0; i < sNumCPUs; i++) {
-		if ((cpuMask & (cpu_mask_t)1 << i) != 0)
+		if (cpuMask.GetBit(i))
 			targetCPUs++;
 	}
 
@@ -1131,9 +1130,17 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data,
 	msg->data_ptr = dataPointer;
 	msg->ref_count = targetCPUs;
 	msg->flags = flags;
-	msg->proc_bitmap = ~cpuMask;
 	msg->done = 0;
 
+	int currentCPU = smp_get_current_cpu();
+	msg->proc_bitmap = cpuMask;
+	msg->proc_bitmap.ClearBit(currentCPU);
+	if (msg->proc_bitmap.IsEmpty()) {
+		panic("smp_send_multicast_ici(): 0 CPU mask");
+		return;
+	}
+
+
 	// stick it in the broadcast mailbox
 	acquire_spinlock_nocheck(&sBroadcastMessageSpinlock);
 	msg->next = sBroadcastMessages;
@@ -1142,7 +1149,7 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data,
 
 	atomic_add(&sBroadcastMessageCounter, 1);
 	for (int32 i = 0; i < sNumCPUs; i++) {
-		if ((cpuMask & (cpu_mask_t)1 << i) == 0)
+		if (!cpuMask.GetBit(i))
 			atomic_add(&gCPU[i].ici_counter, 1);
 	}
 
@@ -1193,7 +1200,8 @@ smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
 		msg->data_ptr = dataPointer;
 		msg->ref_count = sNumCPUs - 1;
 		msg->flags = flags;
-		msg->proc_bitmap = SET_BIT(0, currentCPU);
+		msg->proc_bitmap.SetAll();
+		msg->proc_bitmap.ClearBit(currentCPU);
 		msg->done = 0;
 
 		TRACE("smp_send_broadcast_ici%d: inserting msg %p into broadcast "
@@ -1258,7 +1266,8 @@ smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
 	msg->data_ptr = dataPointer;
 	msg->ref_count = sNumCPUs - 1;
 	msg->flags = flags;
-	msg->proc_bitmap = SET_BIT(0, currentCPU);
+	msg->proc_bitmap.SetAll();
+	msg->proc_bitmap.ClearBit(currentCPU);
 	msg->done = 0;
 
 	TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: inserting msg %p "
@@ -1320,7 +1329,7 @@ smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous)
 	smp_cpu_rendezvous(rendezVous);
 
 	while (atomic_get(&sBootCPUSpin) == 0) {
-		if ((atomic_get(&sEarlyCPUCall) & (1 << cpu)) != 0)
+		if (sEarlyCPUCallSet.GetBit(cpu))
 			process_early_cpu_call(cpu);
 
 		cpu_pause();