From d4817afe36b0e32e2134c0cb06e8e2cb951bc490 Mon Sep 17 00:00:00 2001
From: Augustin Cavalier <waddlesplash@gmail.com>
Date: Tue, 17 Mar 2026 13:06:35 -0400
Subject: [PATCH] kernel/smp: Use a single-CPU ICI if multicasting to only one
 CPU.

This avoids the broadcast mailbox (and the incrementing of
other CPU's counters) entirely in this case.

As this is used for TLB invalidation, processes that are only
running two threads at once will benefit immensely from this.
It also seems to be triggered by the kernel's "idle_scan_active_pages"
logic, which sometimes sends invalidations to CPUs besides its own.

On the VM setup, this path was hit around 200 times during boot,
around 600 times during a rebuild of HaikuDepot + mime_db, and
50,000+ times while running Iceweasel (Firefox). On bare metal,
it was hit around 300 times during boot, 300 times during a rebuild,
and 90,000+ times while running Iceweasel (over a different length
of time, though.)
---
 src/system/kernel/smp.cpp | 62 +++++++++++++++++++++++++--------------
 1 file changed, 40 insertions(+), 22 deletions(-)

diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp
index 661787363e..306c988cba 100644
--- a/src/system/kernel/smp.cpp
+++ b/src/system/kernel/smp.cpp
@@ -1130,10 +1130,17 @@ smp_multicast_ici(const CPUSet& cpuMask, int32 message, addr_t data,
 		return;
 	}
 
-	int32 targetCPUs = 0;
+	ASSERT(thread_get_current_thread()->pinned_to_cpu);
+	int32 currentCPU = smp_get_current_cpu();
+	bool self = cpuMask.GetBit(currentCPU);
+
+	int32 targetCPUs = 0, firstNonCurrentCPU = -1;
 	for (int32 i = 0; i < sNumCPUs; i++) {
-		if (cpuMask.GetBit(i))
+		if (cpuMask.GetBit(i)) {
 			targetCPUs++;
+			if (firstNonCurrentCPU < 0 && i != currentCPU)
+				firstNonCurrentCPU = i;
+		}
 	}
 
 	if (targetCPUs == 0) {
@@ -1155,28 +1162,39 @@ smp_multicast_ici(const CPUSet& cpuMask, int32 message, addr_t data,
 	msg->done = 0;
 	msg->proc_bitmap = cpuMask;
 
-	int32 currentCPU = smp_get_current_cpu();
-	bool self = cpuMask.GetBit(currentCPU);
-	bool broadcast = (!self && targetCPUs == sNumCPUs - 1)
-		|| (self && targetCPUs == sNumCPUs);
+	if ((!self && targetCPUs == 1) || (self && targetCPUs == 2)) {
+		// stick it in the appropriate cpu's mailbox
+		prepend_message(sCPUMessages[firstNonCurrentCPU], msg);
 
-	// stick it in the broadcast mailbox
-	acquire_read_spinlock_nocheck(&sBroadcastMessageSpinlock);
-	prepend_message(sBroadcastMessages, msg);
-	release_read_spinlock(&sBroadcastMessageSpinlock);
+		arch_smp_send_ici(firstNonCurrentCPU);
 
-	atomic_add(&sBroadcastMessageCounter, 1);
-	for (int32 i = 0; i < sNumCPUs; i++) {
-		if (!cpuMask.GetBit(i))
-			atomic_add(&gCPU[i].ici_counter, 1);
-	}
-
-	if (broadcast) {
-		arch_smp_send_broadcast_ici();
+		if (self) {
+			// invoke for ourselves
+			invoke_smp_msg(msg, currentCPU, NULL);
+			finish_message_processing(currentCPU, msg, MAILBOX_LOCAL);
+		}
 	} else {
-		CPUSet sendMask = cpuMask;
-		sendMask.ClearBit(currentCPU);
-		arch_smp_send_multicast_ici(sendMask);
+		bool broadcast = (!self && targetCPUs == sNumCPUs - 1)
+			|| (self && targetCPUs == sNumCPUs);
+
+		// stick it in the broadcast mailbox
+		acquire_read_spinlock_nocheck(&sBroadcastMessageSpinlock);
+		prepend_message(sBroadcastMessages, msg);
+		release_read_spinlock(&sBroadcastMessageSpinlock);
+
+		atomic_add(&sBroadcastMessageCounter, 1);
+		for (int32 i = 0; i < sNumCPUs; i++) {
+			if (!cpuMask.GetBit(i))
+				atomic_add(&gCPU[i].ici_counter, 1);
+		}
+
+		if (broadcast) {
+			arch_smp_send_broadcast_ici();
+		} else {
+			CPUSet sendMask = cpuMask;
+			sendMask.ClearBit(currentCPU);
+			arch_smp_send_multicast_ici(sendMask);
+		}
 	}
 
 	if ((flags & SMP_MSG_FLAG_SYNC) != 0) {
@@ -1192,7 +1210,7 @@ smp_multicast_ici(const CPUSet& cpuMask, int32 message, addr_t data,
 		// back into the free list
 		return_free_message(msg);
 	} else if (self) {
-		// make sure this CPU has processed the message at least
+		// if broadcast, make sure this CPU has processed the message at least
 		process_all_pending_ici(currentCPU);
 	}