i2p/haiku-beta6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kernel/smp: Refactor ICI broadcast/multicast to invoke the current CPU too.

Browse Source 
Since the methods now don't just "send", the names are changed
also (which is fitting since they're now API-incompatible.)

The changes in the TranslationMaps to adopt this fixes a correctness
issue: previously TLB invalidations would be done either with or
without interrupts enabled depending on how they were invoked.
Now, they'll always be done with interrupts disabled. (This is
the same fix that was recently done to x86.)

The idea with processing the ICIs on the current CPU too is
that presumably the other CPUs will take just as much, if not
more, time processing the ICI than the sending CPU. So, we might
as well do the processing while we wait, rather than before or after.

Testing shows this to indeed be the case. On a bare metal i3, kernel
map invalidations (with global invalidations always forced) across
a boot-to-desktop go from around ~60.8ms cumulative before,
to ~51.8ms cumulative after this change.

Change-Id: Icd0343d462de7cf11043e6438f47b02f6b88bb8b
Reviewed-on: https://review.haiku-os.org/c/haiku/+/10472
Reviewed-by: waddlesplash <[email protected]>
Tested-by: Commit checker robot <[email protected]>
This commit is contained in:
Augustin Cavalier
2026-03-16 13:41:00 +00:00
committed by waddlesplash
parent 988ca40b01
commit 7512053757
11 changed files with 247 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files
headers/private/kernel/smp.h
+6 -5
View File
@@ -82,14 +82,15 @@ status_t smp_init_post_generic_syscalls(void);
bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous);
void smp_wake_up_non_boot_cpus(void);
void smp_cpu_rendezvous(uint32* var);
void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3,
void *data_ptr, uint32 flags);
void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
void smp_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
void *data_ptr, uint32 flags);
void smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
void smp_multicast_ici(const CPUSet& cpuMask, int32 message, addr_t data,
addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
void smp_multicast_ici_interrupts_disabled(int32 currentCPU, const CPUSet& cpuMask,
int32 message, addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
int32 smp_get_num_cpus(void);
void smp_set_num_cpus(int32 numCPUs);
src/system/kernel/arch/arm/paging/ARMVMTranslationMap.cpp
+14 -15
View File
@@ -104,42 +104,41 @@ ARMVMTranslationMap::Flush()
fInvalidPagesCount);
if (fIsKernelMap) {
arch_cpu_global_tlb_invalidate();
smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
smp_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
NULL, SMP_MSG_FLAG_SYNC);
} else {
cpu_status state = disable_interrupts();
arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state);
int cpu = smp_get_current_cpu();
int32 cpu = smp_get_current_cpu();
CPUSet cpuMask = PagingStructures()->active_on_cpus;
cpuMask.ClearBit(cpu);
if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
cpuMask.SetBit(cpu);
smp_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
0, 0, 0, NULL, SMP_MSG_FLAG_SYNC);
} else {
cpu_status state = disable_interrupts();
arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state);
}
}
} else {
TRACE("flush_tmap: %d pages to invalidate, invalidate list\n",
fInvalidPagesCount);
arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
if (fIsKernelMap) {
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
smp_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC);
} else {
int cpu = smp_get_current_cpu();
int32 cpu = smp_get_current_cpu();
CPUSet cpuMask = PagingStructures()->active_on_cpus;
cpuMask.ClearBit(cpu);
if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
cpuMask.SetBit(cpu);
smp_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC);
} else {
arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
}
}
}
src/system/kernel/arch/arm/paging/arm_physical_page_mapper_large_memory.cpp
+1 -1
View File
@@ -508,7 +508,7 @@ LargeMemoryPhysicalPageMapper::GetPage(phys_addr_t physicalAddress,
*handle = slot;
*virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE;
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, 0,
smp_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, 0,
*virtualAddress, *virtualAddress, NULL, SMP_MSG_FLAG_SYNC);
return B_OK;
src/system/kernel/arch/m68k/paging/M68KVMTranslationMap.cpp
+14 -15
View File
@@ -104,42 +104,41 @@ M68KVMTranslationMap::Flush()
fInvalidPagesCount);
if (fIsKernelMap) {
arch_cpu_global_tlb_invalidate();
smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
smp_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
NULL, SMP_MSG_FLAG_SYNC);
} else {
cpu_status state = disable_interrupts();
arch_cpu_user_tlb_invalidate();
restore_interrupts(state);
int cpu = smp_get_current_cpu();
int32 cpu = smp_get_current_cpu();
CPUSet cpuMask = PagingStructures()->active_on_cpus;
cpuMask.ClearBit(cpu);
if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
cpuMask.SetBit(cpu);
smp_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
0, 0, 0, NULL, SMP_MSG_FLAG_SYNC);
} else {
cpu_status state = disable_interrupts();
arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state);
}
}
} else {
TRACE("flush_tmap: %d pages to invalidate, invalidate list\n",
fInvalidPagesCount);
arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
if (fIsKernelMap) {
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
smp_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC);
} else {
int cpu = smp_get_current_cpu();
int32 cpu = smp_get_current_cpu();
CPUSet cpuMask = PagingStructures()->active_on_cpus;
cpuMask.ClearBit(cpu);
if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
cpuMask.SetBit(cpu);
smp_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC);
} else {
arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
}
}
}
src/system/kernel/arch/m68k/paging/m68k_physical_page_mapper_large_memory.cpp
+1 -1
View File
@@ -506,7 +506,7 @@ LargeMemoryPhysicalPageMapper::GetPage(phys_addr_t physicalAddress,
*handle = slot;
*virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE;
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, 0,
smp_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, 0,
*virtualAddress, *virtualAddress, NULL, SMP_MSG_FLAG_SYNC);
return B_OK;
src/system/kernel/arch/ppc/paging/PPCVMTranslationMap.cpp
+14 -15
View File
@@ -104,42 +104,41 @@ PPCVMTranslationMap::Flush()
fInvalidPagesCount);
if (fIsKernelMap) {
arch_cpu_global_tlb_invalidate();
smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
smp_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
NULL, SMP_MSG_FLAG_SYNC);
} else {
cpu_status state = disable_interrupts();
arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state);
int cpu = smp_get_current_cpu();
int32 cpu = smp_get_current_cpu();
CPUSet cpuMask = PagingStructures()->active_on_cpus;
cpuMask.ClearBit(cpu);
if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
cpuMask.SetBit(cpu);
smp_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
0, 0, 0, NULL, SMP_MSG_FLAG_SYNC);
} else {
cpu_status state = disable_interrupts();
arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state);
}
}
} else {
TRACE("flush_tmap: %d pages to invalidate, invalidate list\n",
fInvalidPagesCount);
arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
if (fIsKernelMap) {
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
smp_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC);
} else {
int cpu = smp_get_current_cpu();
int32 cpu = smp_get_current_cpu();
CPUSet cpuMask = PagingStructures()->active_on_cpus;
cpuMask.ClearBit(cpu);
if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
cpuMask.SetBit(cpu);
smp_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC);
} else {
arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
}
}
}
src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp
+14 -16
View File
@@ -637,43 +637,41 @@ RISCV64VMTranslationMap::Flush()
fInvalidPagesCount);
if (fIsKernel) {
arch_cpu_global_tlb_invalidate();
smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
smp_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
NULL, SMP_MSG_FLAG_SYNC);
} else {
cpu_status state = disable_interrupts();
arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state);
int cpu = smp_get_current_cpu();
int32 cpu = smp_get_current_cpu();
CPUSet cpuMask = fActiveOnCpus;
cpuMask.ClearBit(cpu);
if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
cpuMask.SetBit(cpu);
smp_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
0, 0, 0, NULL, SMP_MSG_FLAG_SYNC);
} else {
cpu_status state = disable_interrupts();
arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state);
}
}
} else {
TRACE("flush_tmap: %d pages to invalidate, invalidate list\n",
fInvalidPagesCount);
arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
if (fIsKernel) {
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
smp_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC);
} else {
int cpu = smp_get_current_cpu();
int32 cpu = smp_get_current_cpu();
CPUSet cpuMask = fActiveOnCpus;
cpuMask.ClearBit(cpu);
if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
cpuMask.SetBit(cpu);
smp_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC);
} else {
arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
}
}
}
src/system/kernel/arch/x86/paging/X86VMTranslationMap.cpp
+18 -22
View File
@@ -104,23 +104,20 @@ X86VMTranslationMap::Flush()
fInvalidPagesCount);
if (fIsKernelMap) {
cpu_status state = disable_interrupts();
arch_cpu_global_tlb_invalidate();
restore_interrupts(state);
smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
smp_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
NULL, SMP_MSG_FLAG_SYNC);
} else {
cpu_status state = disable_interrupts();
arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state);
int cpu = smp_get_current_cpu();
int32 cpu = smp_get_current_cpu();
CPUSet cpuMask = PagingStructures()->active_on_cpus;
cpuMask.ClearBit(cpu);
if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
cpuMask.SetBit(cpu);
smp_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
x86_read_cr3(), 0, 0, NULL, SMP_MSG_FLAG_SYNC);
} else {
cpu_status state = disable_interrupts();
arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state);
}
}
} else {
@@ -128,21 +125,20 @@ X86VMTranslationMap::Flush()
fInvalidPagesCount);
if (fIsKernelMap) {
arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
0, (addr_t)fInvalidPages, fInvalidPagesCount,
NULL, SMP_MSG_FLAG_SYNC);
smp_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC);
} else {
arch_cpu_invalidate_tlb_list(x86_read_cr3(), fInvalidPages, fInvalidPagesCount);
int cpu = smp_get_current_cpu();
int32 cpu = smp_get_current_cpu();
CPUSet cpuMask = PagingStructures()->active_on_cpus;
cpuMask.ClearBit(cpu);
if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
x86_read_cr3(), (addr_t)fInvalidPages, fInvalidPagesCount,
NULL, SMP_MSG_FLAG_SYNC);
cpuMask.SetBit(cpu);
smp_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
x86_read_cr3(), (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC);
} else {
arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
}
}
}
src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp
+1 -1
View File
@@ -503,7 +503,7 @@ LargeMemoryPhysicalPageMapper::GetPage(phys_addr_t physicalAddress,
*handle = slot;
*virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE;
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, 0,
smp_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, 0,
*virtualAddress, *virtualAddress, NULL, SMP_MSG_FLAG_SYNC);
return B_OK;
src/system/kernel/debug/debug.cpp
+4 -1
View File
@@ -975,7 +975,10 @@ enter_kernel_debugger(int32 cpu, int32& previousCPU)
// First entry on a MP system, send a halt request to all of the other
// CPUs. Should they try to enter the debugger they will be caught in
// the loop above.
smp_send_broadcast_ici_interrupts_disabled(cpu, SMP_MSG_CPU_HALT, 0, 0,
CPUSet cpuMask;
cpuMask.SetAll();
cpuMask.ClearBit(cpu);
smp_multicast_ici_interrupts_disabled(cpu, cpuMask, SMP_MSG_CPU_HALT, 0, 0,
0, NULL, SMP_MSG_FLAG_SYNC);
}
src/system/kernel/smp.cpp
+160 -115
View File
@@ -839,18 +839,9 @@ finish_message_processing(int currentCPU, struct smp_msg* msg,
}
static status_t
process_pending_ici(int32 currentCPU)
static void
invoke_smp_msg(struct smp_msg* msg, int currentCPU, bool* haltCPU)
{
mailbox_source sourceMailbox;
struct smp_msg* msg = check_for_message(currentCPU, sourceMailbox);
if (msg == NULL)
return B_ENTRY_NOT_FOUND;
TRACE(" cpu %" B_PRId32 " message = %" B_PRId32 "\n", currentCPU, msg->message);
bool haltCPU = false;
switch (msg->message) {
case SMP_MSG_INVALIDATE_PAGE_RANGE:
arch_cpu_invalidate_tlb_range(msg->data, msg->data2, msg->data3);
@@ -865,7 +856,7 @@ process_pending_ici(int32 currentCPU)
arch_cpu_global_tlb_invalidate();
break;
case SMP_MSG_CPU_HALT:
haltCPU = true;
*haltCPU = true;
break;
case SMP_MSG_CALL_FUNCTION:
{
@@ -882,6 +873,21 @@ process_pending_ici(int32 currentCPU)
msg->message);
break;
}
}
static status_t
process_pending_ici(int32 currentCPU)
{
mailbox_source sourceMailbox;
struct smp_msg* msg = check_for_message(currentCPU, sourceMailbox);
if (msg == NULL)
return B_ENTRY_NOT_FOUND;
TRACE(" cpu %" B_PRId32 " message = %" B_PRId32 "\n", currentCPU, msg->message);
bool haltCPU = false;
invoke_smp_msg(msg, currentCPU, &haltCPU);
// finish dealing with this message, possibly removing it from the list
finish_message_processing(currentCPU, msg, sourceMailbox);
@@ -1032,32 +1038,108 @@ smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2,
void
smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
addr_t data2, addr_t data3, void *dataPointer, uint32 flags)
smp_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
void *dataPointer, uint32 flags)
{
if (!sICIEnabled)
return;
if (!sICIEnabled) {
smp_msg dummy;
dummy.message = message;
dummy.data = data;
dummy.data2 = data2;
dummy.data3 = data3;
dummy.data_ptr = dataPointer;
int currentCPU = smp_get_current_cpu();
cpu_status state = disable_interrupts();
invoke_smp_msg(&dummy, 0, NULL);
restore_interrupts(state);
return;
}
TRACE("smp_broadcast_ici: cpu %" B_PRId32 " mess 0x%" B_PRIx32 ", data 0x%lx, data2 "
"0x%lx, data3 0x%lx, ptr %p, flags 0x%" B_PRIx32 "\n", smp_get_current_cpu(),
message, data, data2, data3, dataPointer, flags);
// find_free_message leaves interrupts disabled
struct smp_msg *msg;
int state = find_free_message(&msg);
cpu_status state = find_free_message(&msg);
msg->proc_bitmap = cpuMask;
msg->proc_bitmap.ClearBit(currentCPU);
int32 currentCPU = smp_get_current_cpu();
msg->message = message;
msg->data = data;
msg->data2 = data2;
msg->data3 = data3;
msg->data_ptr = dataPointer;
msg->ref_count = sNumCPUs;
msg->flags = flags;
msg->proc_bitmap.SetAll();
msg->done = 0;
TRACE("smp_broadcast_ici%d: inserting msg %p into broadcast "
"mbox\n", currentCPU, msg);
// stick it in the appropriate cpu's mailbox
acquire_write_spinlock_nocheck(&sBroadcastMessageSpinlock);
msg->next = sBroadcastMessages;
sBroadcastMessages = msg;
release_write_spinlock(&sBroadcastMessageSpinlock);
atomic_add(&sBroadcastMessageCounter, 1);
arch_smp_send_broadcast_ici();
TRACE("smp_broadcast_ici: sent interrupt\n");
if ((flags & SMP_MSG_FLAG_SYNC) != 0) {
// wait for the other cpus to finish processing it
// the interrupt handler will ref count it to <0
// if the message is sync after it has removed it from the mailbox
TRACE("smp_broadcast_ici: waiting for ack\n");
while (msg->done == 0) {
process_all_pending_ici(currentCPU);
cpu_wait(&msg->done, 1);
}
TRACE("smp_broadcast_ici: returning message to free list\n");
// for SYNC messages, it's our responsibility to put it
// back into the free list
return_free_message(msg);
} else {
// make sure this CPU has processed the message at least
process_all_pending_ici(currentCPU);
}
restore_interrupts(state);
}
void
smp_multicast_ici(const CPUSet& cpuMask, int32 message, addr_t data,
addr_t data2, addr_t data3, void *dataPointer, uint32 flags)
{
if (!sICIEnabled) {
// how did you decide what CPUs to interrupt?
ASSERT(false);
return;
}
int32 targetCPUs = 0;
for (int32 i = 0; i < sNumCPUs; i++) {
if (msg->proc_bitmap.GetBit(i))
if (cpuMask.GetBit(i))
targetCPUs++;
}
if (targetCPUs == 0) {
panic("smp_send_multicast_ici(): 0 CPU mask");
panic("smp_multicast_ici(): 0 CPU mask");
return;
}
// find_free_message leaves interrupts disabled
struct smp_msg *msg;
cpu_status state = find_free_message(&msg);
msg->message = message;
msg->data = data;
msg->data2 = data2;
@@ -1066,8 +1148,12 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
msg->ref_count = targetCPUs;
msg->flags = flags;
msg->done = 0;
msg->proc_bitmap = cpuMask;
bool broadcast = targetCPUs == sNumCPUs - 1;
int32 currentCPU = smp_get_current_cpu();
bool self = cpuMask.GetBit(currentCPU);
bool broadcast = (!self && targetCPUs == sNumCPUs - 1)
|| (self && targetCPUs == sNumCPUs);
// stick it in the broadcast mailbox
acquire_write_spinlock_nocheck(&sBroadcastMessageSpinlock);
@@ -1081,10 +1167,13 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
atomic_add(&gCPU[i].ici_counter, 1);
}
if (broadcast)
if (broadcast) {
arch_smp_send_broadcast_ici();
else
arch_smp_send_multicast_ici(cpuMask);
} else {
CPUSet sendMask = cpuMask;
sendMask.ClearBit(currentCPU);
arch_smp_send_multicast_ici(sendMask);
}
if ((flags & SMP_MSG_FLAG_SYNC) != 0) {
// wait for the other cpus to finish processing it
@@ -1098,6 +1187,9 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
// for SYNC messages, it's our responsibility to put it
// back into the free list
return_free_message(msg);
} else if (self) {
// make sure this CPU has processed the message at least
process_all_pending_ici(currentCPU);
}
restore_interrupts(state);
@@ -1105,84 +1197,27 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
void
smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
void *dataPointer, uint32 flags)
{
struct smp_msg *msg;
TRACE("smp_send_broadcast_ici: cpu %" B_PRId32 " mess 0x%" B_PRIx32 ", data 0x%lx, data2 "
"0x%lx, data3 0x%lx, ptr %p, flags 0x%" B_PRIx32 "\n", smp_get_current_cpu(),
message, data, data2, data3, dataPointer, flags);
if (sICIEnabled) {
// find_free_message leaves interrupts disabled
cpu_status state = find_free_message(&msg);
int32 currentCPU = smp_get_current_cpu();
msg->message = message;
msg->data = data;
msg->data2 = data2;
msg->data3 = data3;
msg->data_ptr = dataPointer;
msg->ref_count = sNumCPUs - 1;
msg->flags = flags;
msg->proc_bitmap.SetAll();
msg->proc_bitmap.ClearBit(currentCPU);
msg->done = 0;
TRACE("smp_send_broadcast_ici%d: inserting msg %p into broadcast "
"mbox\n", currentCPU, msg);
// stick it in the appropriate cpu's mailbox
acquire_write_spinlock_nocheck(&sBroadcastMessageSpinlock);
msg->next = sBroadcastMessages;
sBroadcastMessages = msg;
release_write_spinlock(&sBroadcastMessageSpinlock);
atomic_add(&sBroadcastMessageCounter, 1);
atomic_add(&gCPU[currentCPU].ici_counter, 1);
arch_smp_send_broadcast_ici();
TRACE("smp_send_broadcast_ici: sent interrupt\n");
if ((flags & SMP_MSG_FLAG_SYNC) != 0) {
// wait for the other cpus to finish processing it
// the interrupt handler will ref count it to <0
// if the message is sync after it has removed it from the mailbox
TRACE("smp_send_broadcast_ici: waiting for ack\n");
while (msg->done == 0) {
process_all_pending_ici(currentCPU);
cpu_wait(&msg->done, 1);
}
TRACE("smp_send_broadcast_ici: returning message to free list\n");
// for SYNC messages, it's our responsibility to put it
// back into the free list
return_free_message(msg);
}
restore_interrupts(state);
}
TRACE("smp_send_broadcast_ici: done\n");
}
void
smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
addr_t data, addr_t data2, addr_t data3, void *dataPointer, uint32 flags)
smp_multicast_ici_interrupts_disabled(int32 currentCPU, const CPUSet& cpuMask,
int32 message, addr_t data, addr_t data2, addr_t data3, void *dataPointer, uint32 flags)
{
if (!sICIEnabled)
return;
TRACE("smp_send_broadcast_ici_interrupts_disabled: cpu %" B_PRId32 " mess 0x%" B_PRIx32 ", "
TRACE("smp_multicast_ici_interrupts_disabled: cpu %" B_PRId32 " mess 0x%" B_PRIx32 ", "
"data 0x%lx, data2 0x%lx, data3 0x%lx, ptr %p, flags 0x%" B_PRIx32 "\n",
currentCPU, message, data, data2, data3, dataPointer, flags);
int32 targetCPUs = 0;
for (int32 i = 0; i < sNumCPUs; i++) {
if (cpuMask.GetBit(i))
targetCPUs++;
}
if (targetCPUs == 0) {
panic("smp_multicast_ici_interrupts_disabled(): 0 CPU mask");
return;
}
struct smp_msg *msg;
find_free_message_interrupts_disabled(currentCPU, &msg);
@@ -1191,13 +1226,16 @@ smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
msg->data2 = data2;
msg->data3 = data3;
msg->data_ptr = dataPointer;
msg->ref_count = sNumCPUs - 1;
msg->ref_count = targetCPUs;
msg->flags = flags;
msg->proc_bitmap.SetAll();
msg->proc_bitmap.ClearBit(currentCPU);
msg->done = 0;
msg->proc_bitmap = cpuMask;
TRACE("smp_send_broadcast_ici_interrupts_disabled %" B_PRId32 ": inserting msg %p "
bool self = cpuMask.GetBit(currentCPU);
bool broadcast = (!self && targetCPUs == sNumCPUs - 1)
|| (self && targetCPUs == sNumCPUs);
TRACE("smp_multicast_ici_interrupts_disabled %" B_PRId32 ": inserting msg %p "
"into broadcast mbox\n", currentCPU, msg);
// stick it in the appropriate cpu's mailbox
@@ -1207,18 +1245,27 @@ smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
release_write_spinlock(&sBroadcastMessageSpinlock);
atomic_add(&sBroadcastMessageCounter, 1);
atomic_add(&gCPU[currentCPU].ici_counter, 1);
for (int32 i = 0; i < sNumCPUs; i++) {
if (!cpuMask.GetBit(i))
atomic_add(&gCPU[i].ici_counter, 1);
}
arch_smp_send_broadcast_ici();
if (broadcast) {
arch_smp_send_broadcast_ici();
} else {
CPUSet sendMask = cpuMask;
sendMask.ClearBit(currentCPU);
arch_smp_send_multicast_ici(sendMask);
}
TRACE("smp_send_broadcast_ici_interrupts_disabled %" B_PRId32 ": sent interrupt\n",
TRACE("smp_multicast_ici_interrupts_disabled %" B_PRId32 ": sent interrupt\n",
currentCPU);
if ((flags & SMP_MSG_FLAG_SYNC) != 0) {
// wait for the other cpus to finish processing it
// the interrupt handler will ref count it to <0
// if the message is sync after it has removed it from the mailbox
TRACE("smp_send_broadcast_ici_interrupts_disabled %" B_PRId32 ": waiting for "
TRACE("smp_multicast_ici_interrupts_disabled %" B_PRId32 ": waiting for "
"ack\n", currentCPU);
while (msg->done == 0) {
@@ -1226,15 +1273,18 @@ smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
cpu_wait(&msg->done, 1);
}
TRACE("smp_send_broadcast_ici_interrupts_disabled %" B_PRId32 ": returning "
TRACE("smp_multicast_ici_interrupts_disabled %" B_PRId32 ": returning "
"message to free list\n", currentCPU);
// for SYNC messages, it's our responsibility to put it
// back into the free list
return_free_message(msg);
} else if (self) {
// make sure this CPU has processed the message at least
process_all_pending_ici(currentCPU);
}
TRACE("smp_send_broadcast_ici_interrupts_disabled: done\n");
TRACE("smp_multicast_ici_interrupts_disabled: done\n");
}
@@ -1435,17 +1485,12 @@ call_all_cpus(void (*func)(void*, int), void* cookie, bool sync)
return;
}
thread_pin_to_current_cpu(thread_get_current_thread());
if (sNumCPUs > 1) {
smp_send_broadcast_ici(SMP_MSG_CALL_FUNCTION, (addr_t)cookie,
smp_broadcast_ici(SMP_MSG_CALL_FUNCTION, (addr_t)cookie,
0, 0, (void*)func, sync ? SMP_MSG_FLAG_SYNC : SMP_MSG_FLAG_ASYNC);
} else {
func(cookie, smp_get_current_cpu());
}
// we need to call this function ourselves as well
func(cookie, smp_get_current_cpu());
thread_unpin_from_current_cpu(thread_get_current_thread());
}