i2p/haiku-beta6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kernel: Refactor arch_cpu_invalidate* methods to take a context parameter.

Browse Source 
This will be useful on architectures that support ASIDs, and on
ones that don't, it can be used to avoid a race between invalidation
messages being sent and context switches occurring on other CPUs.

In this commit, no behavioral change intended on any architecture.

Change-Id: I6d2ee4b69150f9ff8503329761266ddbdb550eab
Reviewed-on: https://review.haiku-os.org/c/haiku/+/10349
Reviewed-by: waddlesplash <[email protected]>
Tested-by: Commit checker robot <[email protected]>
Reviewed-by: Adrien Destugues <[email protected]>
This commit is contained in:
Augustin Cavalier
2026-02-26 17:17:06 +00:00
committed by waddlesplash
parent afaeb2d45d
commit 3e1ccaf329
22 changed files with 93 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
headers/private/kernel/arch/cpu.h
+4 -4
View File
@@ -29,10 +29,10 @@ status_t arch_cpu_init_post_vm(struct kernel_args *args);
status_t arch_cpu_init_post_modules(struct kernel_args *args); status_t arch_cpu_init_post_modules(struct kernel_args *args);
status_t arch_cpu_shutdown(bool reboot); status_t arch_cpu_shutdown(bool reboot);
void arch_cpu_invalidate_TLB_range(addr_t start, addr_t end); void arch_cpu_invalidate_tlb_range(intptr_t context, addr_t start, addr_t end);
void arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages); void arch_cpu_invalidate_tlb_list(intptr_t context, addr_t pages[], int num_pages);
void arch_cpu_user_TLB_invalidate(void); void arch_cpu_user_tlb_invalidate(intptr_t context);
void arch_cpu_global_TLB_invalidate(void); void arch_cpu_global_tlb_invalidate(void);
void arch_cpu_sync_icache(void *address, size_t length); void arch_cpu_sync_icache(void *address, size_t length);
src/system/kernel/arch/arm/arch_cpu.cpp
+4 -4
View File
@@ -90,7 +90,7 @@ arch_cpu_invalidate_TLB_page(addr_t page)
} }
void void
arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) arch_cpu_invalidate_tlb_range(intptr_t, addr_t start, addr_t end)
{ {
// ensure visibility of the update to translation table walks // ensure visibility of the update to translation table walks
dsb(); dsb();
@@ -109,7 +109,7 @@ arch_cpu_invalidate_TLB_range(addr_t start, addr_t end)
void void
arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) arch_cpu_invalidate_tlb_list(intptr_t, addr_t pages[], int num_pages)
{ {
// ensure visibility of the update to translation table walks // ensure visibility of the update to translation table walks
dsb(); dsb();
@@ -126,7 +126,7 @@ arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages)
void void
arch_cpu_global_TLB_invalidate(void) arch_cpu_global_tlb_invalidate(void)
{ {
// ensure visibility of the update to translation table walks // ensure visibility of the update to translation table walks
dsb(); dsb();
@@ -142,7 +142,7 @@ arch_cpu_global_TLB_invalidate(void)
void void
arch_cpu_user_TLB_invalidate(void) arch_cpu_user_tlb_invalidate(intptr_t)
{/* {/*
cpu_ops.flush_insn_pipeline(); cpu_ops.flush_insn_pipeline();
cpu_ops.flush_atc_user(); cpu_ops.flush_atc_user();
src/system/kernel/arch/arm/arch_thread.cpp
+1 -1
View File
@@ -128,7 +128,7 @@ arm_swap_pgdir(uint32_t pageDirectoryAddress)
arm_set_ttbr0(pageDirectoryAddress); arm_set_ttbr0(pageDirectoryAddress);
isb(); isb();
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
//TODO: update Context ID (incl. ASID) //TODO: update Context ID (incl. ASID)
//TODO: check if any additional TLB or Cache maintenance is needed //TODO: check if any additional TLB or Cache maintenance is needed
src/system/kernel/arch/arm/paging/ARMVMTranslationMap.cpp
+5 -5
View File
@@ -104,12 +104,12 @@ ARMVMTranslationMap::Flush()
fInvalidPagesCount); fInvalidPagesCount);
if (fIsKernelMap) { if (fIsKernelMap) {
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
NULL, SMP_MSG_FLAG_SYNC); NULL, SMP_MSG_FLAG_SYNC);
} else { } else {
cpu_status state = disable_interrupts(); cpu_status state = disable_interrupts();
arch_cpu_user_TLB_invalidate(); arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state); restore_interrupts(state);
int cpu = smp_get_current_cpu(); int cpu = smp_get_current_cpu();
@@ -125,11 +125,11 @@ ARMVMTranslationMap::Flush()
TRACE("flush_tmap: %d pages to invalidate, invalidate list\n", TRACE("flush_tmap: %d pages to invalidate, invalidate list\n",
fInvalidPagesCount); fInvalidPagesCount);
arch_cpu_invalidate_TLB_list(fInvalidPages, fInvalidPagesCount); arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
if (fIsKernelMap) { if (fIsKernelMap) {
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, 0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC); SMP_MSG_FLAG_SYNC);
} else { } else {
int cpu = smp_get_current_cpu(); int cpu = smp_get_current_cpu();
@@ -138,7 +138,7 @@ ARMVMTranslationMap::Flush()
if (!cpuMask.IsEmpty()) { if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST, smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, 0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC); SMP_MSG_FLAG_SYNC);
} }
} }
src/system/kernel/arch/arm/paging/arm_physical_page_mapper_large_memory.cpp
+3 -3
View File
@@ -395,7 +395,7 @@ LargeMemoryTranslationMapPhysicalPageMapper::GetPageTableAt(
fNextSlot = (i + 1) & (fSlotCount - 1); fNextSlot = (i + 1) & (fSlotCount - 1);
if (!slot.valid.GetBit(currentCPU)) { if (!slot.valid.GetBit(currentCPU)) {
// not valid on this CPU -- invalidate the TLB entry // not valid on this CPU -- invalidate the TLB entry
arch_cpu_invalidate_TLB_range(slot.slot->address, arch_cpu_invalidate_tlb_range(0, slot.slot->address,
slot.slot->address + B_PAGE_SIZE); slot.slot->address + B_PAGE_SIZE);
slot.valid.SetBit(currentCPU); slot.valid.SetBit(currentCPU);
} }
@@ -508,8 +508,8 @@ LargeMemoryPhysicalPageMapper::GetPage(phys_addr_t physicalAddress,
*handle = slot; *handle = slot;
*virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE; *virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE;
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, *virtualAddress, smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, 0,
*virtualAddress, 0, NULL, SMP_MSG_FLAG_SYNC); *virtualAddress, *virtualAddress, NULL, SMP_MSG_FLAG_SYNC);
return B_OK; return B_OK;
} }
src/system/kernel/arch/arm64/arch_cpu.cpp
+7 -7
View File
@@ -87,21 +87,21 @@ arch_cpu_sync_icache(void *address, size_t len)
void void
arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) arch_cpu_invalidate_tlb_range(intptr_t, addr_t start, addr_t end)
{ {
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
} }
void void
arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) arch_cpu_invalidate_tlb_list(intptr_t, addr_t pages[], int num_pages)
{ {
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
} }
void void
arch_cpu_global_TLB_invalidate(void) arch_cpu_global_tlb_invalidate()
{ {
asm( asm(
"dsb ishst\n" "dsb ishst\n"
@@ -113,7 +113,7 @@ arch_cpu_global_TLB_invalidate(void)
void void
arch_cpu_user_TLB_invalidate(void) arch_cpu_user_tlb_invalidate(intptr_t)
{ {
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
} }
src/system/kernel/arch/m68k/arch_cpu.cpp
+4 -4
View File
@@ -126,7 +126,7 @@ arch_cpu_memory_write_barrier(void)
void void
arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) arch_cpu_invalidate_tlb_range(intptr_t, addr_t start, addr_t end)
{ {
int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE;
cpu_ops.flush_insn_pipeline(); cpu_ops.flush_insn_pipeline();
@@ -140,7 +140,7 @@ arch_cpu_invalidate_TLB_range(addr_t start, addr_t end)
void void
arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) arch_cpu_invalidate_tlb_list(intptr_t, addr_t pages[], int num_pages)
{ {
int i; int i;
@@ -154,7 +154,7 @@ arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages)
void void
arch_cpu_global_TLB_invalidate(void) arch_cpu_global_tlb_invalidate()
{ {
cpu_ops.flush_insn_pipeline(); cpu_ops.flush_insn_pipeline();
cpu_ops.flush_atc_all(); cpu_ops.flush_atc_all();
@@ -163,7 +163,7 @@ arch_cpu_global_TLB_invalidate(void)
void void
arch_cpu_user_TLB_invalidate(void) arch_cpu_user_tlb_invalidate(intptr_t)
{ {
cpu_ops.flush_insn_pipeline(); cpu_ops.flush_insn_pipeline();
cpu_ops.flush_atc_user(); cpu_ops.flush_atc_user();
src/system/kernel/arch/m68k/arch_vm_translation_map_impl.cpp
+10 -10
View File
@@ -739,20 +739,20 @@ query_tmap_interrupt(vm_translation_map *map, addr_t va, addr_t *_physical,
index = VADDR_TO_PRENT(va); index = VADDR_TO_PRENT(va);
if (pr && pr[index].type == DT_ROOT) { if (pr && pr[index].type == DT_ROOT) {
put_page_table_entry_in_pgtable(&sQueryDesc, PRE_TO_TA(pr[index]), B_KERNEL_READ_AREA, false); put_page_table_entry_in_pgtable(&sQueryDesc, PRE_TO_TA(pr[index]), B_KERNEL_READ_AREA, false);
arch_cpu_invalidate_TLB_range((addr_t)pt, (addr_t)pt); arch_cpu_invalidate_tlb_range(0, (addr_t)pt, (addr_t)pt);
pd = (page_directory_entry *)sQueryPage; pd = (page_directory_entry *)sQueryPage;
index = VADDR_TO_PDENT(va); index = VADDR_TO_PDENT(va);
if (pd && pd[index].type == DT_DIR) { if (pd && pd[index].type == DT_DIR) {
put_page_table_entry_in_pgtable(&sQueryDesc, PDE_TO_TA(pd[index]), B_KERNEL_READ_AREA, false); put_page_table_entry_in_pgtable(&sQueryDesc, PDE_TO_TA(pd[index]), B_KERNEL_READ_AREA, false);
arch_cpu_invalidate_TLB_range((addr_t)pt, (addr_t)pt); arch_cpu_invalidate_tlb_range(0, (addr_t)pt, (addr_t)pt);
pt = (page_table_entry *)sQueryPage; pt = (page_table_entry *)sQueryPage;
index = VADDR_TO_PTENT(va); index = VADDR_TO_PTENT(va);
if (pt && pt[index].type == DT_INDIRECT) { if (pt && pt[index].type == DT_INDIRECT) {
pi = (page_indirect_entry *)pt; pi = (page_indirect_entry *)pt;
put_page_table_entry_in_pgtable(&sQueryDesc, PIE_TO_TA(pi[index]), B_KERNEL_READ_AREA, false); put_page_table_entry_in_pgtable(&sQueryDesc, PIE_TO_TA(pi[index]), B_KERNEL_READ_AREA, false);
arch_cpu_invalidate_TLB_range((addr_t)pt, (addr_t)pt); arch_cpu_invalidate_tlb_range(0, (addr_t)pt, (addr_t)pt);
pt = (page_table_entry *)sQueryPage; pt = (page_table_entry *)sQueryPage;
index = 0; // single descriptor index = 0; // single descriptor
} }
@@ -1046,15 +1046,15 @@ flush_tmap(vm_translation_map *map)
map->arch_data->num_invalidate_pages)); map->arch_data->num_invalidate_pages));
if (IS_KERNEL_MAP(map)) { if (IS_KERNEL_MAP(map)) {
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
} else { } else {
arch_cpu_user_TLB_invalidate(); arch_cpu_user_tlb_invalidate(0);
} }
} else { } else {
TRACE(("flush_tmap: %d pages to invalidate, invalidate list\n", TRACE(("flush_tmap: %d pages to invalidate, invalidate list\n",
map->arch_data->num_invalidate_pages)); map->arch_data->num_invalidate_pages));
arch_cpu_invalidate_TLB_list(map->arch_data->pages_to_invalidate, arch_cpu_invalidate_tlb_list(0, map->arch_data->pages_to_invalidate,
map->arch_data->num_invalidate_pages); map->arch_data->num_invalidate_pages);
} }
map->arch_data->num_invalidate_pages = 0; map->arch_data->num_invalidate_pages = 0;
@@ -1090,9 +1090,9 @@ map_iospace_chunk(addr_t va, addr_t pa, uint32 flags)
} }
state = disable_interrupts(); state = disable_interrupts();
arch_cpu_invalidate_TLB_range(va, va + (IOSPACE_CHUNK_SIZE - B_PAGE_SIZE)); arch_cpu_invalidate_tlb_range(0, va, va + (IOSPACE_CHUNK_SIZE - B_PAGE_SIZE));
//smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, //smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, 0,
// va, va + (IOSPACE_CHUNK_SIZE - B_PAGE_SIZE), 0, // va, va + (IOSPACE_CHUNK_SIZE - B_PAGE_SIZE),
// NULL, SMP_MSG_FLAG_SYNC); // NULL, SMP_MSG_FLAG_SYNC);
restore_interrupts(state); restore_interrupts(state);
@@ -1510,7 +1510,7 @@ m68k_vm_translation_map_early_map(kernel_args *args, addr_t va, addr_t pa,
put_page_table_entry_in_pgtable(&pt[index], pa, attributes, put_page_table_entry_in_pgtable(&pt[index], pa, attributes,
IS_KERNEL_ADDRESS(va)); IS_KERNEL_ADDRESS(va));
arch_cpu_invalidate_TLB_range(va, va); arch_cpu_invalidate_tlb_range(0, va, va);
return B_OK; return B_OK;
} }
src/system/kernel/arch/m68k/paging/040/M68KPagingMethod040.cpp
+2 -2
View File
@@ -252,7 +252,7 @@ M68KPagingMethod040::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress,
pte = TA_TO_PTEA(physicalAddress) | DT_PAGE pte = TA_TO_PTEA(physicalAddress) | DT_PAGE
| M68K_PTE_SUPERVISOR | M68K_PTE_GLOBAL; | M68K_PTE_SUPERVISOR | M68K_PTE_GLOBAL;
arch_cpu_invalidate_TLB_range(virtualAddress, virtualAddress); arch_cpu_invalidate_tlb_range(0, virtualAddress, virtualAddress);
} }
@@ -541,7 +541,7 @@ M68KPagingMethod040::MapEarly(kernel_args* args, addr_t virtualAddress,
PutPageTableEntryInTable(&pt[index], PutPageTableEntryInTable(&pt[index],
physicalAddress, attributes, 0, IS_KERNEL_ADDRESS(virtualAddress)); physicalAddress, attributes, 0, IS_KERNEL_ADDRESS(virtualAddress));
arch_cpu_invalidate_TLB_range(va, va); arch_cpu_invalidate_tlb_range(0, va, va);
return B_OK; return B_OK;
src/system/kernel/arch/m68k/paging/M68KVMTranslationMap.cpp
+5 -5
View File
@@ -104,12 +104,12 @@ M68KVMTranslationMap::Flush()
fInvalidPagesCount); fInvalidPagesCount);
if (fIsKernelMap) { if (fIsKernelMap) {
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
NULL, SMP_MSG_FLAG_SYNC); NULL, SMP_MSG_FLAG_SYNC);
} else { } else {
cpu_status state = disable_interrupts(); cpu_status state = disable_interrupts();
arch_cpu_user_TLB_invalidate(); arch_cpu_user_tlb_invalidate();
restore_interrupts(state); restore_interrupts(state);
int cpu = smp_get_current_cpu(); int cpu = smp_get_current_cpu();
@@ -125,11 +125,11 @@ M68KVMTranslationMap::Flush()
TRACE("flush_tmap: %d pages to invalidate, invalidate list\n", TRACE("flush_tmap: %d pages to invalidate, invalidate list\n",
fInvalidPagesCount); fInvalidPagesCount);
arch_cpu_invalidate_TLB_list(fInvalidPages, fInvalidPagesCount); arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
if (fIsKernelMap) { if (fIsKernelMap) {
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, 0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC); SMP_MSG_FLAG_SYNC);
} else { } else {
int cpu = smp_get_current_cpu(); int cpu = smp_get_current_cpu();
@@ -138,7 +138,7 @@ M68KVMTranslationMap::Flush()
if (!cpuMask.IsEmpty()) { if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST, smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, 0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC); SMP_MSG_FLAG_SYNC);
} }
} }
src/system/kernel/arch/m68k/paging/m68k_physical_page_mapper_large_memory.cpp
+3 -3
View File
@@ -399,7 +399,7 @@ LargeMemoryTranslationMapPhysicalPageMapper::GetPageTableAt(
fNextSlot = (i + 1) & (fSlotCount - 1); fNextSlot = (i + 1) & (fSlotCount - 1);
if ((slot.valid.GetBit(currentCPU)) == 0) { if ((slot.valid.GetBit(currentCPU)) == 0) {
// not valid on this CPU -- invalidate the TLB entry // not valid on this CPU -- invalidate the TLB entry
arch_cpu_invalidate_TLB_range(slot.slot->address, arch_cpu_invalidate_tlb_range(0, slot.slot->address,
slot.slot->address); slot.slot->address);
slot.valid.SetBit(currentCPU); slot.valid.SetBit(currentCPU);
} }
@@ -506,8 +506,8 @@ LargeMemoryPhysicalPageMapper::GetPage(phys_addr_t physicalAddress,
*handle = slot; *handle = slot;
*virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE; *virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE;
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, *virtualAddress, smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, 0,
*virtualAddress, 0, NULL, SMP_MSG_FLAG_SYNC); *virtualAddress, *virtualAddress, NULL, SMP_MSG_FLAG_SYNC);
return B_OK; return B_OK;
} }
src/system/kernel/arch/ppc/arch_cpu.cpp
+5 -5
View File
@@ -110,7 +110,7 @@ arch_cpu_memory_write_barrier(void)
void void
arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) arch_cpu_invalidate_tlb_range(intptr_t, addr_t start, addr_t end)
{ {
asm volatile("sync"); asm volatile("sync");
while (start < end) { while (start < end) {
@@ -125,7 +125,7 @@ arch_cpu_invalidate_TLB_range(addr_t start, addr_t end)
void void
arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) arch_cpu_invalidate_tlb_list(intptr_t, addr_t pages[], int num_pages)
{ {
int i; int i;
@@ -141,7 +141,7 @@ arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages)
void void
arch_cpu_global_TLB_invalidate(void) arch_cpu_global_tlb_invalidate()
{ {
if (sHasTlbia) { if (sHasTlbia) {
ppc_sync(); ppc_sync();
@@ -166,9 +166,9 @@ arch_cpu_global_TLB_invalidate(void)
void void
arch_cpu_user_TLB_invalidate(void) arch_cpu_user_tlb_invalidate(intptr_t)
{ {
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
} }
src/system/kernel/arch/ppc/paging/PPCVMTranslationMap.cpp
+5 -5
View File
@@ -104,12 +104,12 @@ PPCVMTranslationMap::Flush()
fInvalidPagesCount); fInvalidPagesCount);
if (fIsKernelMap) { if (fIsKernelMap) {
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
NULL, SMP_MSG_FLAG_SYNC); NULL, SMP_MSG_FLAG_SYNC);
} else { } else {
cpu_status state = disable_interrupts(); cpu_status state = disable_interrupts();
arch_cpu_user_TLB_invalidate(); arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state); restore_interrupts(state);
int cpu = smp_get_current_cpu(); int cpu = smp_get_current_cpu();
@@ -125,11 +125,11 @@ PPCVMTranslationMap::Flush()
TRACE("flush_tmap: %d pages to invalidate, invalidate list\n", TRACE("flush_tmap: %d pages to invalidate, invalidate list\n",
fInvalidPagesCount); fInvalidPagesCount);
arch_cpu_invalidate_TLB_list(fInvalidPages, fInvalidPagesCount); arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
if (fIsKernelMap) { if (fIsKernelMap) {
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, 0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC); SMP_MSG_FLAG_SYNC);
} else { } else {
int cpu = smp_get_current_cpu(); int cpu = smp_get_current_cpu();
@@ -138,7 +138,7 @@ PPCVMTranslationMap::Flush()
if (!cpuMask.IsEmpty()) { if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST, smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, 0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC); SMP_MSG_FLAG_SYNC);
} }
} }
src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp
+5 -5
View File
@@ -637,13 +637,13 @@ RISCV64VMTranslationMap::Flush()
fInvalidPagesCount); fInvalidPagesCount);
if (fIsKernel) { if (fIsKernel) {
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
NULL, SMP_MSG_FLAG_SYNC); NULL, SMP_MSG_FLAG_SYNC);
} else { } else {
cpu_status state = disable_interrupts(); cpu_status state = disable_interrupts();
arch_cpu_user_TLB_invalidate(); arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state); restore_interrupts(state);
int cpu = smp_get_current_cpu(); int cpu = smp_get_current_cpu();
@@ -659,11 +659,11 @@ RISCV64VMTranslationMap::Flush()
TRACE("flush_tmap: %d pages to invalidate, invalidate list\n", TRACE("flush_tmap: %d pages to invalidate, invalidate list\n",
fInvalidPagesCount); fInvalidPagesCount);
arch_cpu_invalidate_TLB_list(fInvalidPages, fInvalidPagesCount); arch_cpu_invalidate_tlb_list(0, fInvalidPages, fInvalidPagesCount);
if (fIsKernel) { if (fIsKernel) {
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, 0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC); SMP_MSG_FLAG_SYNC);
} else { } else {
int cpu = smp_get_current_cpu(); int cpu = smp_get_current_cpu();
@@ -672,7 +672,7 @@ RISCV64VMTranslationMap::Flush()
if (!cpuMask.IsEmpty()) { if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST, smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, 0, (addr_t)fInvalidPages, fInvalidPagesCount, NULL,
SMP_MSG_FLAG_SYNC); SMP_MSG_FLAG_SYNC);
} }
} }
src/system/kernel/arch/riscv64/arch_cpu.cpp
+4 -4
View File
@@ -102,7 +102,7 @@ arch_cpu_sync_icache(void *address, size_t len)
void void
arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) arch_cpu_invalidate_tlb_range(intptr_t, addr_t start, addr_t end)
{ {
addr_t kernelStart = std::max<addr_t>(start, KERNEL_BASE); addr_t kernelStart = std::max<addr_t>(start, KERNEL_BASE);
addr_t kernelEnd = std::min<addr_t>(end, KERNEL_TOP); addr_t kernelEnd = std::min<addr_t>(end, KERNEL_TOP);
@@ -129,7 +129,7 @@ arch_cpu_invalidate_TLB_range(addr_t start, addr_t end)
void void
arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) arch_cpu_invalidate_tlb_list(intptr_t, addr_t pages[], int num_pages)
{ {
for (int i = 0; i < num_pages; i++) { for (int i = 0; i < num_pages; i++) {
addr_t page = pages[i]; addr_t page = pages[i];
@@ -142,14 +142,14 @@ arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages)
void void
arch_cpu_global_TLB_invalidate(void) arch_cpu_global_tlb_invalidate(void)
{ {
FlushTlbAll(); FlushTlbAll();
} }
void void
arch_cpu_user_TLB_invalidate(void) arch_cpu_user_tlb_invalidate(intptr_t)
{ {
FlushTlbAllAsid(0); FlushTlbAllAsid(0);
} }
src/system/kernel/arch/sparc/arch_cpu.cpp
+4 -4
View File
@@ -69,25 +69,25 @@ arch_cpu_memory_write_barrier(void)
void void
arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) arch_cpu_invalidate_tlb_range(intptr_t, addr_t start, addr_t end)
{ {
} }
void void
arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) arch_cpu_invalidate_tlb_list(intptr_t, addr_t pages[], int num_pages)
{ {
} }
void void
arch_cpu_global_TLB_invalidate(void) arch_cpu_global_tlb_invalidate()
{ {
} }
void void
arch_cpu_user_TLB_invalidate(void) arch_cpu_user_tlb_invalidate(intptr_t)
{ {
} }
src/system/kernel/arch/x86/arch_cpu.cpp
+7 -7
View File
@@ -165,7 +165,7 @@ disable_caches()
x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE) x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE)
& ~CR0_NOT_WRITE_THROUGH); & ~CR0_NOT_WRITE_THROUGH);
wbinvd(); wbinvd();
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
} }
@@ -174,7 +174,7 @@ static void
enable_caches() enable_caches()
{ {
wbinvd(); wbinvd();
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
x86_write_cr0(x86_read_cr0() x86_write_cr0(x86_read_cr0()
& ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH)); & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH));
} }
@@ -1981,14 +1981,14 @@ arch_cpu_init_post_modules(kernel_args* args)
void void
arch_cpu_user_TLB_invalidate(void) arch_cpu_user_tlb_invalidate(intptr_t)
{ {
x86_write_cr3(x86_read_cr3()); x86_write_cr3(x86_read_cr3());
} }
void void
arch_cpu_global_TLB_invalidate(void) arch_cpu_global_tlb_invalidate()
{ {
uint32 flags = x86_read_cr4(); uint32 flags = x86_read_cr4();
@@ -1999,14 +1999,14 @@ arch_cpu_global_TLB_invalidate(void)
x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES);
} else { } else {
cpu_status state = disable_interrupts(); cpu_status state = disable_interrupts();
arch_cpu_user_TLB_invalidate(); arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state); restore_interrupts(state);
} }
} }
void void
arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) arch_cpu_invalidate_tlb_range(intptr_t, addr_t start, addr_t end)
{ {
int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE;
while (num_pages-- >= 0) { while (num_pages-- >= 0) {
@@ -2017,7 +2017,7 @@ arch_cpu_invalidate_TLB_range(addr_t start, addr_t end)
void void
arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) arch_cpu_invalidate_tlb_list(intptr_t, addr_t pages[], int num_pages)
{ {
int i; int i;
for (i = 0; i < num_pages; i++) { for (i = 0; i < num_pages; i++) {
src/system/kernel/arch/x86/paging/64bit/X86PagingMethod64Bit.cpp
+1 -1
View File
@@ -79,7 +79,7 @@ X86PagingMethod64Bit::Init(kernel_args* args,
memset(virtualPML4, 0, sizeof(uint64) * 256); memset(virtualPML4, 0, sizeof(uint64) * 256);
} }
memset(fKernelVirtualPMLTop, 0, sizeof(uint64) * 256); memset(fKernelVirtualPMLTop, 0, sizeof(uint64) * 256);
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
*_physicalPageMapper = fPhysicalPageMapper; *_physicalPageMapper = fPhysicalPageMapper;
return B_ERROR; return B_ERROR;
src/system/kernel/arch/x86/paging/X86VMTranslationMap.cpp
+7 -7
View File
@@ -104,12 +104,12 @@ X86VMTranslationMap::Flush()
fInvalidPagesCount); fInvalidPagesCount);
if (fIsKernelMap) { if (fIsKernelMap) {
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
NULL, SMP_MSG_FLAG_SYNC); NULL, SMP_MSG_FLAG_SYNC);
} else { } else {
cpu_status state = disable_interrupts(); cpu_status state = disable_interrupts();
arch_cpu_user_TLB_invalidate(); arch_cpu_user_tlb_invalidate(0);
restore_interrupts(state); restore_interrupts(state);
int cpu = smp_get_current_cpu(); int cpu = smp_get_current_cpu();
@@ -125,12 +125,12 @@ X86VMTranslationMap::Flush()
TRACE("flush_tmap: %d pages to invalidate, invalidate list\n", TRACE("flush_tmap: %d pages to invalidate, invalidate list\n",
fInvalidPagesCount); fInvalidPagesCount);
arch_cpu_invalidate_TLB_list(fInvalidPages, fInvalidPagesCount); arch_cpu_invalidate_tlb_list(x86_read_cr3(), fInvalidPages, fInvalidPagesCount);
if (fIsKernelMap) { if (fIsKernelMap) {
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, 0, (addr_t)fInvalidPages, fInvalidPagesCount,
SMP_MSG_FLAG_SYNC); NULL, SMP_MSG_FLAG_SYNC);
} else { } else {
int cpu = smp_get_current_cpu(); int cpu = smp_get_current_cpu();
CPUSet cpuMask = PagingStructures()->active_on_cpus; CPUSet cpuMask = PagingStructures()->active_on_cpus;
@@ -138,8 +138,8 @@ X86VMTranslationMap::Flush()
if (!cpuMask.IsEmpty()) { if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST, smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, 0, (addr_t)fInvalidPages, fInvalidPagesCount,
SMP_MSG_FLAG_SYNC); NULL, SMP_MSG_FLAG_SYNC);
} }
} }
} }
src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp
+2 -2
View File
@@ -503,8 +503,8 @@ LargeMemoryPhysicalPageMapper::GetPage(phys_addr_t physicalAddress,
*handle = slot; *handle = slot;
*virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE; *virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE;
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, *virtualAddress, smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, 0,
*virtualAddress, 0, NULL, SMP_MSG_FLAG_SYNC); *virtualAddress, *virtualAddress, NULL, SMP_MSG_FLAG_SYNC);
return B_OK; return B_OK;
} }
src/system/kernel/main.cpp
+1 -1
View File
@@ -258,7 +258,7 @@ _start(kernel_args *bootKernelArgs, int currentCPU)
// the boot processor has probably been sending us // the boot processor has probably been sending us
// tlb sync messages all along the way, but we've // tlb sync messages all along the way, but we've
// been ignoring them // been ignoring them
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
// this is run for each non boot processor after they've been set loose // this is run for each non boot processor after they've been set loose
smp_per_cpu_init(&sKernelArgs, currentCPU); smp_per_cpu_init(&sKernelArgs, currentCPU);
src/system/kernel/smp.cpp
+4 -4
View File
@@ -924,16 +924,16 @@ process_pending_ici(int32 currentCPU)
switch (msg->message) { switch (msg->message) {
case SMP_MSG_INVALIDATE_PAGE_RANGE: case SMP_MSG_INVALIDATE_PAGE_RANGE:
arch_cpu_invalidate_TLB_range(msg->data, msg->data2); arch_cpu_invalidate_tlb_range(msg->data, msg->data2, msg->data3);
break; break;
case SMP_MSG_INVALIDATE_PAGE_LIST: case SMP_MSG_INVALIDATE_PAGE_LIST:
arch_cpu_invalidate_TLB_list((addr_t*)msg->data, (int)msg->data2); arch_cpu_invalidate_tlb_list(msg->data, (addr_t*)msg->data2, (int)msg->data3);
break; break;
case SMP_MSG_USER_INVALIDATE_PAGES: case SMP_MSG_USER_INVALIDATE_PAGES:
arch_cpu_user_TLB_invalidate(); arch_cpu_user_tlb_invalidate(msg->data);
break; break;
case SMP_MSG_GLOBAL_INVALIDATE_PAGES: case SMP_MSG_GLOBAL_INVALIDATE_PAGES:
arch_cpu_global_TLB_invalidate(); arch_cpu_global_tlb_invalidate();
break; break;
case SMP_MSG_CPU_HALT: case SMP_MSG_CPU_HALT:
haltCPU = true; haltCPU = true;