diff --git a/headers/private/kernel/arch/riscv64/arch_thread_types.h b/headers/private/kernel/arch/riscv64/arch_thread_types.h index 1081777a3e..e4ca0f86ff 100644 --- a/headers/private/kernel/arch/riscv64/arch_thread_types.h +++ b/headers/private/kernel/arch/riscv64/arch_thread_types.h @@ -17,9 +17,8 @@ struct iframe { uint64 status; uint64 cause; uint64 tval; - uint64 align1; // structure need to be 16 byte aligned - uint64 ra; + uint64 ra __attribute__((aligned (16))); uint64 t6; uint64 sp; uint64 gp; @@ -58,7 +57,6 @@ struct arch_context { uint64 ra; // 0 uint64 s[12]; // 12 uint64 sp; // 13 - uint64 satp; // 14 }; struct fpu_context { diff --git a/headers/private/system/arch/riscv64/arch_cpu_defs.h b/headers/private/system/arch/riscv64/arch_cpu_defs.h index 631fc5878a..827a383c9c 100644 --- a/headers/private/system/arch/riscv64/arch_cpu_defs.h +++ b/headers/private/system/arch/riscv64/arch_cpu_defs.h @@ -256,7 +256,7 @@ static B_ALWAYS_INLINE void FlushTlbPage(uint64 x) { static B_ALWAYS_INLINE void FlushTlbAllAsid(uint64 asid) { asm volatile("sfence.vma x0, %0" : : "r" (asid) : "memory");} static B_ALWAYS_INLINE void FlushTlbPageAsid(uint64 page, uint64 asid) { - asm volatile("sfence.vma %0, %0" : : "r" (page), "r" (asid) : "memory");} + asm volatile("sfence.vma %0, %1" : : "r" (page), "r" (asid) : "memory");} // flush instruction cache static B_ALWAYS_INLINE void FenceI() { diff --git a/src/system/boot/platform/efi/arch/riscv64/arch_mmu.cpp b/src/system/boot/platform/efi/arch/riscv64/arch_mmu.cpp index 53e2870593..1da7552f8e 100644 --- a/src/system/boot/platform/efi/arch/riscv64/arch_mmu.cpp +++ b/src/system/boot/platform/efi/arch/riscv64/arch_mmu.cpp @@ -158,7 +158,7 @@ LookupPte(addr_t virtAdr, bool alloc) if (pte->ppn == 0) return NULL; memset((Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE); - pte->flags |= (1 << pteValid); + pte->flags |= (1 << pteValid) | (IS_KERNEL_ADDRESS(virtAdr) ? (1 << pteGlobal) : 0); } pte = (Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn); } @@ -175,7 +175,9 @@ Map(addr_t virtAdr, phys_addr_t physAdr, uint64 flags) if (pte == NULL) panic("can't allocate page table"); pte->ppn = physAdr / B_PAGE_SIZE; - pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty) | flags; + pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty) + | (IS_KERNEL_ADDRESS(virtAdr) ? (1 << pteGlobal) : 0) + | flags; } @@ -232,7 +234,7 @@ PreallocKernelRange() pte->ppn = mmu_allocate_page() / B_PAGE_SIZE; if (pte->ppn == 0) panic("can't alloc early physical page"); memset(VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE); - pte->flags |= (1 << pteValid); + pte->flags |= (1 << pteValid) | (1 << pteGlobal); } } diff --git a/src/system/boot/platform/riscv/mmu.cpp b/src/system/boot/platform/riscv/mmu.cpp index cc92bd3097..ca1446fa0c 100644 --- a/src/system/boot/platform/riscv/mmu.cpp +++ b/src/system/boot/platform/riscv/mmu.cpp @@ -146,7 +146,7 @@ LookupPte(addr_t virtAdr, bool alloc) if (pte->ppn == 0) return NULL; memset((Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE); - pte->flags |= (1 << pteValid); + pte->flags |= (1 << pteValid) | (IS_KERNEL_ADDRESS(virtAdr) ? (1 << pteGlobal) : 0); } pte = (Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn); } @@ -164,7 +164,9 @@ Map(addr_t virtAdr, phys_addr_t physAdr, uint64 flags) panic("can't allocate page table"); pte->ppn = physAdr / B_PAGE_SIZE; - pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty) | flags; + pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty) + | (IS_KERNEL_ADDRESS(virtAdr) ? (1 << pteGlobal) : 0) + | flags; } @@ -216,7 +218,7 @@ PreallocKernelRange() pte->ppn = AllocPhysPage() / B_PAGE_SIZE; if (pte->ppn == 0) panic("can't alloc early physical page"); memset(VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE); - pte->flags |= (1 << pteValid); + pte->flags |= (1 << pteValid) | (1 << pteGlobal); } } diff --git a/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp b/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp index b00e98bda2..4a01174ca0 100644 --- a/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp +++ b/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp @@ -202,7 +202,7 @@ RISCV64VMTranslationMap::LookupPte(addr_t virtAdr, bool alloc, return NULL; DEBUG_PAGE_ACCESS_END(page); fPageTableSize++; - pte->flags |= (1 << pteValid); + pte->flags |= (1 << pteValid) | (fIsKernel ? (1 << pteGlobal) : 0); } pte = (Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn); } @@ -319,7 +319,7 @@ RISCV64VMTranslationMap::Map(addr_t virtualAddress, phys_addr_t physicalAddress, Pte newPte; newPte.ppn = physicalAddress / B_PAGE_SIZE; - newPte.flags = (1 << pteValid); + newPte.flags = (1 << pteValid) | (fIsKernel ? (1 << pteGlobal) : 0); if ((attributes & B_USER_PROTECTION) != 0) { newPte.flags |= (1 << pteUser); @@ -702,7 +702,7 @@ status_t RISCV64VMTranslationMap::Protect(addr_t base, addr_t top, Pte oldPte = *pte; Pte newPte = oldPte; - newPte.flags &= (1 << pteValid) + newPte.flags &= (1 << pteValid) | (1 << pteGlobal) | (1 << pteAccessed) | (1 << pteDirty); if ((attributes & B_USER_PROTECTION) != 0) { @@ -760,16 +760,24 @@ ConvertAccessedFlags(uint32 flags) } -status_t +void RISCV64VMTranslationMap::SetFlags(addr_t address, uint32 flags) { - ThreadCPUPinner pinner(thread_get_current_thread()); + // Only called from interrupt handler with interrupts disabled for CPUs that don't support + // setting accessed/modified flags by hardware. + Pte* pte = LookupPte(address, false, NULL); if (pte == NULL || ((1 << pteValid) & pte->flags) == 0) - return B_OK; + return; + pte->flags |= ConvertAccessedFlags(flags); - FlushTlbPage(address); - return B_OK; + + if (IS_KERNEL_ADDRESS(address)) + FlushTlbPage(address); + else + FlushTlbPageAsid(address, 0); + + return; } @@ -852,15 +860,7 @@ RISCV64VMTranslationMap::Flush() if (fInvalidPagesCount <= 0) return; -/* - dprintf("+Flush(%p)\n", this); - struct ScopeExit { - ~ScopeExit() - { - dprintf("-Flush(%p)\n", this); - } - } scopeExit; -*/ + ThreadCPUPinner pinner(thread_get_current_thread()); if (fInvalidPagesCount > PAGE_INVALIDATE_CACHE_SIZE) { @@ -871,11 +871,8 @@ RISCV64VMTranslationMap::Flush() if (fIsKernel) { arch_cpu_global_TLB_invalidate(); - // dprintf("+smp_send_broadcast_ici\n"); smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, NULL, SMP_MSG_FLAG_SYNC); - // dprintf("-smp_send_broadcast_ici\n"); - } else { cpu_status state = disable_interrupts(); arch_cpu_user_TLB_invalidate(); @@ -886,10 +883,8 @@ RISCV64VMTranslationMap::Flush() cpuMask.ClearBit(cpu); if (!cpuMask.IsEmpty()) { - // dprintf("+smp_send_multicast_ici\n"); smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES, 0, 0, 0, NULL, SMP_MSG_FLAG_SYNC); - // dprintf("-smp_send_multicast_ici\n"); } } } else { @@ -899,22 +894,18 @@ RISCV64VMTranslationMap::Flush() arch_cpu_invalidate_TLB_list(fInvalidPages, fInvalidPagesCount); if (fIsKernel) { - // dprintf("+smp_send_broadcast_ici\n"); smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, (addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, SMP_MSG_FLAG_SYNC); - // dprintf("-smp_send_broadcast_ici\n"); } else { int cpu = smp_get_current_cpu(); CPUSet cpuMask = fActiveOnCpus; cpuMask.ClearBit(cpu); if (!cpuMask.IsEmpty()) { - // dprintf("+smp_send_multicast_ici\n"); smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST, (addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, SMP_MSG_FLAG_SYNC); - // dprintf("-smp_send_multicast_ici\n"); } } } diff --git a/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.h b/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.h index 145ed0488c..df9e02ba60 100644 --- a/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.h +++ b/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.h @@ -62,7 +62,7 @@ struct RISCV64VMTranslationMap: public VMTranslationMap { status_t ProtectArea(VMArea* area, uint32 attributes); - status_t SetFlags(addr_t virtualAddress, + void SetFlags(addr_t virtualAddress, uint32 flags); virtual status_t ClearFlags(addr_t virtualAddress, diff --git a/src/system/kernel/arch/riscv64/arch_asm.S b/src/system/kernel/arch/riscv64/arch_asm.S index b34f5eaef4..f5a567363c 100644 --- a/src/system/kernel/arch/riscv64/arch_asm.S +++ b/src/system/kernel/arch/riscv64/arch_asm.S @@ -33,8 +33,6 @@ FUNCTION(arch_context_switch): sd s10, 11*8(a0) sd s11, 12*8(a0) sd sp, 13*8(a0) - csrr t0, satp - sd t0, 14*8(a0) # load `to` context ld ra, 0*8(a1) @@ -51,9 +49,6 @@ FUNCTION(arch_context_switch): ld s10, 11*8(a1) ld s11, 12*8(a1) ld sp, 13*8(a1) - ld t0, 14*8(a1) - csrw satp, t0 - sfence.vma ret FUNCTION_END(arch_context_switch) diff --git a/src/system/kernel/arch/riscv64/arch_cpu.cpp b/src/system/kernel/arch/riscv64/arch_cpu.cpp index 972336a733..290284b5e8 100644 --- a/src/system/kernel/arch/riscv64/arch_cpu.cpp +++ b/src/system/kernel/arch/riscv64/arch_cpu.cpp @@ -14,6 +14,8 @@ #include #include +#include + extern "C" void SVec(); @@ -99,25 +101,29 @@ arch_cpu_sync_icache(void *address, size_t len) } -void -arch_cpu_memory_read_barrier(void) -{ -} - - -void -arch_cpu_memory_write_barrier(void) -{ -} - - void arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) { - int32 numPages = end / B_PAGE_SIZE - start / B_PAGE_SIZE; - while (numPages-- >= 0) { - FlushTlbPage(start); - start += B_PAGE_SIZE; + addr_t kernelStart = std::max(start, KERNEL_BASE); + addr_t kernelEnd = std::min(end, KERNEL_TOP); + + addr_t userStart = std::max(start, USER_BASE); + addr_t userEnd = std::min(end, USER_TOP); + + if (kernelStart <= kernelEnd) { + int64 numPages = kernelStart / B_PAGE_SIZE - kernelEnd / B_PAGE_SIZE; + while (numPages-- >= 0) { + FlushTlbPage(start); + start += B_PAGE_SIZE; + } + } + + if (userStart <= userEnd) { + int64 numPages = userStart / B_PAGE_SIZE - userEnd / B_PAGE_SIZE; + while (numPages-- >= 0) { + FlushTlbPageAsid(start, 0); + start += B_PAGE_SIZE; + } } } @@ -125,8 +131,13 @@ arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) void arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) { - for (int i = 0; i < num_pages; i++) - FlushTlbPage(pages[i]); + for (int i = 0; i < num_pages; i++) { + addr_t page = pages[i]; + if (IS_KERNEL_ADDRESS(page)) + FlushTlbPage(page); + else + FlushTlbPageAsid(page, 0); + } } @@ -140,7 +151,7 @@ arch_cpu_global_TLB_invalidate(void) void arch_cpu_user_TLB_invalidate(void) { - FlushTlbAll(); + FlushTlbAllAsid(0); } diff --git a/src/system/kernel/arch/riscv64/arch_debug.cpp b/src/system/kernel/arch/riscv64/arch_debug.cpp index 9ecd2ee36d..a2c159a230 100644 --- a/src/system/kernel/arch/riscv64/arch_debug.cpp +++ b/src/system/kernel/arch/riscv64/arch_debug.cpp @@ -15,6 +15,8 @@ #include #include +#include "RISCV64VMTranslationMap.h" + kernel_args *sKernelArgs; bool sInitCalled = false; @@ -321,11 +323,19 @@ stack_trace(int argc, char **argv) kprintf("could not find thread %" B_PRId32 "\n", id); return 0; } + + auto map = (RISCV64VMTranslationMap*)thread->team->address_space->TranslationMap(); + uint64 oldSatp = Satp(); - SetSatp(thread->arch_info.context.satp); + SetSatp(map->Satp()); + FlushTlbAllAsid(0); + DebuggedThreadSetter threadSetter(thread); DoStackTraceEx(thread, thread->arch_info.context.s[0], thread->arch_info.context.ra); + SetSatp(oldSatp); + FlushTlbAllAsid(0); + return 0; } DoStackTrace(Fp(), 0); diff --git a/src/system/kernel/arch/riscv64/arch_int.cpp b/src/system/kernel/arch/riscv64/arch_int.cpp index 935d68dc27..e86fe34a60 100644 --- a/src/system/kernel/arch/riscv64/arch_int.cpp +++ b/src/system/kernel/arch/riscv64/arch_int.cpp @@ -334,9 +334,6 @@ SetAccessedFlags(addr_t addr, bool isWrite) != (PAGE_ACCESSED | PAGE_MODIFIED) ) { map->SetFlags(addr, PAGE_ACCESSED | PAGE_MODIFIED); -/* - dprintf("SetAccessedFlags(%#" B_PRIxADDR ", %d)\n", addr, isWrite); -*/ return true; } } else { @@ -345,9 +342,6 @@ SetAccessedFlags(addr_t addr, bool isWrite) && (PAGE_ACCESSED & pageFlags) == 0 ) { map->SetFlags(addr, PAGE_ACCESSED); -/* - dprintf("SetAccessedFlags(%#" B_PRIxADDR ", %d)\n", addr, isWrite); -*/ return true; } } diff --git a/src/system/kernel/arch/riscv64/arch_thread.cpp b/src/system/kernel/arch/riscv64/arch_thread.cpp index d305647a11..0f3a7cdfef 100644 --- a/src/system/kernel/arch/riscv64/arch_thread.cpp +++ b/src/system/kernel/arch/riscv64/arch_thread.cpp @@ -68,9 +68,6 @@ arch_thread_init_kthread_stack(Thread* thread, void* _stack, void* _stackTop, thread->arch_info.context.s[1] = (addr_t)function; thread->arch_info.context.s[2] = (addr_t)data; thread->arch_info.context.ra = (addr_t)arch_thread_entry; - RISCV64VMTranslationMap* map = (RISCV64VMTranslationMap*) - thread->team->address_space->TranslationMap(); - thread->arch_info.context.satp = map->Satp(); memset(&thread->arch_info.fpuContext, 0, sizeof(fpu_context)); } @@ -93,11 +90,8 @@ arch_thread_context_switch(Thread *from, Thread *to) to, to->name); */ - RISCV64VMTranslationMap* fromMap = (RISCV64VMTranslationMap*)from->team - ->address_space->TranslationMap(); - - RISCV64VMTranslationMap* toMap = (RISCV64VMTranslationMap*)to->team - ->address_space->TranslationMap(); + auto fromMap = (RISCV64VMTranslationMap*)from->team->address_space->TranslationMap(); + auto toMap = (RISCV64VMTranslationMap*)to->team->address_space->TranslationMap(); int cpu = to->cpu->cpu_num; toMap->ActiveOnCpus().SetBitAtomic(cpu); @@ -107,6 +101,9 @@ arch_thread_context_switch(Thread *from, Thread *to) save_fpu(&from->arch_info.fpuContext); restore_fpu(&to->arch_info.fpuContext); + SetSatp(toMap->Satp()); + FlushTlbAllAsid(0); + arch_context_switch(&from->arch_info.context, &to->arch_info.context); } diff --git a/src/system/kernel/arch/riscv64/arch_vm.cpp b/src/system/kernel/arch/riscv64/arch_vm.cpp index b64ceba11c..87b58be985 100644 --- a/src/system/kernel/arch/riscv64/arch_vm.cpp +++ b/src/system/kernel/arch/riscv64/arch_vm.cpp @@ -361,7 +361,7 @@ arch_vm_aspace_swap(struct VMAddressSpace *from, struct VMAddressSpace *to) // go away as long as they are still used on any CPU. SetSatp(((RISCV64VMTranslationMap*)to->TranslationMap())->Satp()); - FlushTlbAll(); + FlushTlbAllAsid(0); } diff --git a/src/system/kernel/arch/riscv64/arch_vm_translation_map.cpp b/src/system/kernel/arch/riscv64/arch_vm_translation_map.cpp index e35301e62e..bf4bc0d952 100644 --- a/src/system/kernel/arch/riscv64/arch_vm_translation_map.cpp +++ b/src/system/kernel/arch/riscv64/arch_vm_translation_map.cpp @@ -55,7 +55,7 @@ LookupPte(addr_t virtAdr, bool alloc, kernel_args* args, if (pte->ppn == 0) return NULL; memset((Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE); - pte->flags |= (1 << pteValid); + pte->flags |= (1 << pteValid) | (1 << pteGlobal); } pte = (Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn); } @@ -73,7 +73,9 @@ Map(addr_t virtAdr, phys_addr_t physAdr, uint64 flags, kernel_args* args, if (pte == NULL) panic("can't allocate page table"); pte->ppn = physAdr / B_PAGE_SIZE; - pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty) | flags; + pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty) + | (1 << pteGlobal) // we map only kernel pages here so always set global flag + | flags; FlushTlbPage(virtAdr); }