mmu/riscv64: implement global page mapping support

ASID allocation is not supported yet, so always use ASID 0 for user pages for now.

Change-Id: I021e77dae692c22984bc625dd0588362bece45b7
Reviewed-on: https://review.haiku-os.org/c/haiku/+/6698
Tested-by: Commit checker robot <[email protected]>
Reviewed-by: Adrien Destugues <[email protected]>
Reviewed-by: Alex von Gluck IV <[email protected]>

headers/private/kernel/arch/riscv64/arch_thread_types.h

@@ -17,9 +17,8 @@ struct iframe {
 	uint64 status;
 	uint64 cause;
 	uint64 tval;
-	uint64 align1; // structure need to be 16 byte aligned
 
-	uint64 ra;
+	uint64 ra __attribute__((aligned (16)));
 	uint64 t6;
 	uint64 sp;
 	uint64 gp;
@@ -58,7 +57,6 @@ struct arch_context {
 	uint64 ra;    //  0
 	uint64 s[12]; // 12
 	uint64 sp;    // 13
-	uint64 satp;  // 14
 };
 
 struct fpu_context {

headers/private/system/arch/riscv64/arch_cpu_defs.h

@@ -256,7 +256,7 @@ static B_ALWAYS_INLINE void FlushTlbPage(uint64 x) {
 static B_ALWAYS_INLINE void FlushTlbAllAsid(uint64 asid) {
 	asm volatile("sfence.vma x0, %0" : : "r" (asid) : "memory");}
 static B_ALWAYS_INLINE void FlushTlbPageAsid(uint64 page, uint64 asid) {
-	asm volatile("sfence.vma %0, %0" : : "r" (page), "r" (asid) : "memory");}
+	asm volatile("sfence.vma %0, %1" : : "r" (page), "r" (asid) : "memory");}
 
 // flush instruction cache
 static B_ALWAYS_INLINE void FenceI() {

src/system/boot/platform/efi/arch/riscv64/arch_mmu.cpp

@@ -158,7 +158,7 @@ LookupPte(addr_t virtAdr, bool alloc)
 			if (pte->ppn == 0)
 				return NULL;
 			memset((Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE);
-			pte->flags |= (1 << pteValid);
+			pte->flags |= (1 << pteValid) | (IS_KERNEL_ADDRESS(virtAdr) ? (1 << pteGlobal) : 0);
 		}
 		pte = (Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn);
 	}
@@ -175,7 +175,9 @@ Map(addr_t virtAdr, phys_addr_t physAdr, uint64 flags)
 	if (pte == NULL) panic("can't allocate page table");
 
 	pte->ppn = physAdr / B_PAGE_SIZE;
-	pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty) | flags;
+	pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty)
+		| (IS_KERNEL_ADDRESS(virtAdr) ? (1 << pteGlobal) : 0)
+		| flags;
 }
 
 
@@ -232,7 +234,7 @@ PreallocKernelRange()
 		pte->ppn = mmu_allocate_page() / B_PAGE_SIZE;
 		if (pte->ppn == 0) panic("can't alloc early physical page");
 		memset(VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE);
-		pte->flags |= (1 << pteValid);
+		pte->flags |= (1 << pteValid) | (1 << pteGlobal);
 	}
 }
 

src/system/boot/platform/riscv/mmu.cpp

@@ -146,7 +146,7 @@ LookupPte(addr_t virtAdr, bool alloc)
 			if (pte->ppn == 0)
 				return NULL;
 			memset((Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE);
-			pte->flags |= (1 << pteValid);
+			pte->flags |= (1 << pteValid) | (IS_KERNEL_ADDRESS(virtAdr) ? (1 << pteGlobal) : 0);
 		}
 		pte = (Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn);
 	}
@@ -164,7 +164,9 @@ Map(addr_t virtAdr, phys_addr_t physAdr, uint64 flags)
 		panic("can't allocate page table");
 
 	pte->ppn = physAdr / B_PAGE_SIZE;
-	pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty) | flags;
+	pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty)
+		| (IS_KERNEL_ADDRESS(virtAdr) ? (1 << pteGlobal) : 0)
+		| flags;
 }
 
 
@@ -216,7 +218,7 @@ PreallocKernelRange()
 		pte->ppn = AllocPhysPage() / B_PAGE_SIZE;
 		if (pte->ppn == 0) panic("can't alloc early physical page");
 		memset(VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE);
-		pte->flags |= (1 << pteValid);
+		pte->flags |= (1 << pteValid) | (1 << pteGlobal);
 	}
 }
 

src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp

@@ -202,7 +202,7 @@ RISCV64VMTranslationMap::LookupPte(addr_t virtAdr, bool alloc,
 				return NULL;
 			DEBUG_PAGE_ACCESS_END(page);
 			fPageTableSize++;
-			pte->flags |= (1 << pteValid);
+			pte->flags |= (1 << pteValid) | (fIsKernel ? (1 << pteGlobal) : 0);
 		}
 		pte = (Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn);
 	}
@@ -319,7 +319,7 @@ RISCV64VMTranslationMap::Map(addr_t virtualAddress, phys_addr_t physicalAddress,
 
 	Pte newPte;
 	newPte.ppn = physicalAddress / B_PAGE_SIZE;
-	newPte.flags = (1 << pteValid);
+	newPte.flags = (1 << pteValid) | (fIsKernel ? (1 << pteGlobal) : 0);
 
 	if ((attributes & B_USER_PROTECTION) != 0) {
 		newPte.flags |= (1 << pteUser);
@@ -702,7 +702,7 @@ status_t RISCV64VMTranslationMap::Protect(addr_t base, addr_t top,
 
 		Pte oldPte = *pte;
 		Pte newPte = oldPte;
-		newPte.flags &= (1 << pteValid)
+		newPte.flags &= (1 << pteValid) | (1 << pteGlobal)
 			| (1 << pteAccessed) | (1 << pteDirty);
 
 		if ((attributes & B_USER_PROTECTION) != 0) {
@@ -760,16 +760,24 @@ ConvertAccessedFlags(uint32 flags)
 }
 
 
-status_t
+void
 RISCV64VMTranslationMap::SetFlags(addr_t address, uint32 flags)
 {
-	ThreadCPUPinner pinner(thread_get_current_thread());
+	// Only called from interrupt handler with interrupts disabled for CPUs that don't support
+	// setting accessed/modified flags by hardware.
+
 	Pte* pte = LookupPte(address, false, NULL);
 	if (pte == NULL || ((1 << pteValid) & pte->flags) == 0)
-		return B_OK;
+		return;
+
 	pte->flags |= ConvertAccessedFlags(flags);
-	FlushTlbPage(address);
+
-	return B_OK;
+	if (IS_KERNEL_ADDRESS(address))
+		FlushTlbPage(address);
+	else
+		FlushTlbPageAsid(address, 0);
+
+	return;
 }
 
 
@@ -852,15 +860,7 @@ RISCV64VMTranslationMap::Flush()
 
 	if (fInvalidPagesCount <= 0)
 		return;
-/*
+
-	dprintf("+Flush(%p)\n", this);
-	struct ScopeExit {
-		~ScopeExit()
-		{
-			dprintf("-Flush(%p)\n", this);
-		}
-	} scopeExit;
-*/
 	ThreadCPUPinner pinner(thread_get_current_thread());
 
 	if (fInvalidPagesCount > PAGE_INVALIDATE_CACHE_SIZE) {
@@ -871,11 +871,8 @@ RISCV64VMTranslationMap::Flush()
 		if (fIsKernel) {
 			arch_cpu_global_TLB_invalidate();
 
-			// dprintf("+smp_send_broadcast_ici\n");
 			smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
 				NULL, SMP_MSG_FLAG_SYNC);
-			// dprintf("-smp_send_broadcast_ici\n");
-
 		} else {
 			cpu_status state = disable_interrupts();
 			arch_cpu_user_TLB_invalidate();
@@ -886,10 +883,8 @@ RISCV64VMTranslationMap::Flush()
 			cpuMask.ClearBit(cpu);
 
 			if (!cpuMask.IsEmpty()) {
-				// dprintf("+smp_send_multicast_ici\n");
 				smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
 					0, 0, 0, NULL, SMP_MSG_FLAG_SYNC);
-				// dprintf("-smp_send_multicast_ici\n");
 			}
 		}
 	} else {
@@ -899,22 +894,18 @@ RISCV64VMTranslationMap::Flush()
 		arch_cpu_invalidate_TLB_list(fInvalidPages, fInvalidPagesCount);
 
 		if (fIsKernel) {
-			// dprintf("+smp_send_broadcast_ici\n");
 			smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
 				(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL,
 				SMP_MSG_FLAG_SYNC);
-			// dprintf("-smp_send_broadcast_ici\n");
 		} else {
 			int cpu = smp_get_current_cpu();
 			CPUSet cpuMask = fActiveOnCpus;
 			cpuMask.ClearBit(cpu);
 
 			if (!cpuMask.IsEmpty()) {
-				// dprintf("+smp_send_multicast_ici\n");
 				smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
 					(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL,
 					SMP_MSG_FLAG_SYNC);
-				// dprintf("-smp_send_multicast_ici\n");
 			}
 		}
 	}

src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.h

@@ -62,7 +62,7 @@ struct RISCV64VMTranslationMap: public VMTranslationMap {
 			status_t			ProtectArea(VMArea* area,
 									uint32 attributes);
 
-			status_t			SetFlags(addr_t virtualAddress,
+			void				SetFlags(addr_t virtualAddress,
 									uint32 flags);
 
 	virtual	status_t			ClearFlags(addr_t virtualAddress,

src/system/kernel/arch/riscv64/arch_asm.S

@@ -33,8 +33,6 @@ FUNCTION(arch_context_switch):
 	sd s10, 11*8(a0)
 	sd s11, 12*8(a0)
 	sd sp,  13*8(a0)
-	csrr t0, satp
-	sd t0,  14*8(a0)
 
 # load `to` context
 	ld ra,   0*8(a1)
@@ -51,9 +49,6 @@ FUNCTION(arch_context_switch):
 	ld s10, 11*8(a1)
 	ld s11, 12*8(a1)
 	ld sp,  13*8(a1)
-	ld t0,  14*8(a1)
-	csrw satp, t0
-	sfence.vma
 
 	ret
 FUNCTION_END(arch_context_switch)

src/system/kernel/arch/riscv64/arch_cpu.cpp

@@ -14,6 +14,8 @@
 #include <Htif.h>
 #include <platform/sbi/sbi_syscalls.h>
 
+#include <algorithm>
+
 
 extern "C" void SVec();
 
@@ -99,25 +101,29 @@ arch_cpu_sync_icache(void *address, size_t len)
 }
 
 
-void
-arch_cpu_memory_read_barrier(void)
-{
-}
-
-
-void
-arch_cpu_memory_write_barrier(void)
-{
-}
-
-
 void
 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end)
 {
-	int32 numPages = end / B_PAGE_SIZE - start / B_PAGE_SIZE;
+	addr_t kernelStart = std::max<addr_t>(start, KERNEL_BASE);
-	while (numPages-- >= 0) {
+	addr_t kernelEnd   = std::min<addr_t>(end,   KERNEL_TOP);
-		FlushTlbPage(start);
+
-		start += B_PAGE_SIZE;
+	addr_t userStart = std::max<addr_t>(start, USER_BASE);
+	addr_t userEnd   = std::min<addr_t>(end,   USER_TOP);
+
+	if (kernelStart <= kernelEnd) {
+		int64 numPages = kernelStart / B_PAGE_SIZE - kernelEnd / B_PAGE_SIZE;
+		while (numPages-- >= 0) {
+			FlushTlbPage(start);
+			start += B_PAGE_SIZE;
+		}
+	}
+
+	if (userStart <= userEnd) {
+		int64 numPages = userStart / B_PAGE_SIZE - userEnd / B_PAGE_SIZE;
+		while (numPages-- >= 0) {
+			FlushTlbPageAsid(start, 0);
+			start += B_PAGE_SIZE;
+		}
 	}
 }
 
@@ -125,8 +131,13 @@ arch_cpu_invalidate_TLB_range(addr_t start, addr_t end)
 void
 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages)
 {
-	for (int i = 0; i < num_pages; i++)
+	for (int i = 0; i < num_pages; i++) {
-		FlushTlbPage(pages[i]);
+		addr_t page = pages[i];
+		if (IS_KERNEL_ADDRESS(page))
+			FlushTlbPage(page);
+		else
+			FlushTlbPageAsid(page, 0);
+	}
 }
 
 
@@ -140,7 +151,7 @@ arch_cpu_global_TLB_invalidate(void)
 void
 arch_cpu_user_TLB_invalidate(void)
 {
-	FlushTlbAll();
+	FlushTlbAllAsid(0);
 }
 
 

src/system/kernel/arch/riscv64/arch_debug.cpp

@@ -15,6 +15,8 @@
 #include <arch/generic/user_memory.h>
 #include <AutoDeleterDrivers.h>
 
+#include "RISCV64VMTranslationMap.h"
+
 
 kernel_args *sKernelArgs;
 bool sInitCalled = false;
@@ -321,11 +323,19 @@ stack_trace(int argc, char **argv)
 			kprintf("could not find thread %" B_PRId32 "\n", id);
 			return 0;
 		}
+
+		auto map = (RISCV64VMTranslationMap*)thread->team->address_space->TranslationMap();
+
 		uint64 oldSatp = Satp();
-		SetSatp(thread->arch_info.context.satp);
+		SetSatp(map->Satp());
+		FlushTlbAllAsid(0);
+
 		DebuggedThreadSetter threadSetter(thread);
 		DoStackTraceEx(thread, thread->arch_info.context.s[0], thread->arch_info.context.ra);
+
 		SetSatp(oldSatp);
+		FlushTlbAllAsid(0);
+
 		return 0;
 	}
 	DoStackTrace(Fp(), 0);

src/system/kernel/arch/riscv64/arch_int.cpp

@@ -334,9 +334,6 @@ SetAccessedFlags(addr_t addr, bool isWrite)
 				!= (PAGE_ACCESSED | PAGE_MODIFIED)
 		) {
 			map->SetFlags(addr, PAGE_ACCESSED | PAGE_MODIFIED);
-/*
-			dprintf("SetAccessedFlags(%#" B_PRIxADDR ", %d)\n", addr, isWrite);
-*/
 			return true;
 		}
 	} else {
@@ -345,9 +342,6 @@ SetAccessedFlags(addr_t addr, bool isWrite)
 			&& (PAGE_ACCESSED & pageFlags) == 0
 		) {
 			map->SetFlags(addr, PAGE_ACCESSED);
-/*
-			dprintf("SetAccessedFlags(%#" B_PRIxADDR ", %d)\n", addr, isWrite);
-*/
 			return true;
 		}
 	}

src/system/kernel/arch/riscv64/arch_thread.cpp

@@ -68,9 +68,6 @@ arch_thread_init_kthread_stack(Thread* thread, void* _stack, void* _stackTop,
 	thread->arch_info.context.s[1] = (addr_t)function;
 	thread->arch_info.context.s[2] = (addr_t)data;
 	thread->arch_info.context.ra = (addr_t)arch_thread_entry;
-	RISCV64VMTranslationMap* map = (RISCV64VMTranslationMap*)
-		thread->team->address_space->TranslationMap();
-	thread->arch_info.context.satp = map->Satp();
 
 	memset(&thread->arch_info.fpuContext, 0, sizeof(fpu_context));
 }
@@ -93,11 +90,8 @@ arch_thread_context_switch(Thread *from, Thread *to)
 		to, to->name);
 	*/
 
-	RISCV64VMTranslationMap* fromMap = (RISCV64VMTranslationMap*)from->team
+	auto fromMap = (RISCV64VMTranslationMap*)from->team->address_space->TranslationMap();
-		->address_space->TranslationMap();
+	auto toMap = (RISCV64VMTranslationMap*)to->team->address_space->TranslationMap();
-
-	RISCV64VMTranslationMap* toMap = (RISCV64VMTranslationMap*)to->team
-		->address_space->TranslationMap();
 
 	int cpu = to->cpu->cpu_num;
 	toMap->ActiveOnCpus().SetBitAtomic(cpu);
@@ -107,6 +101,9 @@ arch_thread_context_switch(Thread *from, Thread *to)
 	save_fpu(&from->arch_info.fpuContext);
 	restore_fpu(&to->arch_info.fpuContext);
 
+	SetSatp(toMap->Satp());
+	FlushTlbAllAsid(0);
+
 	arch_context_switch(&from->arch_info.context, &to->arch_info.context);
 }
 

src/system/kernel/arch/riscv64/arch_vm.cpp

@@ -361,7 +361,7 @@ arch_vm_aspace_swap(struct VMAddressSpace *from, struct VMAddressSpace *to)
 	// go away as long as they are still used on any CPU.
 
 	SetSatp(((RISCV64VMTranslationMap*)to->TranslationMap())->Satp());
-	FlushTlbAll();
+	FlushTlbAllAsid(0);
 }
 
 

src/system/kernel/arch/riscv64/arch_vm_translation_map.cpp

@@ -55,7 +55,7 @@ LookupPte(addr_t virtAdr, bool alloc, kernel_args* args,
 			if (pte->ppn == 0)
 				return NULL;
 			memset((Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn), 0, B_PAGE_SIZE);
-			pte->flags |= (1 << pteValid);
+			pte->flags |= (1 << pteValid) | (1 << pteGlobal);
 		}
 		pte = (Pte*)VirtFromPhys(B_PAGE_SIZE * pte->ppn);
 	}
@@ -73,7 +73,9 @@ Map(addr_t virtAdr, phys_addr_t physAdr, uint64 flags, kernel_args* args,
 	if (pte == NULL) panic("can't allocate page table");
 
 	pte->ppn = physAdr / B_PAGE_SIZE;
-	pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty) | flags;
+	pte->flags = (1 << pteValid) | (1 << pteAccessed) | (1 << pteDirty)
+		| (1 << pteGlobal) // we map only kernel pages here so always set global flag
+		| flags;
 
 	FlushTlbPage(virtAdr);
 }