From f0e1bca09c3659b22615c642b24624e3d56144ed Mon Sep 17 00:00:00 2001 From: X512 Date: Sun, 6 Jun 2021 22:20:49 +0900 Subject: [PATCH] kernel/arch/int: implement for riscv64 Change-Id: I62d9bff75d35a685983c626720514ff17b1cef00 Reviewed-on: https://review.haiku-os.org/c/haiku/+/4052 Reviewed-by: Alex von Gluck IV --- .../private/kernel/arch/riscv64/arch_int.h | 29 +- .../private/kernel/arch/riscv64/arch_traps.h | 88 +++ src/system/kernel/arch/riscv64/Jamfile | 1 + src/system/kernel/arch/riscv64/arch_int.cpp | 518 +++++++++++++++++- src/system/kernel/arch/riscv64/arch_traps.S | 66 +++ 5 files changed, 687 insertions(+), 15 deletions(-) create mode 100644 headers/private/kernel/arch/riscv64/arch_traps.h create mode 100644 src/system/kernel/arch/riscv64/arch_traps.S diff --git a/headers/private/kernel/arch/riscv64/arch_int.h b/headers/private/kernel/arch/riscv64/arch_int.h index 3fd814d512..8df2659273 100644 --- a/headers/private/kernel/arch/riscv64/arch_int.h +++ b/headers/private/kernel/arch/riscv64/arch_int.h @@ -10,6 +10,7 @@ #define _KERNEL_ARCH_RISCV64_INT_H #include +#include #define NUM_IO_VECTORS 256 @@ -17,30 +18,38 @@ static inline void arch_int_enable_interrupts_inline(void) { - // TODO: implement + // TODO: Use atomic CSRRS? + SstatusReg status(Sstatus()); + status.ie |= (1 << modeS); + SetSstatus(status.val); } static inline int arch_int_disable_interrupts_inline(void) { - // TODO: implement - return 0; + // TODO: Use atomic CSRRC? + SstatusReg status(Sstatus()); + int oldState = ((1 << modeS) & status.ie) != 0; + status.ie &= ~(1 << modeS); + SetSstatus(status.val); + return oldState; } static inline void arch_int_restore_interrupts_inline(int oldState) { - // TODO: implement + if (oldState) + arch_int_enable_interrupts_inline(); } static inline bool arch_int_are_interrupts_enabled_inline(void) { - // TODO: implement - return false; + SstatusReg status(Sstatus()); + return ((1 << modeS) & status.ie) != 0; } @@ -53,4 +62,12 @@ arch_int_are_interrupts_enabled_inline(void) arch_int_are_interrupts_enabled_inline() +enum { + kMSyscallSwitchToSmode = 0, + kMSyscallSetTimer = 1, +}; + +extern "C" status_t MSyscall(uint64 op, ...); + + #endif /* _KERNEL_ARCH_RISCV64_INT_H */ diff --git a/headers/private/kernel/arch/riscv64/arch_traps.h b/headers/private/kernel/arch/riscv64/arch_traps.h new file mode 100644 index 0000000000..8afc947b7e --- /dev/null +++ b/headers/private/kernel/arch/riscv64/arch_traps.h @@ -0,0 +1,88 @@ +/* + * Copyright 2021, Haiku, Inc. + * Distributed under the terms of the MIT License. + */ + + +#ifndef _ARCH_TRAPS_H_ +#define _ARCH_TRAPS_H_ + + +# NOTE: this macro don't save SP, it should be saved manually +.macro PushTrapFrame + addi sp, sp, -256 + + sd ra, 0*8(sp) + sd t6, 1*8(sp) +# sd sp, 2*8(sp) # sp + sd gp, 3*8(sp) + sd tp, 4*8(sp) + sd t0, 5*8(sp) + sd t1, 6*8(sp) + sd t2, 7*8(sp) + sd t5, 8*8(sp) + sd s1, 9*8(sp) + sd a0, 10*8(sp) + sd a1, 11*8(sp) + sd a2, 12*8(sp) + sd a3, 13*8(sp) + sd a4, 14*8(sp) + sd a5, 15*8(sp) + sd a6, 16*8(sp) + sd a7, 17*8(sp) + sd s2, 18*8(sp) + sd s3, 19*8(sp) + sd s4, 20*8(sp) + sd s5, 21*8(sp) + sd s6, 22*8(sp) + sd s7, 23*8(sp) + sd s8, 24*8(sp) + sd s9, 25*8(sp) + sd s10, 26*8(sp) + sd s11, 27*8(sp) + sd t3, 28*8(sp) + sd t4, 29*8(sp) + sd fp, 30*8(sp) + + addi fp, sp, 256 +.endm + + +.macro PopTrapFrame + ld ra, 0*8(sp) + ld t6, 1*8(sp) +# ld sp, 2*8(sp) restore later + ld gp, 3*8(sp) +# ld tp, 4*8(sp) + ld t0, 5*8(sp) + ld t1, 6*8(sp) + ld t2, 7*8(sp) + ld t5, 8*8(sp) + ld s1, 9*8(sp) + ld a0, 10*8(sp) + ld a1, 11*8(sp) + ld a2, 12*8(sp) + ld a3, 13*8(sp) + ld a4, 14*8(sp) + ld a5, 15*8(sp) + ld a6, 16*8(sp) + ld a7, 17*8(sp) + ld s2, 18*8(sp) + ld s3, 19*8(sp) + ld s4, 20*8(sp) + ld s5, 21*8(sp) + ld s6, 22*8(sp) + ld s7, 23*8(sp) + ld s8, 24*8(sp) + ld s9, 25*8(sp) + ld s10, 26*8(sp) + ld s11, 27*8(sp) + ld t3, 28*8(sp) + ld t4, 29*8(sp) + ld fp, 30*8(sp) + + ld sp, 2*8(sp) +.endm + + +#endif // _ARCH_TRAPS_H_ diff --git a/src/system/kernel/arch/riscv64/Jamfile b/src/system/kernel/arch/riscv64/Jamfile index 51afc59b6a..305bed6e73 100644 --- a/src/system/kernel/arch/riscv64/Jamfile +++ b/src/system/kernel/arch/riscv64/Jamfile @@ -7,6 +7,7 @@ SEARCH_SOURCE += [ FDirName $(SUBDIR) $(DOTDOT) generic ] ; KernelMergeObject kernel_arch_riscv64.o : arch_asm.S + arch_traps.S arch_commpage.cpp arch_cpu.cpp arch_debug.cpp diff --git a/src/system/kernel/arch/riscv64/arch_int.cpp b/src/system/kernel/arch/riscv64/arch_int.cpp index 885044d2aa..df3b395a85 100644 --- a/src/system/kernel/arch/riscv64/arch_int.cpp +++ b/src/system/kernel/arch/riscv64/arch_int.cpp @@ -8,24 +8,517 @@ #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "RISCV64VMTranslationMap.h" + +#include + + +static uint32 sBootHartId = 0; +static int32 sPlicContextOfs = 0; + + +extern "C" void SVec(); +extern "C" void SVecU(); + + +//#pragma mark debug output + +void +WriteMode(int mode) +{ + switch (mode) { + case modeU: dprintf("u"); break; + case modeS: dprintf("s"); break; + case modeM: dprintf("m"); break; + default: dprintf("%d", mode); + } +} + + +void +WriteModeSet(uint32_t val) +{ + bool first = true; + dprintf("{"); + for (int i = 0; i < 32; i++) { + if (((1LL << i) & val) != 0) { + if (first) first = false; else dprintf(", "); + WriteMode(i); + } + } + dprintf("}"); +} + + +void +WriteMstatus(uint64_t val) +{ + MstatusReg status(val); + dprintf("("); + dprintf("ie: "); WriteModeSet(status.ie); + dprintf(", pie: "); WriteModeSet(status.pie); + dprintf(", spp: "); WriteMode(status.spp); + dprintf(", mpp: "); WriteMode(status.mpp); + dprintf(", sum: %d", (int)status.sum); + dprintf(")"); +} + + +void +WriteSstatus(uint64_t val) +{ + SstatusReg status(val); + dprintf("("); + dprintf("ie: "); WriteModeSet(status.ie); + dprintf(", pie: "); WriteModeSet(status.pie); + dprintf(", spp: "); WriteMode(status.spp); + dprintf(", sum: %d", (int)status.sum); + dprintf(")"); +} + + +void +WriteInterrupt(uint64_t val) +{ + switch (val) { + case 0 + modeU: dprintf("uSoft"); break; + case 0 + modeS: dprintf("sSoft"); break; + case 0 + modeM: dprintf("mSoft"); break; + case 4 + modeU: dprintf("uTimer"); break; + case 4 + modeS: dprintf("sTimer"); break; + case 4 + modeM: dprintf("mTimer"); break; + case 8 + modeU: dprintf("uExtern"); break; + case 8 + modeS: dprintf("sExtern"); break; + case 8 + modeM: dprintf("mExtern"); break; + default: dprintf("%" B_PRId64, val); + } +} + + +void +WriteInterruptSet(uint64_t val) +{ + bool first = true; + dprintf("{"); + for (int i = 0; i < 64; i++) { + if (((1LL << i) & val) != 0) { + if (first) first = false; else dprintf(", "); + WriteInterrupt(i); + } + } + dprintf("}"); +} + + +void +WriteCause(uint64_t cause) +{ + if ((cause & causeInterrupt) == 0) { + dprintf("exception "); + switch (cause) { + case causeExecMisalign: dprintf("execMisalign"); break; + case causeExecAccessFault: dprintf("execAccessFault"); break; + case causeIllegalInst: dprintf("illegalInst"); break; + case causeBreakpoint: dprintf("breakpoint"); break; + case causeLoadMisalign: dprintf("loadMisalign"); break; + case causeLoadAccessFault: dprintf("loadAccessFault"); break; + case causeStoreMisalign: dprintf("storeMisalign"); break; + case causeStoreAccessFault: dprintf("storeAccessFault"); break; + case causeUEcall: dprintf("uEcall"); break; + case causeSEcall: dprintf("sEcall"); break; + case causeMEcall: dprintf("mEcall"); break; + case causeExecPageFault: dprintf("execPageFault"); break; + case causeLoadPageFault: dprintf("loadPageFault"); break; + case causeStorePageFault: dprintf("storePageFault"); break; + default: dprintf("%" B_PRId64, cause); + } + } else { + dprintf("interrupt "); WriteInterrupt(cause & ~causeInterrupt); + } +} + + +void +WriteTrapInfo() +{ + InterruptsLocker locker; + dprintf("STrap("); WriteCause(Scause()); dprintf(")\n"); + dprintf(" sstatus: "); WriteSstatus(Sstatus()); dprintf("\n"); + dprintf(" sie: "); WriteInterruptSet(Sie()); dprintf("\n"); + dprintf(" sip: "); WriteInterruptSet(Sip()); dprintf("\n"); + //dprintf(" stval: "); WritePC(Stval()); dprintf("\n"); + dprintf(" stval: 0x%" B_PRIx64 "\n", Stval()); + dprintf(" tp: 0x%" B_PRIxADDR "(%s)\n", Tp(), + thread_get_current_thread()->name); +} + + +//#pragma mark - + +static void +SendSignal(debug_exception_type type, uint32 signalNumber, int32 signalCode, + addr_t signalAddress = 0, int32 signalError = B_ERROR) +{ + if (SstatusReg(Sstatus()).spp == modeU) { + struct sigaction action; + Thread* thread = thread_get_current_thread(); + + WriteTrapInfo(); + DoStackTrace(Fp(), 0); + + enable_interrupts(); + + // If the thread has a signal handler for the signal, we simply send it + // the signal. Otherwise we notify the user debugger first. + if ((sigaction(signalNumber, NULL, &action) == 0 + && action.sa_handler != SIG_DFL + && action.sa_handler != SIG_IGN) + || user_debug_exception_occurred(type, signalNumber)) { + Signal signal(signalNumber, signalCode, signalError, + thread->team->id); + signal.SetAddress((void*)signalAddress); + send_signal_to_thread(thread, signal, 0); + } + } else { + WriteTrapInfo(); + panic("Unexpected exception occurred in kernel mode!"); + } +} + + +static void +AfterInterrupt() +{ + if (debug_debugger_running()) + return; + + Thread* thread = thread_get_current_thread(); + cpu_status state = disable_interrupts(); + if (thread->cpu->invoke_scheduler) { + SpinLocker schedulerLocker(thread->scheduler_lock); + scheduler_reschedule(B_THREAD_READY); + schedulerLocker.Unlock(); + restore_interrupts(state); + } else if (thread->post_interrupt_callback != NULL) { + void (*callback)(void*) = thread->post_interrupt_callback; + void* data = thread->post_interrupt_data; + + thread->post_interrupt_callback = NULL; + thread->post_interrupt_data = NULL; + + restore_interrupts(state); + + callback(data); + } +} + + +static bool +SetAccessedFlags(addr_t addr, bool isWrite) +{ + VMAddressSpacePutter addressSpace; + if (IS_KERNEL_ADDRESS(addr)) + addressSpace.SetTo(VMAddressSpace::GetKernel()); + else if (IS_USER_ADDRESS(addr)) + addressSpace.SetTo(VMAddressSpace::GetCurrent()); + + if(!addressSpace.IsSet()) + return false; + + RISCV64VMTranslationMap* map + = (RISCV64VMTranslationMap*)addressSpace->TranslationMap(); + + phys_addr_t physAdr; + uint32 pageFlags; + map->QueryInterrupt(addr, &physAdr, &pageFlags); + if (isWrite) { + if ( + ((B_WRITE_AREA | B_KERNEL_WRITE_AREA) & pageFlags) != 0 + && ((PAGE_ACCESSED | PAGE_MODIFIED) & pageFlags) + != (PAGE_ACCESSED | PAGE_MODIFIED) + ) { + map->SetFlags(addr, PAGE_ACCESSED | PAGE_MODIFIED); +/* + dprintf("SetAccessedFlags(%#" B_PRIxADDR ", %d)\n", addr, isWrite); +*/ + return true; + } + } else { + if ( + ((B_READ_AREA | B_KERNEL_READ_AREA) & pageFlags) != 0 + && (PAGE_ACCESSED & pageFlags) == 0 + ) { + map->SetFlags(addr, PAGE_ACCESSED); +/* + dprintf("SetAccessedFlags(%#" B_PRIxADDR ", %d)\n", addr, isWrite); +*/ + return true; + } + } + return false; +} + + +static void +WriteProtection(uint32 flags) +{ + dprintf("kernel: {"); + if (B_KERNEL_READ_AREA & flags) dprintf("R"); + if (B_KERNEL_WRITE_AREA & flags) dprintf("W"); + if (B_KERNEL_EXECUTE_AREA & flags) dprintf("X"); + if (B_KERNEL_STACK_AREA & flags) dprintf("S"); + dprintf("}, user: {"); + if (B_READ_AREA & flags) dprintf("R"); + if (B_WRITE_AREA & flags) dprintf("W"); + if (B_EXECUTE_AREA & flags) dprintf("X"); + if (B_STACK_AREA & flags) dprintf("S"); + dprintf("}"); +} + + +// TODO: needs moved into an arch-agnostic location? + +template +class ScopeExit +{ +public: + explicit ScopeExit(F&& fn) : fFn(fn) + { + } + + ~ScopeExit() + { + fFn(); + } + + ScopeExit(ScopeExit&& other) : fFn(std::move(other.fFn)) + { + } + +private: + ScopeExit(const ScopeExit&); + ScopeExit& operator=(const ScopeExit&); + +private: + F fFn; +}; + +template +ScopeExit MakeScopeExit(F&& fn) +{ + return ScopeExit(std::move(fn)); +} + + +extern "C" void +STrap(iframe* frame) +{ + // dprintf("STrap("); WriteCause(Scause()); dprintf(")\n"); + + SstatusReg status(Sstatus()); + uint64 cause = Scause(); + + const auto& statusRestorer = MakeScopeExit([&]() { + SetSstatus(status.val); + }); + + switch (cause) { + case causeExecPageFault: + case causeLoadPageFault: + case causeStorePageFault: { + if (SetAccessedFlags(Stval(), cause == causeStorePageFault)) + return; + } + } + + if (status.spp == modeU) { + thread_get_current_thread()->arch_info.userFrame = frame; + thread_at_kernel_entry(system_time()); + } + const auto& kernelExit = MakeScopeExit([&]() { + if (status.spp == modeU) { + disable_interrupts(); + if ((thread_get_current_thread()->flags + & (THREAD_FLAGS_SIGNALS_PENDING + | THREAD_FLAGS_DEBUG_THREAD + | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) { + enable_interrupts(); + thread_at_kernel_exit(); + } else { + thread_at_kernel_exit_no_signals(); + } + thread_get_current_thread()->arch_info.userFrame = NULL; + } + }); + + switch (cause) { + case causeIllegalInst: { + return SendSignal(B_INVALID_OPCODE_EXCEPTION, SIGILL, ILL_ILLOPC, + frame->epc); + } + case causeExecMisalign: + case causeLoadMisalign: + case causeStoreMisalign: { + return SendSignal(B_ALIGNMENT_EXCEPTION, SIGBUS, BUS_ADRALN, + Stval()); + } + // case causeBreakpoint: + case causeExecAccessFault: + case causeLoadAccessFault: + case causeStoreAccessFault: { + return SendSignal(B_SEGMENT_VIOLATION, SIGBUS, BUS_ADRERR, + Stval()); + } + case causeExecPageFault: + case causeLoadPageFault: + case causeStorePageFault: { + uint64 stval = Stval(); + + if (debug_debugger_running()) { + Thread* thread = thread_get_current_thread(); + if (thread != NULL) { + cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; + if (cpu->fault_handler != 0) { + debug_set_page_fault_info(stval, frame->epc, + (cause == causeStorePageFault) + ? DEBUG_PAGE_FAULT_WRITE : 0); + frame->epc = cpu->fault_handler; + frame->sp = cpu->fault_handler_stack_pointer; + return; + } + + if (thread->fault_handler != 0) { + kprintf("ERROR: thread::fault_handler used in kernel " + "debugger!\n"); + debug_set_page_fault_info(stval, frame->epc, + cause == causeStorePageFault + ? DEBUG_PAGE_FAULT_WRITE : 0); + frame->epc = (addr_t)thread->fault_handler; + return; + } + } + + panic("page fault in debugger without fault handler! Touching " + "address %p from ip %p\n", (void*)stval, (void*)frame->epc); + return; + } + + if (status.pie == 0) { + WriteTrapInfo(); + panic("page fault with interrupts disabled@!dump_virt_page %#" B_PRIx64, stval); + } + + addr_t newIP = 0; + enable_interrupts(); + vm_page_fault(stval, frame->epc, cause == causeStorePageFault, + cause == causeExecPageFault, status.spp == modeU, status.sum != 0, &newIP); + if (newIP != 0) + frame->epc = newIP; + + return; + } + case causeInterrupt + sTimerInt: { + timer_interrupt(); + AfterInterrupt(); + return; + } + case causeInterrupt + sExternInt: { + // TODO: get PLIC context ID mapping for HARD ID from FDT? + uint64 irq = gPlicRegs->contexts[modeS + 2 * sBootHartId + + sPlicContextOfs].claimAndComplete; + int_io_interrupt_handler(irq, true); + gPlicRegs->contexts[modeS + 2*sBootHartId + + sPlicContextOfs].claimAndComplete = irq; + AfterInterrupt(); + return; + } + case causeUEcall: { + frame->epc += 4; // skip ecall + uint64 syscall = frame->t0; + uint64 args[20]; + if (syscall < (uint64)kSyscallCount) { + uint32 argCnt = kExtendedSyscallInfos[syscall].parameter_count; + memcpy(&args[0], &frame->a0, + sizeof(uint64)*std::min(argCnt, 8)); + if (argCnt > 8) { + if (status_t res = user_memcpy(&args[8], (void*)frame->sp, + sizeof(uint64)*(argCnt - 8)) < B_OK) { + dprintf("can't read syscall arguments on user " + "stack\n"); + frame->a0 = res; + return; + } + } + } +/* + switch (syscall) { + case SYSCALL_READ_PORT_ETC: + case SYSCALL_WRITE_PORT_ETC: + WriteTrapInfo(); + DoStackTrace(Fp(), 0); + break; + } +*/ + // dprintf("syscall: %s\n", kExtendedSyscallInfos[syscall].name); + enable_interrupts(); + uint64 returnValue = 0; + syscall_dispatcher(syscall, (void*)args, &returnValue); + frame->a0 = returnValue; + return; + } + } + WriteTrapInfo(); + panic("unhandled STrap"); +} + + +//#pragma mark - + +status_t +arch_int_init(kernel_args* args) +{ + sBootHartId = args->arch_args.bootHart; + sPlicContextOfs = (sBootHartId == 0) ? 0 : -1; + + // TODO: Kernel mode FPU handling needs improved? + SetStvec((uint64)SVec); + SstatusReg sstatus(Sstatus()); + sstatus.ie = 0; + sstatus.fs = extStatusInitial; // enable FPU + sstatus.xs = extStatusOff; + SetSstatus(sstatus.val); + SetSie(Sie() | (1 << sTimerInt) | (1 << sExternInt)); + + // TODO: read from FDT + reserve_io_interrupt_vectors(128, 0, INTERRUPT_TYPE_IRQ); + + gPlicRegs->contexts[modeS + 2*sBootHartId + sPlicContextOfs].priorityThreshold = 0; + return B_OK; +} status_t -arch_int_init(kernel_args *args) +arch_int_init_post_vm(kernel_args* args) { return B_OK; } status_t -arch_int_init_post_vm(kernel_args *args) -{ - return B_OK; -} - - -status_t -arch_int_init_post_device_manager(struct kernel_args *args) +arch_int_init_post_device_manager(struct kernel_args* args) { return B_OK; } @@ -41,16 +534,23 @@ arch_int_init_io(kernel_args* args) void arch_int_enable_io_interrupt(int irq) { + dprintf("arch_int_enable_io_interrupt(%d)\n", irq); + gPlicRegs->priority[irq] = 1; + gPlicRegs->enable[modeS + 2*sBootHartId + sPlicContextOfs][irq / 32] |= 1 << (irq % 32); } void arch_int_disable_io_interrupt(int irq) { + dprintf("arch_int_disable_io_interrupt(%d)\n", irq); + gPlicRegs->priority[irq] = 0; + gPlicRegs->enable[modeS + 2*sBootHartId + sPlicContextOfs][irq / 32] &= ~(1 << (irq % 32)); } void arch_int_assign_to_cpu(int32 irq, int32 cpu) { + // SMP not yet supported } diff --git a/src/system/kernel/arch/riscv64/arch_traps.S b/src/system/kernel/arch/riscv64/arch_traps.S new file mode 100644 index 0000000000..dcadb6ed51 --- /dev/null +++ b/src/system/kernel/arch/riscv64/arch_traps.S @@ -0,0 +1,66 @@ +/* + * Copyright 2021, Haiku, Inc. + * Distributed under the terms of the MIT License. + */ + + +#include "arch_traps.h" + + +.globl SVec +.type SVec, @function +.align 4 +SVec: + PushTrapFrame + sd fp, 2*8(sp) + csrr t0, sepc + sd t0, 31*8(sp) + + mv a0, sp + call STrap + + ld t0, 31*8(sp) + csrw sepc, t0 + PopTrapFrame + sret +.size SVec, .-SVec + + +.globl SVecU +.type SVecU, @function +.align 4 +SVecU: + csrrw t0, sscratch, t0 # t0: &arch_thread + ld tp, 0*8(t0) # tp = arch_thread.thread + ld t0, (1 + 13)*8(t0) # t0 = arch_thread.context.sp + sd sp, 2*8 - 256(t0) # save user SP + mv sp, t0 # switch to kernel stack + csrr t0, sscratch + + PushTrapFrame + + csrr t0, sepc + sd t0, 31*8(sp) + + la t0, SVec + csrw stvec, t0 + + mv a0, sp + call STrap + +.globl SVecURet +.type SVecURet, @function +SVecURet: + call RestoreUserRegs + + csrr t0, sscratch + sd fp, (1 + 13)*8(t0) # arch_thread.context.sp = fp + + la t0, SVecU + csrw stvec, t0 + + ld t0, 31*8(sp) + csrw sepc, t0 + PopTrapFrame + sret +.size SVecU, .-SVecU