diff --git a/headers/private/kernel/arch/arm64/arch_cpu.h b/headers/private/kernel/arch/arm64/arch_cpu.h index 0eb353ac3c..91a0533935 100644 --- a/headers/private/kernel/arch/arm64/arch_cpu.h +++ b/headers/private/kernel/arch/arm64/arch_cpu.h @@ -110,46 +110,66 @@ arm64_address_translate_ ##stage (uint64 addr) \ return (ret); \ } + ADDRESS_TRANSLATE_FUNC(s1e0r) ADDRESS_TRANSLATE_FUNC(s1e0w) ADDRESS_TRANSLATE_FUNC(s1e1r) ADDRESS_TRANSLATE_FUNC(s1e1w) + +struct aarch64_fpu_state +{ + uint64 regs[32 * 2]; + uint64 fpsr; + uint64 fpcr; +}; + + /* raw exception frames */ struct iframe { - uint64 sp; - uint64 lr; - uint64 elr; - uint32 spsr; - uint32 esr; - uint64 x[30]; + // return info + uint64 elr; + uint64 spsr; + uint64 x[20]; + uint64 lr; + uint64 sp; + + // exception info + uint64 esr; + uint64 far; }; + #ifdef __cplusplus namespace BKernel { struct Thread; } // namespace BKernel + typedef struct arch_cpu_info { uint32 mpidr; BKernel::Thread* last_vfp_user; } arch_cpu_info; #endif + #ifdef __cplusplus extern "C" { #endif + static inline void arch_cpu_pause(void) { arm64_yield(); } + static inline void arch_cpu_idle(void) { arm64_yield(); } + #ifdef __cplusplus } #endif diff --git a/headers/private/kernel/arch/arm64/arch_int.h b/headers/private/kernel/arch/arm64/arch_int.h index a1c292d67e..9d119ea3d7 100644 --- a/headers/private/kernel/arch/arm64/arch_int.h +++ b/headers/private/kernel/arch/arm64/arch_int.h @@ -15,7 +15,7 @@ static inline void arch_int_enable_interrupts_inline(void) { - __asm__ __volatile__("msr daifclr, #2"); + asm volatile("msr daifclr, #0xf" : : : "memory"); } @@ -24,10 +24,7 @@ arch_int_disable_interrupts_inline(void) { uint32 flags; - __asm__ __volatile__( - "mrs %0, daif\n\t" - "msr daifset, #2\n\t" - : "=&r"(flags)); + asm volatile("mrs %0, daif\n" "msr daifset, #0xf" : "=r"(flags) : : "memory"); return flags; } @@ -36,22 +33,17 @@ arch_int_disable_interrupts_inline(void) static inline void arch_int_restore_interrupts_inline(int oldState) { - WRITE_SPECIALREG(daif, oldState); + asm volatile("msr daif, %0" : : "r"(oldState) : "memory"); } static inline bool arch_int_are_interrupts_enabled_inline(void) { - uint32 flags; - - __asm__ __volatile__( - "mrs %0, daif\n\t" - : "=&r"(flags)); - - return (flags & PSR_I) == 0; + return (READ_SPECIALREG(DAIF) & PSR_I) == 0; } + // map the functions to the inline versions #define arch_int_enable_interrupts() arch_int_enable_interrupts_inline() #define arch_int_disable_interrupts() arch_int_disable_interrupts_inline() @@ -60,4 +52,5 @@ arch_int_are_interrupts_enabled_inline(void) #define arch_int_are_interrupts_enabled() \ arch_int_are_interrupts_enabled_inline() + #endif /* _KERNEL_ARCH_ARM64_ARCH_INT_H_ */ diff --git a/src/system/kernel/arch/arm64/Jamfile b/src/system/kernel/arch/arm64/Jamfile index ec6546bdea..42dd103dfa 100644 --- a/src/system/kernel/arch/arm64/Jamfile +++ b/src/system/kernel/arch/arm64/Jamfile @@ -38,3 +38,5 @@ KernelMergeObject kernel_arch_arm64.o : $(TARGET_KERNEL_PIC_CCFLAGS) -Wno-unused : ; + +CreateAsmStructOffsetsHeader asm_offsets.h : asm_offsets.cpp : $(TARGET_KERNEL_ARCH) ; diff --git a/src/system/kernel/arch/arm64/arch_asm.S b/src/system/kernel/arch/arm64/arch_asm.S index b1704595db..c71934e659 100644 --- a/src/system/kernel/arch/arm64/arch_asm.S +++ b/src/system/kernel/arch/arm64/arch_asm.S @@ -4,9 +4,237 @@ */ #include #include +#include "asm_offsets.h" .text +.macro xchg_sp xt +add sp, sp, \xt +sub \xt, sp, \xt +sub sp, sp, \xt +.endm + +.macro EXCEPTION_ENTRY el + // interrupts are automatically disabled by hardware + + // avoid using sp in case it is misaligned + // swap sp with x19 and use it instead + xchg_sp x19 + + // x19 is now the stack top, make room for IFRAME + sub x19, x19, #(IFRAME_sizeof) + + stp x0, x1, [x19, #(IFRAME_x + 0 * 8)] + stp x2, x3, [x19, #(IFRAME_x + 2 * 8)] + stp x4, x5, [x19, #(IFRAME_x + 4 * 8)] + stp x6, x7, [x19, #(IFRAME_x + 6 * 8)] + stp x8, x9, [x19, #(IFRAME_x + 8 * 8)] + stp x10, x11, [x19, #(IFRAME_x + 10 * 8)] + stp x12, x13, [x19, #(IFRAME_x + 12 * 8)] + stp x14, x15, [x19, #(IFRAME_x + 14 * 8)] + stp x16, x17, [x19, #(IFRAME_x + 16 * 8)] + mov x0, sp // original x19 that we swapped with sp + stp x18, x0, [x19, #(IFRAME_x + 18 * 8)] + + // x20-x29 won't be clobbered + // thus we don't really need to store these + + str x30, [x19, #(IFRAME_lr)] + +.if \el == 0 + mrs x0, SP_EL0 +.else + // add sizeof back here to store original sp + add x0, x19, #(IFRAME_sizeof) +.endif + + mrs x1, ELR_EL1 + mrs x2, SPSR_EL1 + mrs x3, ESR_EL1 + mrs x4, FAR_EL1 + + str x0, [x19, #(IFRAME_sp)] + str x1, [x19, #(IFRAME_elr)] + str x2, [x19, #(IFRAME_spsr)] + str x3, [x19, #(IFRAME_esr)] + str x4, [x19, #(IFRAME_far)] +.endm + +.macro EXCEPTION_RETURN el + // x19 is callee-saved so it still points to IFRAME + // x0, x1, x18, x19 will be restored at the very end + + ldr x0, [x19, #(IFRAME_elr)] + ldr x1, [x19, #(IFRAME_spsr)] + ldr x18, [x19, #(IFRAME_sp)] + + // x0 and x1 will be restored later + ldp x2, x3, [x19, #(IFRAME_x + 2 * 8)] + ldp x4, x5, [x19, #(IFRAME_x + 4 * 8)] + ldp x6, x7, [x19, #(IFRAME_x + 6 * 8)] + ldp x8, x9, [x19, #(IFRAME_x + 8 * 8)] + ldp x10, x11, [x19, #(IFRAME_x + 10 * 8)] + ldp x12, x13, [x19, #(IFRAME_x + 12 * 8)] + ldp x14, x15, [x19, #(IFRAME_x + 14 * 8)] + ldp x16, x17, [x19, #(IFRAME_x + 16 * 8)] + // x18 and x19 will be restored later + ldr x30, [x19, #(IFRAME_lr)] + + // disable interrupts before restoring ELR/SPSR/sp + msr DAIFSet, #0xf + + msr ELR_EL1, x0 + msr SPSR_EL1, x1 + +.if \el == 0 + // load stack pointer for EL0 from IFRAME + msr SP_EL0, x18 + + // unwind our own stack pointer + add sp, x19, #(IFRAME_sizeof) +.else + // we stored original pointer to IFRAME, no need to unwind again there + mov sp, x18 +.endif + + // finally restore remaining registers + ldp x0, x1, [x19, #(IFRAME_x + 0 * 8)] + ldp x18, x19, [x19, #(IFRAME_x + 18 * 8)] + + eret +.endm + +.macro EXCEPTION_HANDLER el name func + FUNCTION(handle_\name): + EXCEPTION_ENTRY \el + + // prepare aligned sp for C function + and sp, x19, #0xfffffffffffffff0 + + // call C handler, passing IFRAME in x0 + // handler can enable interrupts if it wants to + mov x0, x19 + bl \func + + EXCEPTION_RETURN \el + FUNCTION_END(handle_\name) +.endm + +.macro vector name + .align 7 + b handle_\name +.endm + +.macro vempty + .align 7 + brk 0xfff + 1: b 1b +.endm + +.align 11 +.globl _exception_vectors +_exception_vectors: + vempty /* Synchronous EL1t */ + vempty /* IRQ EL1t */ + vempty /* FIQ EL1t */ + vempty /* Error EL1t */ + + vector el1h_sync /* Synchronous EL1h */ + vector el1h_irq /* IRQ EL1h */ + vector el1h_fiq /* FIQ EL1h */ + vector el1h_error /* Error EL1h */ + + vector el0_sync /* Synchronous 64-bit EL0 */ + vector el0_irq /* IRQ 64-bit EL0 */ + vector el0_fiq /* FIQ 64-bit EL0 */ + vector el0_error /* Error 64-bit EL0 */ + + vempty /* Synchronous 32-bit EL0 */ + vempty /* IRQ 32-bit EL0 */ + vempty /* FIQ 32-bit EL0 */ + vempty /* Error 32-bit EL0 */ + +EXCEPTION_HANDLER 1 el1h_sync do_sync_handler +EXCEPTION_HANDLER 1 el1h_irq do_irq_handler +EXCEPTION_HANDLER 1 el1h_fiq do_fiq_handler +EXCEPTION_HANDLER 1 el1h_error do_error_handler + +EXCEPTION_HANDLER 0 el0_sync do_sync_handler +EXCEPTION_HANDLER 0 el0_irq do_irq_handler +EXCEPTION_HANDLER 0 el0_fiq do_fiq_handler +EXCEPTION_HANDLER 0 el0_error do_error_handler + +FUNCTION(_eret_with_iframe): + mov x20, xzr + mov x21, xzr + mov x22, xzr + mov x23, xzr + mov x24, xzr + mov x25, xzr + mov x26, xzr + mov x27, xzr + mov x28, xzr + mov x29, xzr + + mov x19, x0 + EXCEPTION_RETURN 0 +FUNCTION_END(_eret_with_iframe) + +FUNCTION(_fp_save): + stp q0, q1, [x0], #32 + stp q2, q3, [x0], #32 + stp q4, q5, [x0], #32 + stp q6, q7, [x0], #32 + stp q8, q9, [x0], #32 + stp q10, q11, [x0], #32 + stp q12, q13, [x0], #32 + stp q14, q15, [x0], #32 + stp q16, q17, [x0], #32 + stp q18, q19, [x0], #32 + stp q20, q21, [x0], #32 + stp q22, q23, [x0], #32 + stp q24, q25, [x0], #32 + stp q26, q27, [x0], #32 + stp q28, q29, [x0], #32 + stp q30, q31, [x0], #32 + mrs x1, FPSR + mrs x2, FPCR + str x1, [x0], #8 + str x2, [x0], #8 + ret +FUNCTION_END(_fp_save) + +FUNCTION(_fp_restore): + ldp q0, q1, [x0], #32 + ldp q2, q3, [x0], #32 + ldp q4, q5, [x0], #32 + ldp q6, q7, [x0], #32 + ldp q8, q9, [x0], #32 + ldp q10, q11, [x0], #32 + ldp q12, q13, [x0], #32 + ldp q14, q15, [x0], #32 + ldp q16, q17, [x0], #32 + ldp q18, q19, [x0], #32 + ldp q20, q21, [x0], #32 + ldp q22, q23, [x0], #32 + ldp q24, q25, [x0], #32 + ldp q26, q27, [x0], #32 + ldp q28, q29, [x0], #32 + ldp q30, q31, [x0], #32 + + ldr x1, [x0], #8 + msr FPSR, x1 + + // avoid restoring FPCR if it hasn't changed + ldr x2, [x0], #8 + mrs x3, FPCR + cmp x3, x2 + beq 1f + msr FPCR, x2 +1: + ret +FUNCTION_END(_fp_restore) + FUNCTION(_arch_context_swap): stp x19, x20, [x0], #16 stp x21, x22, [x0], #16 @@ -32,24 +260,6 @@ FUNCTION(_arch_context_swap): ret FUNCTION_END(_arch_context_swap) -/* status_t arch_cpu_user_memcpy(void *to, const void *from, size_t size, addr_t *faultHandler) */ -FUNCTION(_arch_cpu_user_memcpy): - mov x0, xzr - ret -FUNCTION_END(_arch_cpu_user_memcpy) - -/* status_t arch_cpu_user_memset(void *to, char c, size_t count, addr_t *faultHandler) */ -FUNCTION(_arch_cpu_user_memset): - mov x0, xzr - ret -FUNCTION_END(_arch_cpu_user_memset) - -/* ssize_t arch_cpu_user_strlcpy(void *to, const void *from, size_t size, addr_t *faultHandler) */ -FUNCTION(_arch_cpu_user_strlcpy): - mov x0, xzr - ret -FUNCTION_END(_arch_cpu_user_strlcpy) - /*! \fn void arch_debug_call_with_fault_handler(cpu_ent* cpu, jmp_buf jumpBuffer, void (*function)(void*), void* parameter) @@ -68,6 +278,15 @@ FUNCTION_END(_arch_cpu_user_strlcpy) \param parameter The parameter to be passed to the function to be called. */ FUNCTION(arch_debug_call_with_fault_handler): - mov x0, xzr - ret + ldr x4, =fault + str x4, [x0, #CPU_ENT_fault_handler] + str x1, [x0, #CPU_ENT_fault_handler_stack_pointer] + + mov x0, x3 + br x2 + +fault: + mov x0, sp + mov x1, #1 + b longjmp FUNCTION_END(arch_debug_call_with_fault_handler) diff --git a/src/system/kernel/arch/arm64/arch_cpu.cpp b/src/system/kernel/arch/arm64/arch_cpu.cpp index c09c8dd7e1..3c78bb3ed1 100644 --- a/src/system/kernel/arch/arm64/arch_cpu.cpp +++ b/src/system/kernel/arch/arm64/arch_cpu.cpp @@ -12,9 +12,13 @@ #include +extern "C" void _exception_vectors(void); + + status_t arch_cpu_preboot_init_percpu(kernel_args *args, int curr_cpu) { + WRITE_SPECIALREG(VBAR_EL1, _exception_vectors); return B_OK; } diff --git a/src/system/kernel/arch/arm64/arch_int.cpp b/src/system/kernel/arch/arm64/arch_int.cpp index 34c452adeb..f7f90c8f5c 100644 --- a/src/system/kernel/arch/arm64/arch_int.cpp +++ b/src/system/kernel/arch/arm64/arch_int.cpp @@ -8,15 +8,19 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include #include #include +#include "syscall_numbers.h" +#include "VMSAv8TranslationMap.h" #include #define TRACE_ARCH_INT @@ -73,3 +77,240 @@ arch_int_init_post_device_manager(struct kernel_args *args) { return B_ENTRY_NOT_FOUND; } + + +// TODO: reuse things from VMSAv8TranslationMap + + +static int page_bits = 12; + +static constexpr uint64_t kPteAddrMask = (((1UL << 36) - 1) << 12); +static constexpr uint64_t kPteAttrMask = ~(kPteAddrMask | 0x3); +static constexpr uint64_t kAttrSWDBM = (1UL << 55); +static constexpr uint64_t kAttrAF = (1UL << 10); +static constexpr uint64_t kAttrAP2 = (1UL << 7); + + +static uint64_t* +TableFromPa(phys_addr_t pa) +{ + return reinterpret_cast(KERNEL_PMAP_BASE + pa); +} + + +static bool +fixup_entry(phys_addr_t ptPa, int level, addr_t va, bool wr) +{ + int tableBits = page_bits - 3; + uint64_t tableMask = (1UL << tableBits) - 1; + + int shift = tableBits * (3 - level) + page_bits; + uint64_t entrySize = 1UL << shift; + uint64_t entryMask = entrySize - 1; + + int index = (va >> shift) & tableMask; + + uint64_t *pte = &TableFromPa(ptPa)[index]; + + int type = *pte & 0x3; + uint64_t addr = *pte & kPteAddrMask; + + if ((level == 3 && type == 0x3) || (level < 3 && type == 0x1)) { + if (!wr && (*pte & kAttrAF) == 0) { + atomic_or64((int64*)pte, kAttrAF); + return true; + } + if (wr && (*pte & kAttrSWDBM) != 0 && (*pte & kAttrAP2) != 0) { + atomic_and64((int64*)pte, ~kAttrAP2); + asm("tlbi vaae1is, %0 \n dsb ish"::"r"(va >> page_bits)); + return true; + } + } else if (level < 3 && type == 0x3) { + return fixup_entry(addr, level + 1, va, wr); + } + + return false; +} + + +void +after_exception() +{ + Thread* thread = thread_get_current_thread(); + if (thread->cpu->invoke_scheduler) { + disable_interrupts(); + SpinLocker schedulerLocker(thread->scheduler_lock); + scheduler_reschedule(B_THREAD_READY); + } +} + + +extern "C" void +do_sync_handler(iframe * frame) +{ + bool isExec = false; + switch (ESR_ELx_EXCEPTION(frame->esr)) { + case EXCP_INSN_ABORT_L: + case EXCP_INSN_ABORT: + isExec = true; + case EXCP_DATA_ABORT_L: + case EXCP_DATA_ABORT: + { + bool write = (frame->esr & ISS_DATA_WnR) != 0; + bool known = false; + + int initialLevel = VMSAv8TranslationMap::CalcStartLevel(48, 12); + phys_addr_t ptPa; + bool addrType = (frame->far & (1UL << 63)) != 0; + if (addrType) + ptPa = READ_SPECIALREG(TTBR1_EL1); + else + ptPa = READ_SPECIALREG(TTBR0_EL1); + + switch (frame->esr & ISS_DATA_DFSC_MASK) { + case ISS_DATA_DFSC_TF_L0: + case ISS_DATA_DFSC_TF_L1: + case ISS_DATA_DFSC_TF_L2: + case ISS_DATA_DFSC_TF_L3: + known = true; + break; + + case ISS_DATA_DFSC_AFF_L1: + case ISS_DATA_DFSC_AFF_L2: + case ISS_DATA_DFSC_AFF_L3: + known = true; + if (fixup_entry(ptPa, initialLevel, frame->far, false)) + return; + break; + + case ISS_DATA_DFSC_PF_L1: + case ISS_DATA_DFSC_PF_L2: + case ISS_DATA_DFSC_PF_L3: + known = true; + if (write && fixup_entry(ptPa, initialLevel, frame->far, true)) + return; + break; + } + + if (!known) + break; + + if (debug_debugger_running()) { + Thread* thread = thread_get_current_thread(); + if (thread != NULL) { + cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; + if (cpu->fault_handler != 0) { + debug_set_page_fault_info(frame->far, frame->elr, + write ? DEBUG_PAGE_FAULT_WRITE : 0); + frame->elr = cpu->fault_handler; + frame->sp = cpu->fault_handler_stack_pointer; + return; + } + } + } + + Thread *thread = thread_get_current_thread(); + ASSERT(thread); + + bool isUser = (frame->spsr & PSR_M_MASK) == PSR_M_EL0t; + + if ((frame->spsr & PSR_I) != 0) { + // interrupts disabled + uintptr_t handler = reinterpret_cast(thread->fault_handler); + if (thread->fault_handler != 0) { + frame->elr = handler; + return; + } + } else if (thread->page_faults_allowed != 0) { + dprintf("PF: %lx\n", frame->far); + enable_interrupts(); + addr_t ret = 0; + vm_page_fault(frame->far, frame->elr, write, isExec, isUser, &ret); + if (ret != 0) + frame->elr = ret; + return; + } + + panic("unhandled pagefault! FAR=%lx ELR=%lx ESR=%lx", + frame->far, frame->elr, frame->esr); + break; + } + + case EXCP_SVC64: + { + uint32 imm = (frame->esr & 0xffff); + + uint32 count = imm & 0x1f; + uint32 syscall = imm >> 5; + + uint64_t args[20]; + if (count > 20) { + frame->x[0] = B_ERROR; + return; + } + + memset(args, 0, sizeof(args)); + memcpy(args, frame->x, (count < 8 ? count : 8) * 8); + + if (count > 8) { + if (!IS_USER_ADDRESS(frame->sp) + || user_memcpy(&args[8], (void*)frame->sp, (count - 8) * 8) != B_OK) { + frame->x[0] = B_BAD_ADDRESS; + return; + } + } + + thread_at_kernel_entry(system_time()); + + enable_interrupts(); + syscall_dispatcher(syscall, (void*)args, &frame->x[0]); + + { + disable_interrupts(); + atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED); + if ((thread_get_current_thread()->flags + & (THREAD_FLAGS_SIGNALS_PENDING + | THREAD_FLAGS_DEBUG_THREAD + | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) { + enable_interrupts(); + thread_at_kernel_exit(); + } else { + thread_at_kernel_exit_no_signals(); + } + if ((THREAD_FLAGS_RESTART_SYSCALL & thread_get_current_thread()->flags) != 0) { + panic("syscall restart"); + } + } + + return; + } + } + + panic("unhandled exception! FAR=%lx ELR=%lx ESR=%lx (EC=%lx)", + frame->far, frame->elr, frame->esr, (frame->esr >> 26) & 0x3f); +} + + +extern "C" void +do_error_handler(iframe * frame) +{ + panic("unhandled error! FAR=%lx ELR=%lx ESR=%lx", frame->far, frame->elr, frame->esr); +} + + +extern "C" void +do_irq_handler(iframe * frame) +{ + InterruptController *ic = InterruptController::Get(); + if (ic != NULL) + ic->HandleInterrupt(); + + after_exception(); +} + + +extern "C" void +do_fiq_handler(iframe * frame) +{ + panic("do_fiq_handler"); +} diff --git a/src/system/kernel/arch/arm64/asm_offsets.cpp b/src/system/kernel/arch/arm64/asm_offsets.cpp new file mode 100644 index 0000000000..2dfe84f84e --- /dev/null +++ b/src/system/kernel/arch/arm64/asm_offsets.cpp @@ -0,0 +1,44 @@ +/* + * Copyright 2007-2011, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ + +// This file is used to get C structure offsets into assembly code. +// The build system assembles the file and processes the output to create +// a header file with macro definitions, that can be included from assembly +// code. + + +#include + +#include +#include +#include +#include +#include + + +#define DEFINE_MACRO(macro, value) DEFINE_COMPUTED_ASM_MACRO(macro, value) + +#define DEFINE_OFFSET_MACRO(prefix, structure, member) \ + DEFINE_MACRO(prefix##_##member, offsetof(struct structure, member)); + +#define DEFINE_SIZEOF_MACRO(prefix, structure) \ + DEFINE_MACRO(prefix##_sizeof, sizeof(struct structure)); + + +void +dummy() +{ + DEFINE_SIZEOF_MACRO(IFRAME, iframe); + DEFINE_OFFSET_MACRO(IFRAME, iframe, elr); + DEFINE_OFFSET_MACRO(IFRAME, iframe, spsr); + DEFINE_OFFSET_MACRO(IFRAME, iframe, x); + DEFINE_OFFSET_MACRO(IFRAME, iframe, lr); + DEFINE_OFFSET_MACRO(IFRAME, iframe, sp); + DEFINE_OFFSET_MACRO(IFRAME, iframe, esr); + DEFINE_OFFSET_MACRO(IFRAME, iframe, far); + + DEFINE_OFFSET_MACRO(CPU_ENT, cpu_ent, fault_handler); + DEFINE_OFFSET_MACRO(CPU_ENT, cpu_ent, fault_handler_stack_pointer); +}