From 9495126984d664dfa8afc1382b5614ee69ba2a1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Duval?= Date: Tue, 5 May 2020 23:03:39 +0200 Subject: [PATCH] kernel/x86_64: AVX support xsave or xsavec are supported. breaks vregs compatibility. change the thread structure object cache alignment to 64 the xsave fpu_state size isn't defined, it is for instance 832 here, thus I picked 1024. Change-Id: I4a0cab0bc42c1d37f24dcafb8259f8ff24a330d2 Reviewed-on: https://review.haiku-os.org/c/haiku/+/2849 Reviewed-by: Adrien Destugues --- headers/posix/arch/x86_64/signal.h | 14 +++- .../kernel/arch/x86/arch_altcodepatch.h | 2 + headers/private/kernel/arch/x86/arch_cpu.h | 16 +++++ .../kernel/arch/x86/arch_thread_types.h | 5 ++ src/system/kernel/arch/x86/64/arch.S | 13 ++++ src/system/kernel/arch/x86/64/interrupts.S | 66 +++++++++++++++---- src/system/kernel/arch/x86/64/thread.cpp | 49 ++++++++++---- .../kernel/arch/x86/arch_altcodepatch.cpp | 3 +- src/system/kernel/arch/x86/arch_cpu.cpp | 46 +++++++++++++ src/system/kernel/thread.cpp | 4 +- 10 files changed, 191 insertions(+), 27 deletions(-) diff --git a/headers/posix/arch/x86_64/signal.h b/headers/posix/arch/x86_64/signal.h index 871d1f59b6..ad4fa4b128 100644 --- a/headers/posix/arch/x86_64/signal.h +++ b/headers/posix/arch/x86_64/signal.h @@ -89,6 +89,18 @@ struct fpu_state { unsigned char _reserved_416_511[96]; }; +struct xstate_hdr { + unsigned long bv; + unsigned long xcomp_bv; + unsigned char _reserved[48]; +}; + +struct savefpu { + struct fpu_state fp_fxsave; + struct xstate_hdr fp_xstate; + unsigned long fp_ymm[16][2]; +}; + struct vregs { unsigned long rax; unsigned long rbx; @@ -110,7 +122,7 @@ struct vregs { unsigned long rip; unsigned long rflags; - struct fpu_state fpu; + struct savefpu fpu; }; diff --git a/headers/private/kernel/arch/x86/arch_altcodepatch.h b/headers/private/kernel/arch/x86/arch_altcodepatch.h index 99f01e2784..762a083de8 100644 --- a/headers/private/kernel/arch/x86/arch_altcodepatch.h +++ b/headers/private/kernel/arch/x86/arch_altcodepatch.h @@ -22,6 +22,8 @@ #define ALTCODEPATCH_TAG_STAC 1 #define ALTCODEPATCH_TAG_CLAC 2 +#define ALTCODEPATCH_TAG_XSAVE 3 +#define ALTCODEPATCH_TAG_XRSTOR 4 #ifdef _ASSEMBLER diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index fa5215eead..155c88d9d2 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -354,9 +354,15 @@ #define IA32_CR4_GLOBAL_PAGES (1UL << 7) #define CR4_OS_FXSR (1UL << 9) #define CR4_OS_XMM_EXCEPTION (1UL << 10) +#define IA32_CR4_OSXSAVE (1UL << 18) #define IA32_CR4_SMEP (1UL << 20) #define IA32_CR4_SMAP (1UL << 21) +// Extended Control Register XCR0 flags +#define IA32_XCR0_X87 (1UL << 0) +#define IA32_XCR0_SSE (1UL << 1) +#define IA32_XCR0_AVX (1UL << 2) + // page fault error codes (http://wiki.osdev.org/Page_Fault) #define PGFAULT_P 0x01 // Protection violation #define PGFAULT_W 0x02 // Write @@ -547,6 +553,16 @@ struct intel_microcode_extended_signature { #define clear_ac() \ __asm__ volatile (ASM_CLAC : : : "memory") +#define xgetbv(reg) ({ \ + uint32 low, high; \ + __asm__ volatile ("xgetbv" : "=a" (low), "=d" (high), "c" (reg)); \ + (low | (uint64)high << 32); \ +}) + +#define xsetbv(reg, value) { \ + uint32 low = value; uint32 high = value >> 32; \ + __asm__ volatile ("xsetbv" : : "a" (low), "d" (high), "c" (reg)); } + #define out8(value,port) \ __asm__ ("outb %%al,%%dx" : : "a" (value), "d" (port)) diff --git a/headers/private/kernel/arch/x86/arch_thread_types.h b/headers/private/kernel/arch/x86/arch_thread_types.h index f02ae41ad6..5bd1b637d9 100644 --- a/headers/private/kernel/arch/x86/arch_thread_types.h +++ b/headers/private/kernel/arch/x86/arch_thread_types.h @@ -53,8 +53,13 @@ struct arch_thread { struct farcall interrupt_stack; #endif +#ifndef __x86_64__ // 512 byte floating point save point - this must be 16 byte aligned uint8 fpu_state[512] _ALIGNED(16); +#else + // floating point save point - this must be 64 byte aligned for xsave + uint8 fpu_state[1024] _ALIGNED(64); +#endif addr_t GetFramePointer() const; } _ALIGNED(16); diff --git a/src/system/kernel/arch/x86/64/arch.S b/src/system/kernel/arch/x86/64/arch.S index d88b05bfed..4c4aaa66cc 100644 --- a/src/system/kernel/arch/x86/64/arch.S +++ b/src/system/kernel/arch/x86/64/arch.S @@ -119,3 +119,16 @@ FUNCTION_END(_stac) FUNCTION(_clac): clac FUNCTION_END(_clac) + +FUNCTION(_xsave): + xsave64 (%rdi) +FUNCTION_END(_xsave) + +FUNCTION(_xsavec): + xsavec64 (%rdi) +FUNCTION_END(_xsavec) + +FUNCTION(_xrstor): + xrstor64 (%rdi) +FUNCTION_END(_xrstor) + diff --git a/src/system/kernel/arch/x86/64/interrupts.S b/src/system/kernel/arch/x86/64/interrupts.S index f58135cace..124fe061a3 100644 --- a/src/system/kernel/arch/x86/64/interrupts.S +++ b/src/system/kernel/arch/x86/64/interrupts.S @@ -221,16 +221,32 @@ STATIC_FUNCTION(int_bottom): // exception. orq $X86_EFLAGS_RESUME, IFRAME_flags(%rbp) - subq $512, %rsp - andq $~15, %rsp - fxsaveq (%rsp) + // xsave needs a 64-byte alignment + andq $~63, %rsp + movq (gFPUSaveLength), %rcx + subq %rcx, %rsp + leaq (%rsp), %rdi + shrq $3, %rcx + movq $0, %rax + rep stosq + movl (gXsaveMask), %eax + movl (gXsaveMask+4), %edx + movq %rsp, %rdi + CODEPATCH_START + fxsaveq (%rdi) + CODEPATCH_END(ALTCODEPATCH_TAG_XSAVE) // Call the interrupt handler. movq %rbp, %rdi movq IFRAME_vector(%rbp), %rax call *gInterruptHandlerTable(, %rax, 8) - fxrstorq (%rsp) + movl (gXsaveMask), %eax + movl (gXsaveMask+4), %edx + movq %rsp, %rdi + CODEPATCH_START + fxrstorq (%rdi) + CODEPATCH_END(ALTCODEPATCH_TAG_XRSTOR) movq %rbp, %rsp // Restore the saved registers. @@ -253,9 +269,22 @@ STATIC_FUNCTION(int_bottom_user): // Frame pointer is the iframe. movq %rsp, %rbp - subq $512, %rsp - andq $~15, %rsp - fxsaveq (%rsp) + // xsave needs a 64-byte alignment + andq $~63, %rsp + movq (gFPUSaveLength), %rcx + subq %rcx, %rsp + leaq (%rsp), %rdi + shrq $3, %rcx + movq $0, %rax + rep stosq + movl (gXsaveMask), %eax + movl (gXsaveMask+4), %edx + + movq %rsp, %rdi + CODEPATCH_START + fxsaveq (%rdi) + CODEPATCH_END(ALTCODEPATCH_TAG_XSAVE) + movq %rsp, IFRAME_fpu(%rbp) // Set the RF (resume flag) in RFLAGS. This prevents an instruction @@ -286,7 +315,12 @@ STATIC_FUNCTION(int_bottom_user): UPDATE_THREAD_KERNEL_TIME() - fxrstorq (%rsp) + movl (gXsaveMask), %eax + movl (gXsaveMask+4), %edx + movq %rsp, %rdi + CODEPATCH_START + fxrstorq (%rdi) + CODEPATCH_END(ALTCODEPATCH_TAG_XRSTOR) movq %rbp, %rsp // Restore the saved registers. @@ -315,7 +349,12 @@ STATIC_FUNCTION(int_bottom_user): movq %rbp, %rdi call x86_init_user_debug_at_kernel_exit 1: - fxrstorq (%rsp) + movl (gXsaveMask), %eax + movl (gXsaveMask+4), %edx + movq %rsp, %rdi + CODEPATCH_START + fxrstorq (%rdi) + CODEPATCH_END(ALTCODEPATCH_TAG_XRSTOR) movq %rbp, %rsp // Restore the saved registers. @@ -522,8 +561,13 @@ FUNCTION(x86_64_syscall_entry): jmp .Liret .Lrestore_fpu: - movq IFRAME_fpu(%rbp), %rax - fxrstorq (%rax) + movq IFRAME_fpu(%rbp), %rdi + + movl (gXsaveMask), %eax + movl (gXsaveMask+4), %edx + CODEPATCH_START + fxrstorq (%rdi) + CODEPATCH_END(ALTCODEPATCH_TAG_XRSTOR) .Liret: // Restore the saved registers. RESTORE_IFRAME() diff --git a/src/system/kernel/arch/x86/64/thread.cpp b/src/system/kernel/arch/x86/64/thread.cpp index a29697a03f..e50c3a8fa2 100644 --- a/src/system/kernel/arch/x86/64/thread.cpp +++ b/src/system/kernel/arch/x86/64/thread.cpp @@ -68,7 +68,10 @@ class RestartSyscall : public AbstractTraceEntry { extern "C" void x86_64_thread_entry(); // Initial thread saved state. -static arch_thread sInitialState; +static arch_thread sInitialState _ALIGNED(64); +extern uint64 gFPUSaveLength; +extern bool gHasXsave; +extern bool gHasXsavec; void @@ -140,12 +143,36 @@ arch_thread_init(kernel_args* args) { // Save one global valid FPU state; it will be copied in the arch dependent // part of each new thread. - asm volatile ( - "clts;" \ - "fninit;" \ - "fnclex;" \ - "fxsave %0;" - : "=m" (sInitialState.fpu_state)); + if (gHasXsave || gHasXsavec) { + ASSERT(gFPUSaveLength <= sizeof(sInitialState.fpu_state)); + memset(sInitialState.fpu_state, 0, gFPUSaveLength); + if (gHasXsavec) { + asm volatile ( + "clts;" \ + "fninit;" \ + "fnclex;" \ + "movl $0x7,%%eax;" \ + "movl $0x0,%%edx;" \ + "xsavec64 %0" + :: "m" (sInitialState.fpu_state)); + } else { + asm volatile ( + "clts;" \ + "fninit;" \ + "fnclex;" \ + "movl $0x7,%%eax;" \ + "movl $0x0,%%edx;" \ + "xsave64 %0" + :: "m" (sInitialState.fpu_state)); + } + } else { + asm volatile ( + "clts;" \ + "fninit;" \ + "fnclex;" \ + "fxsaveq %0" + :: "m" (sInitialState.fpu_state)); + } return B_OK; } @@ -309,11 +336,10 @@ arch_setup_signal_frame(Thread* thread, struct sigaction* action, if (frame->fpu != nullptr) { memcpy((void*)&signalFrameData->context.uc_mcontext.fpu, frame->fpu, - sizeof(signalFrameData->context.uc_mcontext.fpu)); + gFPUSaveLength); } else { memcpy((void*)&signalFrameData->context.uc_mcontext.fpu, - sInitialState.fpu_state, - sizeof(signalFrameData->context.uc_mcontext.fpu)); + sInitialState.fpu_state, gFPUSaveLength); } // Fill in signalFrameData->context.uc_stack. @@ -385,8 +411,7 @@ arch_restore_signal_frame(struct signal_frame_data* signalFrameData) Thread* thread = thread_get_current_thread(); memcpy(thread->arch_info.fpu_state, - (void*)&signalFrameData->context.uc_mcontext.fpu, - sizeof(thread->arch_info.fpu_state)); + (void*)&signalFrameData->context.uc_mcontext.fpu, gFPUSaveLength); frame->fpu = &thread->arch_info.fpu_state; // The syscall return code overwrites frame->ax with the return value of diff --git a/src/system/kernel/arch/x86/arch_altcodepatch.cpp b/src/system/kernel/arch/x86/arch_altcodepatch.cpp index b99a7d12c7..958448a12b 100644 --- a/src/system/kernel/arch/x86/arch_altcodepatch.cpp +++ b/src/system/kernel/arch/x86/arch_altcodepatch.cpp @@ -52,6 +52,7 @@ arch_altcodepatch_replace(uint16 tag, void* newcodepatch, size_t length) // disable write after patch set_area_protection(info->text_region.id, kernelProtection); - dprintf("arch_altcodepatch_replace found %" B_PRIu32 " altcodepatches\n", count); + dprintf("arch_altcodepatch_replace found %" B_PRIu32 " altcodepatches " + "for tag %u\n", count, tag); } diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index d6fbd74c10..e62c85e28f 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -84,6 +84,13 @@ struct set_mtrrs_parameter { #ifdef __x86_64__ extern addr_t _stac; extern addr_t _clac; +extern addr_t _xsave; +extern addr_t _xsavec; +extern addr_t _xrstor; +uint64 gXsaveMask; +uint64 gFPUSaveLength = 512; +bool gHasXsave = false; +bool gHasXsavec = false; #endif extern "C" void x86_reboot(void); @@ -1406,6 +1413,20 @@ enable_smep(void* dummy, int cpu) { x86_write_cr4(x86_read_cr4() | IA32_CR4_SMEP); } + + +static void +enable_osxsave(void* dummy, int cpu) +{ + x86_write_cr4(x86_read_cr4() | IA32_CR4_OSXSAVE); +} + + +static void +enable_xsavemask(void* dummy, int cpu) +{ + xsetbv(0, gXsaveMask); +} #endif @@ -1459,6 +1480,31 @@ arch_cpu_init_post_vm(kernel_args* args) } else dprintf("SMAP disabled per safemode setting\n"); } + + // if available enable XSAVE (XSAVE and extended states) + gHasXsave = x86_check_feature(IA32_FEATURE_EXT_XSAVE, FEATURE_EXT); + if (gHasXsave) { + gHasXsavec = x86_check_feature(IA32_FEATURE_XSAVEC, + FEATURE_D_1_EAX); + + call_all_cpus_sync(&enable_osxsave, NULL); + gXsaveMask = IA32_XCR0_X87 | IA32_XCR0_SSE; + cpuid_info cpuid; + get_current_cpuid(&cpuid, 0xd, 0); + gXsaveMask |= (cpuid.regs.eax & IA32_XCR0_AVX); + call_all_cpus_sync(&enable_xsavemask, NULL); + get_current_cpuid(&cpuid, 0xd, 0); + gFPUSaveLength = cpuid.regs.ebx; + + arch_altcodepatch_replace(ALTCODEPATCH_TAG_XSAVE, + gHasXsavec ? &_xsavec : &_xsave, 4); + arch_altcodepatch_replace(ALTCODEPATCH_TAG_XRSTOR, + &_xrstor, 4); + + dprintf("enable %s 0x%" B_PRIx64 " %" B_PRId64 "\n", + gHasXsavec ? "XSAVEC" : "XSAVE", gXsaveMask, gFPUSaveLength); + } + #endif return B_OK; diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 4ea3f605c7..2fae73aa2f 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -2679,9 +2679,9 @@ thread_init(kernel_args *args) panic("thread_init(): failed to init thread hash table!"); // create the thread structure object cache - sThreadCache = create_object_cache("threads", sizeof(Thread), 16, NULL, + sThreadCache = create_object_cache("threads", sizeof(Thread), 64, NULL, NULL, NULL); - // Note: The x86 port requires 16 byte alignment of thread structures. + // Note: The x86 port requires 64 byte alignment of thread structures. if (sThreadCache == NULL) panic("thread_init(): failed to allocate thread object cache!");