diff --git a/headers/private/kernel/arch/x86/64/cpu.h b/headers/private/kernel/arch/x86/64/cpu.h index a29891d5de..02860371b8 100644 --- a/headers/private/kernel/arch/x86/64/cpu.h +++ b/headers/private/kernel/arch/x86/64/cpu.h @@ -28,6 +28,10 @@ x86_write_msr(uint32_t msr, uint64_t value) static inline void x86_context_switch(arch_thread* oldState, arch_thread* newState) { + uint16_t fpuControl; + asm volatile("fnstcw %0" : "=m" (fpuControl)); + uint32_t sseControl; + asm volatile("stmxcsr %0" : "=m" (sseControl)); asm volatile( "pushq %%rbp;" "movq $1f, %c[rip](%0);" @@ -41,7 +45,11 @@ x86_context_switch(arch_thread* oldState, arch_thread* newState) [rsp] "i" (offsetof(arch_thread, current_stack)), [rip] "i" (offsetof(arch_thread, instruction_pointer)) : "rbx", "rcx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "r13", - "r14", "r15", "memory"); + "r14", "r15", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", + "xmm14", "xmm15", "memory"); + asm volatile("ldmxcsr %0" : : "m" (sseControl)); + asm volatile("fldcw %0" : : "m" (fpuControl)); } diff --git a/headers/private/kernel/arch/x86/64/iframe.h b/headers/private/kernel/arch/x86/64/iframe.h index 2320949c5b..d4b61829db 100644 --- a/headers/private/kernel/arch/x86/64/iframe.h +++ b/headers/private/kernel/arch/x86/64/iframe.h @@ -8,6 +8,7 @@ struct iframe { uint64 type; + void* fpu; uint64 r15; uint64 r14; uint64 r13; diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index 88c5eb8e6a..a202d08f4a 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -243,6 +243,14 @@ | X86_EFLAGS_AUXILIARY_CARRY | X86_EFLAGS_ZERO | X86_EFLAGS_SIGN \ | X86_EFLAGS_DIRECTION | X86_EFLAGS_OVERFLOW) +#define CR0_CACHE_DISABLE (1UL << 30) +#define CR0_NOT_WRITE_THROUGH (1UL << 29) +#define CR0_FPU_EMULATION (1UL << 2) +#define CR0_MONITOR_FPU (1UL << 1) + +#define CR4_OS_FXSR (1UL << 9) +#define CR4_OS_XMM_EXCEPTION (1UL << 10) + // iframe types #define IFRAME_TYPE_SYSCALL 0x1 @@ -262,13 +270,6 @@ typedef struct x86_mtrr_info { uint8 type; } x86_mtrr_info; -typedef struct x86_optimized_functions { - void (*memcpy)(void* dest, const void* source, size_t count); - void* memcpy_end; - void (*memset)(void* dest, int value, size_t count); - void* memset_end; -} x86_optimized_functions; - typedef struct x86_cpu_module_info { module_info info; uint32 (*count_mtrrs)(void); @@ -280,8 +281,6 @@ typedef struct x86_cpu_module_info { uint8* _type); void (*set_mtrrs)(uint8 defaultType, const x86_mtrr_info* infos, uint32 count); - - void (*get_optimized_functions)(x86_optimized_functions* functions); } x86_cpu_module_info; // features @@ -455,10 +454,7 @@ void __x86_setup_system_time(uint32 conversionFactor, void x86_userspace_thread_exit(void); void x86_end_userspace_thread_exit(void); -void x86_fxsave(void* fpuState); -void x86_fxrstor(const void* fpuState); -void x86_noop_swap(void* oldFpuState, const void* newFpuState); -void x86_fxsave_swap(void* oldFpuState, const void* newFpuState); + addr_t x86_get_stack_frame(); uint32 x86_count_mtrrs(void); void x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type); @@ -489,7 +485,13 @@ void x86_context_switch(struct arch_thread* oldState, void x86_fnsave(void* fpuState); void x86_frstor(const void* fpuState); + +void x86_fxsave(void* fpuState); +void x86_fxrstor(const void* fpuState); + +void x86_noop_swap(void* oldFpuState, const void* newFpuState); void x86_fnsave_swap(void* oldFpuState, const void* newFpuState); +void x86_fxsave_swap(void* oldFpuState, const void* newFpuState); #endif diff --git a/headers/private/system/arch/x86_64/arch_commpage_defs.h b/headers/private/system/arch/x86_64/arch_commpage_defs.h index 85fa54e104..1451011d18 100644 --- a/headers/private/system/arch/x86_64/arch_commpage_defs.h +++ b/headers/private/system/arch/x86_64/arch_commpage_defs.h @@ -9,11 +9,9 @@ # error Must not be included directly. Include instead! #endif -#define COMMPAGE_ENTRY_X86_MEMCPY (COMMPAGE_ENTRY_FIRST_ARCH_SPECIFIC + 0) -#define COMMPAGE_ENTRY_X86_MEMSET (COMMPAGE_ENTRY_FIRST_ARCH_SPECIFIC + 1) #define COMMPAGE_ENTRY_X86_SIGNAL_HANDLER \ - (COMMPAGE_ENTRY_FIRST_ARCH_SPECIFIC + 2) + (COMMPAGE_ENTRY_FIRST_ARCH_SPECIFIC + 0) #define COMMPAGE_ENTRY_X86_THREAD_EXIT \ - (COMMPAGE_ENTRY_FIRST_ARCH_SPECIFIC + 3) + (COMMPAGE_ENTRY_FIRST_ARCH_SPECIFIC + 1) #endif /* _SYSTEM_ARCH_x86_64_COMMPAGE_DEFS_H */ diff --git a/src/system/boot/platform/bios_ia32/long.cpp b/src/system/boot/platform/bios_ia32/long.cpp index 8531716198..e558bae6f8 100644 --- a/src/system/boot/platform/bios_ia32/long.cpp +++ b/src/system/boot/platform/bios_ia32/long.cpp @@ -278,6 +278,14 @@ convert_kernel_args() } +static void +enable_sse() +{ + x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); + x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); +} + + static void long_smp_start_kernel(void) { @@ -287,6 +295,7 @@ long_smp_start_kernel(void) asm("movl %%eax, %%cr0" : : "a" ((1 << 31) | (1 << 16) | (1 << 5) | 1)); asm("cld"); asm("fninit"); + enable_sse(); // Fix our kernel stack address. gKernelArgs.cpu_kstack[cpu].start @@ -308,6 +317,8 @@ long_start_kernel() if ((info.regs.edx & (1 << 29)) == 0) panic("64-bit kernel requires a 64-bit CPU"); + enable_sse(); + preloaded_elf64_image *image = static_cast( gKernelArgs.kernel_image.Pointer()); diff --git a/src/system/kernel/arch/x86/32/syscalls.cpp b/src/system/kernel/arch/x86/32/syscalls.cpp index c3ca4e1a24..7fcee05a3c 100644 --- a/src/system/kernel/arch/x86/32/syscalls.cpp +++ b/src/system/kernel/arch/x86/32/syscalls.cpp @@ -30,6 +30,10 @@ extern "C" void x86_sysenter(); void (*gX86SetSyscallStack)(addr_t stackTop) = NULL; +extern int memcpy_end; +extern int memset_end; + + static bool all_cpus_have_feature(enum x86_feature_type type, int feature) { @@ -109,8 +113,20 @@ x86_initialize_syscall(void) addr_t position = fill_commpage_entry(COMMPAGE_ENTRY_X86_SYSCALL, syscallCode, len); + // put the optimized functions into the commpage + size_t memcpyLen = (addr_t)&memcpy_end - (addr_t)memcpy; + addr_t memcpyPosition = fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMCPY, + (const void*)memcpy, memcpyLen); + size_t memsetLen = (addr_t)&memset_end - (addr_t)memset; + addr_t memsetPosition = fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMSET, + (const void*)memset, memsetLen); + // add syscall to the commpage image image_id image = get_commpage_image(); + elf_add_memory_image_symbol(image, "commpage_memcpy", memcpyPosition, + memcpyLen, B_SYMBOL_TYPE_TEXT); + elf_add_memory_image_symbol(image, "commpage_memset", memsetPosition, + memsetLen, B_SYMBOL_TYPE_TEXT); elf_add_memory_image_symbol(image, "commpage_syscall", position, len, B_SYMBOL_TYPE_TEXT); } diff --git a/src/system/kernel/arch/x86/64/arch.S b/src/system/kernel/arch/x86/64/arch.S index 4cdd22af84..19c74b4d2d 100644 --- a/src/system/kernel/arch/x86/64/arch.S +++ b/src/system/kernel/arch/x86/64/arch.S @@ -19,35 +19,6 @@ .text -/* void x86_fxsave(void* fpuState); */ -FUNCTION(x86_fxsave): - fxsave (%rdi) - ret -FUNCTION_END(x86_fxsave) - - -/* void x86_fxrstor(const void* fpuState); */ -FUNCTION(x86_fxrstor): - fxrstor (%rdi) - ret -FUNCTION_END(x86_fxrstor) - - -/* void x86_noop_swap(void *oldFpuState, const void *newFpuState); */ -FUNCTION(x86_noop_swap): - nop - ret -FUNCTION_END(x86_noop_swap) - - -/* void x86_fxsave_swap(void* oldFpuState, const void* newFpuState); */ -FUNCTION(x86_fxsave_swap): - fxsave (%rdi) - fxrstor (%rsi) - ret -FUNCTION_END(x86_fxsave_swap) - - /* addr_t x86_get_stack_frame(); */ FUNCTION(x86_get_stack_frame): mov %rbp, %rax diff --git a/src/system/kernel/arch/x86/64/interrupts.S b/src/system/kernel/arch/x86/64/interrupts.S index db73a4fba4..c94f0d8dc9 100644 --- a/src/system/kernel/arch/x86/64/interrupts.S +++ b/src/system/kernel/arch/x86/64/interrupts.S @@ -35,11 +35,13 @@ push %r13; \ push %r14; \ push %r15; \ + pushq $0; \ push $iframeType; + // Restore the interrupt frame. #define RESTORE_IFRAME() \ - add $8, %rsp; \ + add $16, %rsp; \ pop %r15; \ pop %r14; \ pop %r13; \ @@ -116,6 +118,23 @@ call x86_exit_user_debug_at_kernel_entry; \ 1: +#define CLEAR_FPU_STATE() \ + pxor %xmm0, %xmm0; \ + pxor %xmm1, %xmm1; \ + pxor %xmm2, %xmm2; \ + pxor %xmm3, %xmm3; \ + pxor %xmm4, %xmm4; \ + pxor %xmm5, %xmm5; \ + pxor %xmm6, %xmm6; \ + pxor %xmm7, %xmm7; \ + pxor %xmm8, %xmm8; \ + pxor %xmm9, %xmm9; \ + pxor %xmm10, %xmm10; \ + pxor %xmm11, %xmm11; \ + pxor %xmm12, %xmm12; \ + pxor %xmm13, %xmm13; \ + pxor %xmm14, %xmm14; \ + pxor %xmm15, %xmm15 // The following code defines the interrupt service routines for all 256 // interrupts. It creates a block of handlers, each 16 bytes, that the IDT @@ -198,11 +217,18 @@ STATIC_FUNCTION(int_bottom): // exception. orq $X86_EFLAGS_RESUME, IFRAME_flags(%rbp) + subq $512, %rsp + andq $~15, %rsp + fxsaveq (%rsp) + // Call the interrupt handler. - movq %rsp, %rdi - movq IFRAME_vector(%rsp), %rax + movq %rbp, %rdi + movq IFRAME_vector(%rbp), %rax call *gInterruptHandlerTable(, %rax, 8) + fxrstorq (%rsp) + movq %rbp, %rsp + // Restore the saved registers. RESTORE_IFRAME() @@ -217,12 +243,16 @@ STATIC_FUNCTION(int_bottom_user): // Push the rest of the interrupt frame to the stack. PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER) - cld // Frame pointer is the iframe. movq %rsp, %rbp + subq $512, %rsp + andq $~15, %rsp + fxsaveq (%rsp) + movq %rsp, IFRAME_fpu(%rbp) + // Set the RF (resume flag) in RFLAGS. This prevents an instruction // breakpoint on the instruction we're returning to to trigger a debug // exception. @@ -235,8 +265,8 @@ STATIC_FUNCTION(int_bottom_user): UPDATE_THREAD_USER_TIME() // Call the interrupt handler. - movq %rsp, %rdi - movq IFRAME_vector(%rsp), %rax + movq %rbp, %rdi + movq IFRAME_vector(%rbp), %rax call *gInterruptHandlerTable(, %rax, 8) // If there are no signals pending or we're not debugging, we can avoid @@ -250,6 +280,9 @@ STATIC_FUNCTION(int_bottom_user): UPDATE_THREAD_KERNEL_TIME() + fxrstorq (%rsp) + movq %rbp, %rsp + // Restore the saved registers. RESTORE_IFRAME() @@ -274,6 +307,9 @@ STATIC_FUNCTION(int_bottom_user): movq %rbp, %rdi call x86_init_user_debug_at_kernel_exit 1: + fxrstorq (%rsp) + movq %rbp, %rsp + // Restore the saved registers. RESTORE_IFRAME() @@ -315,6 +351,7 @@ FUNCTION(x86_64_syscall_entry): // Frame pointer is the iframe. movq %rsp, %rbp + andq $~15, %rsp // Preserve call number (R14 is callee-save), get thread pointer. movq %rax, %r14 @@ -367,10 +404,10 @@ FUNCTION(x86_64_syscall_entry): // TODO: post-syscall tracing +.Lsyscall_return: // Restore the original stack pointer and return. movq %rbp, %rsp -.Lsyscall_return: // Clear the restarted flag. testl $THREAD_FLAGS_SYSCALL_RESTARTED, THREAD_flags(%r12) jz 2f @@ -394,7 +431,9 @@ FUNCTION(x86_64_syscall_entry): // If we've just restored a signal frame, use the IRET path. cmpq $SYSCALL_RESTORE_SIGNAL_FRAME, %r14 - je .Liret + je .Lrestore_fpu + + CLEAR_FPU_STATE() // Restore the iframe and RCX/R11 for SYSRET. RESTORE_IFRAME() @@ -458,14 +497,19 @@ FUNCTION(x86_64_syscall_entry): 1: // Install breakpoints, if defined. testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12) - jz .Liret + jz 1f movq %rbp, %rdi call x86_init_user_debug_at_kernel_exit - +1: // On this return path it is possible that the frame has been modified, // for example to execute a signal handler. In this case it is safer to // return via IRET. + CLEAR_FPU_STATE() + jmp .Liret +.Lrestore_fpu: + movq IFRAME_fpu(%rbp), %rax + fxrstorq (%rax) .Liret: // Restore the saved registers. RESTORE_IFRAME() @@ -493,6 +537,7 @@ FUNCTION(x86_64_syscall_entry): // Make space on the stack. subq %rcx, %rsp + andq $~15, %rsp movq %rsp, %rdi // Set a fault handler. @@ -535,7 +580,7 @@ FUNCTION(x86_return_to_userland): testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \ | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED) \ , THREAD_flags(%r12) - jnz .Lkernel_exit_work + jnz .Luserland_return_work // update the thread's kernel time and return UPDATE_THREAD_KERNEL_TIME() @@ -544,4 +589,33 @@ FUNCTION(x86_return_to_userland): RESTORE_IFRAME() swapgs iretq +.Luserland_return_work: + // Slow path for return to userland. + + // Do we need to handle signals? + testl $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD) \ + , THREAD_flags(%r12) + jnz .Luserland_return_handle_signals + cli + call thread_at_kernel_exit_no_signals + +.Luserland_return_work_done: + // Install breakpoints, if defined. + testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12) + jz 1f + movq %rbp, %rdi + call x86_init_user_debug_at_kernel_exit +1: + // Restore the saved registers. + RESTORE_IFRAME() + + // Restore the previous GS base and return. + swapgs + iretq +.Luserland_return_handle_signals: + // thread_at_kernel_exit requires interrupts to be enabled, it will disable + // them after. + sti + call thread_at_kernel_exit + jmp .Luserland_return_work_done FUNCTION_END(x86_return_to_userland) diff --git a/src/system/kernel/arch/x86/64/thread.cpp b/src/system/kernel/arch/x86/64/thread.cpp index 961a2534c1..9e535fefbe 100644 --- a/src/system/kernel/arch/x86/64/thread.cpp +++ b/src/system/kernel/arch/x86/64/thread.cpp @@ -134,9 +134,12 @@ arch_thread_init(kernel_args* args) { // Save one global valid FPU state; it will be copied in the arch dependent // part of each new thread. - asm volatile ("clts; fninit; fnclex;"); - x86_fxsave(sInitialState.fpu_state); - + asm volatile ( + "clts;" \ + "fninit;" \ + "fnclex;" \ + "fxsave %0;" + : "=m" (sInitialState.fpu_state)); return B_OK; } @@ -296,15 +299,14 @@ arch_setup_signal_frame(Thread* thread, struct sigaction* action, signalFrameData->context.uc_mcontext.rip = frame->ip; signalFrameData->context.uc_mcontext.rflags = frame->flags; - // Store the FPU state. There appears to be a bug in GCC where the aligned - // attribute on a structure is being ignored when the structure is allocated - // on the stack, so even if the fpu_state struct has aligned(16) it may not - // get aligned correctly. Instead, use the current thread's FPU save area - // and then memcpy() to the frame structure. - x86_fxsave(thread->arch_info.fpu_state); - memcpy((void*)&signalFrameData->context.uc_mcontext.fpu, - thread->arch_info.fpu_state, - sizeof(signalFrameData->context.uc_mcontext.fpu)); + if (frame->fpu != nullptr) { + memcpy((void*)&signalFrameData->context.uc_mcontext.fpu, frame->fpu, + sizeof(signalFrameData->context.uc_mcontext.fpu)); + } else { + memcpy((void*)&signalFrameData->context.uc_mcontext.fpu, + sInitialState.fpu_state, + sizeof(signalFrameData->context.uc_mcontext.fpu)); + } // Fill in signalFrameData->context.uc_stack. signal_get_user_stack(frame->user_sp, &signalFrameData->context.uc_stack); @@ -370,13 +372,12 @@ arch_restore_signal_frame(struct signal_frame_data* signalFrameData) frame->flags = (frame->flags & ~(uint64)X86_EFLAGS_USER_FLAGS) | (signalFrameData->context.uc_mcontext.rflags & X86_EFLAGS_USER_FLAGS); - // Same as above, alignment may not be correct. Copy to thread and restore - // from there. Thread* thread = thread_get_current_thread(); + memcpy(thread->arch_info.fpu_state, (void*)&signalFrameData->context.uc_mcontext.fpu, sizeof(thread->arch_info.fpu_state)); - x86_fxrstor(thread->arch_info.fpu_state); + frame->fpu = &thread->arch_info.fpu_state; // The syscall return code overwrites frame->ax with the return value of // the syscall, need to return it here to ensure the correct value is diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 93d58f81e4..65ea7fce1c 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -59,14 +59,6 @@ static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { { "NSC", { "Geode by NSC" } }, }; -#define CR0_CACHE_DISABLE (1UL << 30) -#define CR0_NOT_WRITE_THROUGH (1UL << 29) -#define CR0_FPU_EMULATION (1UL << 2) -#define CR0_MONITOR_FPU (1UL << 1) - -#define CR4_OS_FXSR (1UL << 9) -#define CR4_OS_XMM_EXCEPTION (1UL << 10) - #define K8_SMIONCMPHALT (1ULL << 27) #define K8_C1EONCMPHALT (1ULL << 28) @@ -90,8 +82,10 @@ extern "C" void x86_reboot(void); // from arch.S void (*gCpuIdleFunc)(void); +#ifndef __x86_64__ void (*gX86SwapFPUFunc)(void* oldState, const void* newState) = x86_noop_swap; bool gHasSSE = false; +#endif static uint32 sCpuRendezvous; static uint32 sCpuRendezvous2; @@ -105,18 +99,6 @@ static const size_t kDoubleFaultStackSize = 4096; // size per CPU static x86_cpu_module_info* sCpuModule; -extern "C" void memcpy_generic(void* dest, const void* source, size_t count); -extern int memcpy_generic_end; -extern "C" void memset_generic(void* dest, int value, size_t count); -extern int memset_generic_end; - -x86_optimized_functions gOptimizedFunctions = { - memcpy_generic, - &memcpy_generic_end, - memset_generic, - &memset_generic_end -}; - /* CPU topology information */ static uint32 (*sGetCPUTopologyID)(int currentCPU); static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS]; @@ -338,13 +320,14 @@ x86_init_fpu(void) #endif dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__); - +#ifndef __x86_64__ // enable OS support for SSE x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); gX86SwapFPUFunc = x86_fxsave_swap; gHasSSE = true; +#endif } @@ -1163,33 +1146,6 @@ arch_cpu_init_post_modules(kernel_args* args) call_all_cpus(&init_mtrrs, NULL); } - // get optimized functions from the CPU module - if (sCpuModule != NULL && sCpuModule->get_optimized_functions != NULL) { - x86_optimized_functions functions; - memset(&functions, 0, sizeof(functions)); - - sCpuModule->get_optimized_functions(&functions); - - if (functions.memcpy != NULL) { - gOptimizedFunctions.memcpy = functions.memcpy; - gOptimizedFunctions.memcpy_end = functions.memcpy_end; - } - - if (functions.memset != NULL) { - gOptimizedFunctions.memset = functions.memset; - gOptimizedFunctions.memset_end = functions.memset_end; - } - } - - // put the optimized functions into the commpage - size_t memcpyLen = (addr_t)gOptimizedFunctions.memcpy_end - - (addr_t)gOptimizedFunctions.memcpy; - addr_t memcpyPosition = fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMCPY, - (const void*)gOptimizedFunctions.memcpy, memcpyLen); - size_t memsetLen = (addr_t)gOptimizedFunctions.memset_end - - (addr_t)gOptimizedFunctions.memset; - addr_t memsetPosition = fill_commpage_entry(COMMPAGE_ENTRY_X86_MEMSET, - (const void*)gOptimizedFunctions.memset, memsetLen); size_t threadExitLen = (addr_t)x86_end_userspace_thread_exit - (addr_t)x86_userspace_thread_exit; addr_t threadExitPosition = fill_commpage_entry( @@ -1198,10 +1154,7 @@ arch_cpu_init_post_modules(kernel_args* args) // add the functions to the commpage image image_id image = get_commpage_image(); - elf_add_memory_image_symbol(image, "commpage_memcpy", memcpyPosition, - memcpyLen, B_SYMBOL_TYPE_TEXT); - elf_add_memory_image_symbol(image, "commpage_memset", memsetPosition, - memsetLen, B_SYMBOL_TYPE_TEXT); + elf_add_memory_image_symbol(image, "commpage_thread_exit", threadExitPosition, threadExitLen, B_SYMBOL_TYPE_TEXT); diff --git a/src/system/kernel/arch/x86/arch_thread.cpp b/src/system/kernel/arch/x86/arch_thread.cpp index 25cde443ed..931c74b099 100644 --- a/src/system/kernel/arch/x86/arch_thread.cpp +++ b/src/system/kernel/arch/x86/arch_thread.cpp @@ -31,7 +31,9 @@ extern "C" void x86_return_to_userland(iframe* frame); // from arch_cpu.cpp +#ifndef __x86_64__ extern void (*gX86SwapFPUFunc)(void *oldState, const void *newState); +#endif static struct iframe* @@ -245,7 +247,9 @@ arch_thread_context_switch(Thread* from, Thread* to) activePagingStructures->RemoveReference(); } +#ifndef __x86_64__ gX86SwapFPUFunc(from->arch_info.fpu_state, to->arch_info.fpu_state); +#endif x86_context_switch(&from->arch_info, &to->arch_info); } diff --git a/src/system/kernel/arch/x86/arch_user_debugger.cpp b/src/system/kernel/arch/x86/arch_user_debugger.cpp index 516ed8f340..8b621d9e3c 100644 --- a/src/system/kernel/arch/x86/arch_user_debugger.cpp +++ b/src/system/kernel/arch/x86/arch_user_debugger.cpp @@ -33,7 +33,9 @@ // TODO: Make those real error codes. +#ifndef __x86_64__ extern bool gHasSSE; +#endif // The software breakpoint instruction (int3). const uint8 kX86SoftwareBreakpoint[1] = { 0xcc }; @@ -688,6 +690,12 @@ arch_set_debug_cpu_state(const debug_cpu_state* cpuState) if (iframe* frame = x86_get_user_iframe()) { // For the floating point state to be correct the calling function must // not use these registers (not even indirectly). +#ifdef __x86_64__ + Thread* thread = thread_get_current_thread(); + memcpy(thread->arch_info.fpu_state, &cpuState->extended_registers, + sizeof(cpuState->extended_registers)); + frame->fpu = &thread->arch_info.fpu_state; +#else if (gHasSSE) { // Since fxrstor requires 16-byte alignment and this isn't // guaranteed passed buffer, we use our thread's fpu_state field as @@ -698,12 +706,11 @@ arch_set_debug_cpu_state(const debug_cpu_state* cpuState) memcpy(thread->arch_info.fpu_state, &cpuState->extended_registers, sizeof(cpuState->extended_registers)); x86_fxrstor(thread->arch_info.fpu_state); -#ifndef __x86_64__ } else { // TODO: Implement! We need to convert the format first. // x86_frstor(&cpuState->extended_registers); -#endif } +#endif set_iframe_registers(frame, cpuState); } } @@ -715,6 +722,15 @@ arch_get_debug_cpu_state(debug_cpu_state* cpuState) if (iframe* frame = x86_get_user_iframe()) { // For the floating point state to be correct the calling function must // not use these registers (not even indirectly). +#ifdef __x86_64__ + if (frame->fpu != nullptr) { + memcpy(&cpuState->extended_registers, frame->fpu, + sizeof(cpuState->extended_registers)); + } else { + memset(&cpuState->extended_registers, 0, + sizeof(cpuState->extended_registers)); + } +#else if (gHasSSE) { // Since fxsave requires 16-byte alignment and this isn't guaranteed // passed buffer, we use our thread's fpu_state field as temporary @@ -725,15 +741,14 @@ arch_get_debug_cpu_state(debug_cpu_state* cpuState) // unlike fnsave, fxsave doesn't reinit the FPU state memcpy(&cpuState->extended_registers, thread->arch_info.fpu_state, sizeof(cpuState->extended_registers)); -#ifndef __x86_64__ } else { x86_fnsave(&cpuState->extended_registers); x86_frstor(&cpuState->extended_registers); // fnsave reinits the FPU state after saving, so we need to // load it again // TODO: Convert to fxsave format! -#endif } +#endif get_iframe_registers(frame, cpuState); } } diff --git a/src/system/kernel/arch/x86/asm_offsets.cpp b/src/system/kernel/arch/x86/asm_offsets.cpp index 787fef10e9..d89200bf87 100644 --- a/src/system/kernel/arch/x86/asm_offsets.cpp +++ b/src/system/kernel/arch/x86/asm_offsets.cpp @@ -70,6 +70,7 @@ dummy() DEFINE_OFFSET_MACRO(IFRAME, iframe, r8); DEFINE_OFFSET_MACRO(IFRAME, iframe, r9); DEFINE_OFFSET_MACRO(IFRAME, iframe, r10); + DEFINE_OFFSET_MACRO(IFRAME, iframe, fpu); #else DEFINE_OFFSET_MACRO(IFRAME, iframe, orig_eax); #endif @@ -79,12 +80,6 @@ dummy() DEFINE_OFFSET_MACRO(SYSCALL_INFO, syscall_info, function); DEFINE_OFFSET_MACRO(SYSCALL_INFO, syscall_info, parameter_size); - // struct x86_optimized_functions - DEFINE_OFFSET_MACRO(X86_OPTIMIZED_FUNCTIONS, x86_optimized_functions, - memcpy); - DEFINE_OFFSET_MACRO(X86_OPTIMIZED_FUNCTIONS, x86_optimized_functions, - memset); - // struct signal_frame_data DEFINE_SIZEOF_MACRO(SIGNAL_FRAME_DATA, signal_frame_data); DEFINE_OFFSET_MACRO(SIGNAL_FRAME_DATA, signal_frame_data, info); diff --git a/src/system/kernel/lib/arch/x86/arch_string.S b/src/system/kernel/lib/arch/x86/arch_string.S index 62630627b2..7a3480b2c5 100644 --- a/src/system/kernel/lib/arch/x86/arch_string.S +++ b/src/system/kernel/lib/arch/x86/arch_string.S @@ -6,22 +6,12 @@ * Distributed under the terms of the NewOS License. */ -#if !_BOOT_MODE -# include "asm_offsets.h" -#endif #include -// We don't need the indirection in the boot loader. -#if _BOOT_MODE -# define memcpy_generic memcpy -# define memset_generic memset -#endif - - .align 4 -FUNCTION(memcpy_generic): +FUNCTION(memcpy): pushl %esi pushl %edi movl 12(%esp),%edi /* dest */ @@ -45,13 +35,13 @@ FUNCTION(memcpy_generic): popl %edi popl %esi ret -FUNCTION_END(memcpy_generic) -SYMBOL(memcpy_generic_end): +FUNCTION_END(memcpy) +SYMBOL(memcpy_end): /* void *memset(void *dest, int value, size_t length); */ .align 4 -FUNCTION(memset_generic): +FUNCTION(memset): push %ebp mov %esp, %ebp @@ -111,19 +101,6 @@ FUNCTION(memset_generic): mov %ebp, %esp pop %ebp ret -FUNCTION_END(memset_generic) -SYMBOL(memset_generic_end): - - -#if !_BOOT_MODE - -.align 4 -FUNCTION(memcpy): - jmp *(gOptimizedFunctions + X86_OPTIMIZED_FUNCTIONS_memcpy) -FUNCTION_END(memcpy) - -FUNCTION(memset): - jmp *(gOptimizedFunctions + X86_OPTIMIZED_FUNCTIONS_memset) FUNCTION_END(memset) +SYMBOL(memset_end): -#endif // !_BOOT_MODE diff --git a/src/system/kernel/lib/arch/x86_64/Jamfile b/src/system/kernel/lib/arch/x86_64/Jamfile index 7f06e09e3a..67a3890ac0 100644 --- a/src/system/kernel/lib/arch/x86_64/Jamfile +++ b/src/system/kernel/lib/arch/x86_64/Jamfile @@ -21,6 +21,7 @@ KernelMergeObject kernel_os_arch_$(TARGET_ARCH).o : ; SEARCH_SOURCE += [ FDirName $(posixSources) arch $(TARGET_ARCH) ] ; +SEARCH_SOURCE += [ FDirName $(posixSources) string arch $(TARGET_ARCH) ] ; KernelMergeObject kernel_lib_posix_arch_$(TARGET_ARCH).o : siglongjmp.S @@ -28,12 +29,8 @@ KernelMergeObject kernel_lib_posix_arch_$(TARGET_ARCH).o : kernel_longjmp_return.c kernel_setjmp_save_sigs.c - arch_string.S + arch_string.cpp : $(TARGET_KERNEL_PIC_CCFLAGS) ; -# Explicitly tell the build system that arch_string.S includes the generated -# asm_offsets.h. -Includes [ FGristFiles arch_string.S ] - : asm_offsets.h ; diff --git a/src/system/kernel/lib/arch/x86_64/arch_string.S b/src/system/kernel/lib/arch/x86_64/arch_string.S deleted file mode 100644 index a24bbc8ff6..0000000000 --- a/src/system/kernel/lib/arch/x86_64/arch_string.S +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. - * Distributed under the terms of the MIT License. - */ - - -#include - -#include "asm_offsets.h" - - -.align 8 -FUNCTION(memcpy_generic): - push %rbp - movq %rsp, %rbp - - // Preserve original destination address for return value. - movq %rdi, %rax - - // size -> %rcx - movq %rdx, %rcx - - // For small copies, always do it bytewise, the additional overhead is - // not worth it. - cmp $24, %rcx - jl .Lmemcpy_generic_byte_copy - - // Do both source and dest have the same alignment? - movq %rsi, %r8 - xorq %rdi, %r8 - test $7, %r8 - jnz .Lmemcpy_generic_byte_copy - - // Align up to an 8-byte boundary. - movq %rdi, %r8 - andq $7, %r8 - jz .Lmemcpy_generic_qword_copy - movq $8, %rcx - subq %r8, %rcx - subq %rcx, %rdx // Subtract from the overall count. - rep - movsb - - // Get back the original count value. - movq %rdx, %rcx -.Lmemcpy_generic_qword_copy: - // Move by quadwords. - shrq $3, %rcx - rep - movsq - - // Get the remaining count. - movq %rdx, %rcx - andq $7, %rcx -.Lmemcpy_generic_byte_copy: - // Move any remaining data by bytes. - rep - movsb - - pop %rbp - ret -FUNCTION_END(memcpy_generic) -SYMBOL(memcpy_generic_end): - - -.align 8 -FUNCTION(memset_generic): - push %rbp - movq %rsp, %rbp - - // Preserve original destination address for return value. - movq %rdi, %r8 - - // size -> %rcx, value -> %al - movq %rdx, %rcx - movl %esi, %eax - - // Move by bytes. - rep - stosb - - movq %r8, %rax - pop %rbp - ret -FUNCTION_END(memset_generic) -SYMBOL(memset_generic_end): - - -FUNCTION(memcpy): - jmp *(gOptimizedFunctions + X86_OPTIMIZED_FUNCTIONS_memcpy) -FUNCTION_END(memcpy) - -FUNCTION(memset): - jmp *(gOptimizedFunctions + X86_OPTIMIZED_FUNCTIONS_memset) -FUNCTION_END(memset) - diff --git a/src/system/libroot/posix/string/arch/x86_64/Jamfile b/src/system/libroot/posix/string/arch/x86_64/Jamfile index 6d843894d4..b8cd49040d 100644 --- a/src/system/libroot/posix/string/arch/x86_64/Jamfile +++ b/src/system/libroot/posix/string/arch/x86_64/Jamfile @@ -1,5 +1,7 @@ SubDir HAIKU_TOP src system libroot posix string arch x86_64 ; +SubDirC++Flags -std=gnu++11 ; + local architectureObject ; for architectureObject in [ MultiArchSubDirSetup x86_64 ] { on $(architectureObject) { @@ -8,7 +10,7 @@ for architectureObject in [ MultiArchSubDirSetup x86_64 ] { UsePrivateSystemHeaders ; MergeObject <$(architecture)>posix_string_arch_$(TARGET_ARCH).o : - arch_string.S + arch_string.cpp ; } } diff --git a/src/system/libroot/posix/string/arch/x86_64/arch_string.S b/src/system/libroot/posix/string/arch/x86_64/arch_string.S deleted file mode 100644 index e1273fdc3c..0000000000 --- a/src/system/libroot/posix/string/arch/x86_64/arch_string.S +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. - * Distributed under the terms of the MIT License. - */ - -#include -#include - - -FUNCTION(memcpy): - movq __gCommPageAddress@GOTPCREL(%rip), %rax - movq (%rax), %rax - addq 8 * COMMPAGE_ENTRY_X86_MEMCPY(%rax), %rax - jmp *%rax -FUNCTION_END(memcpy) - -FUNCTION(memset): - movq __gCommPageAddress@GOTPCREL(%rip), %rax - movq (%rax), %rax - addq 8 * COMMPAGE_ENTRY_X86_MEMSET(%rax), %rax - jmp *%rax -FUNCTION_END(memset) diff --git a/src/system/libroot/posix/string/arch/x86_64/arch_string.cpp b/src/system/libroot/posix/string/arch/x86_64/arch_string.cpp new file mode 100644 index 0000000000..8639327b20 --- /dev/null +++ b/src/system/libroot/posix/string/arch/x86_64/arch_string.cpp @@ -0,0 +1,216 @@ +/* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include +#include + +#include + + +namespace { + + +template class Generator, unsigned N, unsigned ...Index> +struct GenerateTable : GenerateTable { +}; + +template class Generator, unsigned ...Index> +struct GenerateTable + : std::array::sValue), sizeof...(Index)> { + constexpr GenerateTable() + : + std::array::sValue), sizeof...(Index)> { + { Generator::sValue... } + } + { + } +}; + + +static inline void memcpy_repmovs(uint8_t* destination, const uint8_t* source, + size_t length) +{ + __asm__ __volatile__("rep movsb" + : "+D" (destination), "+S" (source), "+c" (length) + : + : "memory"); +} + + +template +inline void copy_small(uint8_t* destination, const uint8_t* source) +{ + struct data { + uint8_t x[N]; + }; + *reinterpret_cast(destination) + = *reinterpret_cast(source); +} + + +template +struct SmallGenerator { + constexpr static void (*sValue)(uint8_t*, const uint8_t*) = copy_small; +}; +constexpr static GenerateTable table_small; + + +static inline void memcpy_small(uint8_t* destination, const uint8_t* source, + size_t length) +{ + if (length < 8) { + table_small[length](destination, source); + } else { + auto to = reinterpret_cast(destination); + auto from = reinterpret_cast(source); + *to = *from; + to = reinterpret_cast(destination + length - 8); + from = reinterpret_cast(source + length - 8); + *to = *from; + } +} + + +template +inline void copy_sse(__m128i* destination, const __m128i* source) +{ + auto temp = _mm_loadu_si128(source); + _mm_storeu_si128(destination, temp); + copy_sse(destination + 1, source + 1); +} + + +template<> +inline void copy_sse<0>(__m128i* destination, const __m128i* source) +{ +} + + +template +struct SSEGenerator { + constexpr static void (*sValue)(__m128i*, const __m128i*) = copy_sse; +}; +constexpr static GenerateTable table_sse; + + +static inline void memcpy_sse(uint8_t* destination, const uint8_t* source, size_t length) +{ + auto to = reinterpret_cast<__m128i*>(destination); + auto from = reinterpret_cast(source); + auto toEnd = reinterpret_cast<__m128i*>(destination + length - 16); + auto fromEnd = reinterpret_cast(source + length - 16); + while (length >= 64) { + copy_sse<4>(to, from); + to += 4; + from += 4; + length -= 64; + } + if (length >= 16) { + table_sse[length / 16](to, from); + length %= 16; + } + if (length) { + copy_sse<1>(toEnd, fromEnd); + } +} + + +} + + +extern "C" void* memcpy(void* destination, const void* source, size_t length) +{ + auto to = static_cast(destination); + auto from = static_cast(source); + if (length <= 16) { + memcpy_small(to, from, length); + return destination; + } + if (length < 2048) { + memcpy_sse(to, from, length); + return destination; + } + memcpy_repmovs(to, from, length); + return destination; +} + + +static inline void +memset_repstos(uint8_t* destination, uint8_t value, size_t length) +{ + __asm__ __volatile__("rep stosb" + : "+D" (destination), "+c" (length) + : "a" (value) + : "memory"); +} + + +static inline void +memset_sse(uint8_t* destination, uint8_t value, size_t length) +{ + __m128i packed = _mm_set1_epi8(value); + auto end = reinterpret_cast<__m128i*>(destination + length - 16); + auto diff = reinterpret_cast(destination) % 16; + if (diff) { + diff = 16 - diff; + length -= diff; + _mm_storeu_si128(reinterpret_cast<__m128i*>(destination), packed); + } + auto ptr = reinterpret_cast<__m128i*>(destination + diff); + while (length >= 64) { + _mm_store_si128(ptr++, packed); + _mm_store_si128(ptr++, packed); + _mm_store_si128(ptr++, packed); + _mm_store_si128(ptr++, packed); + length -= 64; + } + while (length >= 16) { + _mm_store_si128(ptr++, packed); + length -= 16; + } + _mm_storeu_si128(end, packed); +} + + +static inline void +memset_small(uint8_t* destination, uint8_t value, size_t length) +{ + if (length >= 8) { + auto packed = value * 0x101010101010101ul; + auto ptr = reinterpret_cast(destination); + auto end = reinterpret_cast(destination + length - 8); + while (length >= 8) { + *ptr++ = packed; + length -= 8; + } + *end = packed; + } else { + while (length--) { + *destination++ = value; + } + } +} + + +extern "C" void* +memset(void* ptr, int chr, size_t length) +{ + auto value = static_cast(chr); + auto destination = static_cast(ptr); + if (length < 32) { + memset_small(destination, value, length); + return ptr; + } + if (length < 2048) { + memset_sse(destination, value, length); + return ptr; + } + memset_repstos(destination, value, length); + return ptr; +} +