From 62793934a147e8e44d300e07c1de195d98fb6c95 Mon Sep 17 00:00:00 2001 From: Augustin Cavalier Date: Wed, 5 Mar 2025 11:59:26 -0500 Subject: [PATCH] x86, x86_64: Move (userland) memcpy, memset out of arch_string.S/.cpp. Put the C++ versions in files with their own names, and rename the assembly thunks to "commpage_string.S", as it really just invokes the commpage versions. The kernel x86 arch_string.S is left alone for the moment as it will be replaced in the near future anyway. --- src/system/boot/arch/x86/Jamfile | 2 +- src/system/kernel/lib/arch/x86_64/Jamfile | 4 +- .../libroot/posix/string/arch/x86/Jamfile | 2 +- .../x86/{arch_string.S => commpage_string.S} | 0 .../libroot/posix/string/arch/x86_64/Jamfile | 3 +- .../x86_64/{arch_string.cpp => memcpy.cpp} | 98 +++---------------- .../posix/string/arch/x86_64/memset.cpp | 88 +++++++++++++++++ 7 files changed, 107 insertions(+), 90 deletions(-) rename src/system/libroot/posix/string/arch/x86/{arch_string.S => commpage_string.S} (100%) rename src/system/libroot/posix/string/arch/x86_64/{arch_string.cpp => memcpy.cpp} (59%) create mode 100644 src/system/libroot/posix/string/arch/x86_64/memset.cpp diff --git a/src/system/boot/arch/x86/Jamfile b/src/system/boot/arch/x86/Jamfile index 5e783064d6..6816e2deae 100644 --- a/src/system/boot/arch/x86/Jamfile +++ b/src/system/boot/arch/x86/Jamfile @@ -35,7 +35,7 @@ for platform in [ MultiBootSubDirSetup bios_ia32 efi pxe_ia32 ] { local kernelLibArchSpecificSources ; if $(TARGET_ARCH) = x86_64 && $(TARGET_BOOT_PLATFORM) = efi { kernelArchSpecificSources = cpuid.cpp ; - kernelLibArchSpecificSources = arch_string.cpp ; + kernelLibArchSpecificSources = memcpy.cpp memset.cpp ; } else { kernelArchSpecificSources = cpuid.S ; kernelLibArchSpecificSources = arch_string.S ; diff --git a/src/system/kernel/lib/arch/x86_64/Jamfile b/src/system/kernel/lib/arch/x86_64/Jamfile index 24c29552bf..9cb6a2be98 100644 --- a/src/system/kernel/lib/arch/x86_64/Jamfile +++ b/src/system/kernel/lib/arch/x86_64/Jamfile @@ -29,8 +29,8 @@ KernelMergeObject kernel_lib_posix_arch_$(TARGET_ARCH).o : kernel_longjmp_return.c kernel_setjmp_save_sigs.c - arch_string.cpp + memcpy.cpp + memset.cpp : $(TARGET_KERNEL_PIC_CCFLAGS) ; - diff --git a/src/system/libroot/posix/string/arch/x86/Jamfile b/src/system/libroot/posix/string/arch/x86/Jamfile index e87545b09a..a654821165 100644 --- a/src/system/libroot/posix/string/arch/x86/Jamfile +++ b/src/system/libroot/posix/string/arch/x86/Jamfile @@ -14,7 +14,7 @@ for architectureObject in [ MultiArchSubDirSetup x86 x86_gcc2 ] { MergeObject <$(architecture)>posix_string_arch_$(TARGET_ARCH).o : [ MultiArchIfPrimary memcpy.c memset.c : - arch_string.S : x86_64 ] + commpage_string.S : x86_64 ] ; } diff --git a/src/system/libroot/posix/string/arch/x86/arch_string.S b/src/system/libroot/posix/string/arch/x86/commpage_string.S similarity index 100% rename from src/system/libroot/posix/string/arch/x86/arch_string.S rename to src/system/libroot/posix/string/arch/x86/commpage_string.S diff --git a/src/system/libroot/posix/string/arch/x86_64/Jamfile b/src/system/libroot/posix/string/arch/x86_64/Jamfile index 389f9c0146..fc7761f479 100644 --- a/src/system/libroot/posix/string/arch/x86_64/Jamfile +++ b/src/system/libroot/posix/string/arch/x86_64/Jamfile @@ -11,7 +11,8 @@ for architectureObject in [ MultiArchSubDirSetup x86_64 ] { UsePrivateSystemHeaders ; MergeObject <$(architecture)>posix_string_arch_$(TARGET_ARCH).o : - arch_string.cpp + memcpy.cpp + memset.cpp ; } } diff --git a/src/system/libroot/posix/string/arch/x86_64/arch_string.cpp b/src/system/libroot/posix/string/arch/x86_64/memcpy.cpp similarity index 59% rename from src/system/libroot/posix/string/arch/x86_64/arch_string.cpp rename to src/system/libroot/posix/string/arch/x86_64/memcpy.cpp index d2f135b14e..3e51e38167 100644 --- a/src/system/libroot/posix/string/arch/x86_64/arch_string.cpp +++ b/src/system/libroot/posix/string/arch/x86_64/memcpy.cpp @@ -44,8 +44,8 @@ struct GenerateTable #pragma GCC diagnostic pop -static inline void memcpy_repmovs(uint8_t* destination, const uint8_t* source, - size_t length) +static inline void +memcpy_repmovs(uint8_t* destination, const uint8_t* source, size_t length) { __asm__ __volatile__("rep movsb" : "+D" (destination), "+S" (source), "+c" (length) @@ -72,8 +72,8 @@ struct SmallGenerator { constexpr static GenerateTable table_small; -static inline void memcpy_small(uint8_t* destination, const uint8_t* source, - size_t length) +static inline void +memcpy_small(uint8_t* destination, const uint8_t* source, size_t length) { if (length < 8) { table_small[length](destination, source); @@ -89,7 +89,8 @@ static inline void memcpy_small(uint8_t* destination, const uint8_t* source, template -inline void copy_sse(__m128i* destination, const __m128i* source) +inline void +copy_sse(__m128i* destination, const __m128i* source) { auto temp = _mm_loadu_si128(source); _mm_storeu_si128(destination, temp); @@ -98,7 +99,8 @@ inline void copy_sse(__m128i* destination, const __m128i* source) template<> -inline void copy_sse<0>(__m128i* destination, const __m128i* source) +inline void +copy_sse<0>(__m128i* destination, const __m128i* source) { } @@ -110,7 +112,8 @@ struct SSEGenerator { constexpr static GenerateTable table_sse; -static inline void memcpy_sse(uint8_t* destination, const uint8_t* source, size_t length) +static inline void +memcpy_sse(uint8_t* destination, const uint8_t* source, size_t length) { auto to = reinterpret_cast<__m128i*>(destination); auto from = reinterpret_cast(source); @@ -132,10 +135,11 @@ static inline void memcpy_sse(uint8_t* destination, const uint8_t* source, size_ } -} +} // namespace -extern "C" void* memcpy(void* destination, const void* source, size_t length) +extern "C" void* +memcpy(void* destination, const void* source, size_t length) { auto to = static_cast(destination); auto from = static_cast(source); @@ -150,79 +154,3 @@ extern "C" void* memcpy(void* destination, const void* source, size_t length) memcpy_repmovs(to, from, length); return destination; } - - -static inline void -memset_repstos(uint8_t* destination, uint8_t value, size_t length) -{ - __asm__ __volatile__("rep stosb" - : "+D" (destination), "+c" (length) - : "a" (value) - : "memory"); -} - - -static inline void -memset_sse(uint8_t* destination, uint8_t value, size_t length) -{ - __m128i packed = _mm_set1_epi8(value); - auto end = reinterpret_cast<__m128i*>(destination + length - 16); - auto diff = reinterpret_cast(destination) % 16; - if (diff) { - diff = 16 - diff; - length -= diff; - _mm_storeu_si128(reinterpret_cast<__m128i*>(destination), packed); - } - auto ptr = reinterpret_cast<__m128i*>(destination + diff); - while (length >= 64) { - _mm_store_si128(ptr++, packed); - _mm_store_si128(ptr++, packed); - _mm_store_si128(ptr++, packed); - _mm_store_si128(ptr++, packed); - length -= 64; - } - while (length >= 16) { - _mm_store_si128(ptr++, packed); - length -= 16; - } - _mm_storeu_si128(end, packed); -} - - -static inline void -memset_small(uint8_t* destination, uint8_t value, size_t length) -{ - if (length >= 8) { - auto packed = value * 0x101010101010101ul; - auto ptr = reinterpret_cast(destination); - auto end = reinterpret_cast(destination + length - 8); - while (length >= 8) { - *ptr++ = packed; - length -= 8; - } - *end = packed; - } else { - while (length--) { - *destination++ = value; - } - } -} - - -extern "C" void* -memset(void* ptr, int chr, size_t length) -{ - auto value = static_cast(chr); - auto destination = static_cast(ptr); - if (length < 32) { - memset_small(destination, value, length); - return ptr; - } - if (length < 2048) { - memset_sse(destination, value, length); - return ptr; - } - memset_repstos(destination, value, length); - return ptr; -} - diff --git a/src/system/libroot/posix/string/arch/x86_64/memset.cpp b/src/system/libroot/posix/string/arch/x86_64/memset.cpp new file mode 100644 index 0000000000..08e970d465 --- /dev/null +++ b/src/system/libroot/posix/string/arch/x86_64/memset.cpp @@ -0,0 +1,88 @@ +/* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include +#include + +#include + + +static inline void +memset_repstos(uint8_t* destination, uint8_t value, size_t length) +{ + __asm__ __volatile__("rep stosb" + : "+D" (destination), "+c" (length) + : "a" (value) + : "memory"); +} + + +static inline void +memset_sse(uint8_t* destination, uint8_t value, size_t length) +{ + __m128i packed = _mm_set1_epi8(value); + auto end = reinterpret_cast<__m128i*>(destination + length - 16); + auto diff = reinterpret_cast(destination) % 16; + if (diff) { + diff = 16 - diff; + length -= diff; + _mm_storeu_si128(reinterpret_cast<__m128i*>(destination), packed); + } + auto ptr = reinterpret_cast<__m128i*>(destination + diff); + while (length >= 64) { + _mm_store_si128(ptr++, packed); + _mm_store_si128(ptr++, packed); + _mm_store_si128(ptr++, packed); + _mm_store_si128(ptr++, packed); + length -= 64; + } + while (length >= 16) { + _mm_store_si128(ptr++, packed); + length -= 16; + } + _mm_storeu_si128(end, packed); +} + + +static inline void +memset_small(uint8_t* destination, uint8_t value, size_t length) +{ + if (length >= 8) { + auto packed = value * 0x101010101010101ul; + auto ptr = reinterpret_cast(destination); + auto end = reinterpret_cast(destination + length - 8); + while (length >= 8) { + *ptr++ = packed; + length -= 8; + } + *end = packed; + } else { + while (length--) { + *destination++ = value; + } + } +} + + +extern "C" void* +memset(void* ptr, int chr, size_t length) +{ + auto value = static_cast(chr); + auto destination = static_cast(ptr); + if (length < 32) { + memset_small(destination, value, length); + return ptr; + } + if (length < 2048) { + memset_sse(destination, value, length); + return ptr; + } + memset_repstos(destination, value, length); + return ptr; +} +