diff --git a/src/system/libroot/posix/string/arch/x86_64/arch_string.cpp b/src/system/libroot/posix/string/arch/x86_64/arch_string.cpp index 33fca22474..8639327b20 100644 --- a/src/system/libroot/posix/string/arch/x86_64/arch_string.cpp +++ b/src/system/libroot/posix/string/arch/x86_64/arch_string.cpp @@ -4,20 +4,139 @@ */ +#include + #include #include #include -extern "C" void* -memcpy(void* destination, const void* source, size_t length) +namespace { + + +template class Generator, unsigned N, unsigned ...Index> +struct GenerateTable : GenerateTable { +}; + +template class Generator, unsigned ...Index> +struct GenerateTable + : std::array::sValue), sizeof...(Index)> { + constexpr GenerateTable() + : + std::array::sValue), sizeof...(Index)> { + { Generator::sValue... } + } + { + } +}; + + +static inline void memcpy_repmovs(uint8_t* destination, const uint8_t* source, + size_t length) { - auto returnValue = destination; __asm__ __volatile__("rep movsb" : "+D" (destination), "+S" (source), "+c" (length) - : : "memory"); - return returnValue; + : + : "memory"); +} + + +template +inline void copy_small(uint8_t* destination, const uint8_t* source) +{ + struct data { + uint8_t x[N]; + }; + *reinterpret_cast(destination) + = *reinterpret_cast(source); +} + + +template +struct SmallGenerator { + constexpr static void (*sValue)(uint8_t*, const uint8_t*) = copy_small; +}; +constexpr static GenerateTable table_small; + + +static inline void memcpy_small(uint8_t* destination, const uint8_t* source, + size_t length) +{ + if (length < 8) { + table_small[length](destination, source); + } else { + auto to = reinterpret_cast(destination); + auto from = reinterpret_cast(source); + *to = *from; + to = reinterpret_cast(destination + length - 8); + from = reinterpret_cast(source + length - 8); + *to = *from; + } +} + + +template +inline void copy_sse(__m128i* destination, const __m128i* source) +{ + auto temp = _mm_loadu_si128(source); + _mm_storeu_si128(destination, temp); + copy_sse(destination + 1, source + 1); +} + + +template<> +inline void copy_sse<0>(__m128i* destination, const __m128i* source) +{ +} + + +template +struct SSEGenerator { + constexpr static void (*sValue)(__m128i*, const __m128i*) = copy_sse; +}; +constexpr static GenerateTable table_sse; + + +static inline void memcpy_sse(uint8_t* destination, const uint8_t* source, size_t length) +{ + auto to = reinterpret_cast<__m128i*>(destination); + auto from = reinterpret_cast(source); + auto toEnd = reinterpret_cast<__m128i*>(destination + length - 16); + auto fromEnd = reinterpret_cast(source + length - 16); + while (length >= 64) { + copy_sse<4>(to, from); + to += 4; + from += 4; + length -= 64; + } + if (length >= 16) { + table_sse[length / 16](to, from); + length %= 16; + } + if (length) { + copy_sse<1>(toEnd, fromEnd); + } +} + + +} + + +extern "C" void* memcpy(void* destination, const void* source, size_t length) +{ + auto to = static_cast(destination); + auto from = static_cast(source); + if (length <= 16) { + memcpy_small(to, from, length); + return destination; + } + if (length < 2048) { + memcpy_sse(to, from, length); + return destination; + } + memcpy_repmovs(to, from, length); + return destination; }