libroot/x86_64: new memcpy implementation

This patch introduces new memcpy() implementation that improves the
performance when the buffer is small. It was written for processors that
support ERMSB, but performs reasonably well on older CPUs as well.

The following benchmarks were done on Haswell i7 running Debian Jessie
with Linux 3.16.1. In each iteration 64MB buffer was copied, the
parameter "size" is the size of the buffer passed in a single call (i.e.
for "size: 2" memcpy() was called ~32 million times to copy the whole
64MB).

f - original implementation, g - new implementation, all buffers 16 byte
aligned

cpy, size:        8, f:    79971 µs, g:    20419 µs, ∆:   74.47%
cpy, size:       32, f:    42068 µs, g:    12159 µs, ∆:   71.10%
cpy, size:      128, f:    13408 µs, g:    10359 µs, ∆:   22.74%
cpy, size:      512, f:    10634 µs, g:    10433 µs, ∆:    1.89%
cpy, size:     1024, f:    10474 µs, g:    10536 µs, ∆:   -0.59%
cpy, size:     4096, f:     9419 µs, g:     8630 µs, ∆:    8.38%

f - glibc 2.19 implementation, g - new implementation, all buffers 16 byte
aligned

cpy, size:        8, f:    26299 µs, g:    20919 µs, ∆:   20.46%
cpy, size:       32, f:    11146 µs, g:    12159 µs, ∆:   -9.09%
cpy, size:      128, f:    10778 µs, g:    10354 µs, ∆:    3.93%
cpy, size:      512, f:    12291 µs, g:    10426 µs, ∆:   15.17%
cpy, size:     1024, f:    13923 µs, g:    10571 µs, ∆:   24.08%
cpy, size:     4096, f:    11770 µs, g:     8671 µs, ∆:   26.33%

f - glibc 2.19 implementation, g - new implementation, all buffers unaligned

cpy, size:       16, f:    13376 µs, g:    13009 µs, ∆:    2.74%
cpy, size:       32, f:    11130 µs, g:    12171 µs, ∆:   -9.35%
cpy, size:       64, f:    11017 µs, g:    11231 µs, ∆:   -1.94%
cpy, size:      128, f:    10884 µs, g:    10407 µs, ∆:    4.38%
cpy, size:      256, f:    10826 µs, g:    10106 µs, ∆:    6.65%
cpy, size:      512, f:    12354 µs, g:    10396 µs, ∆:   15.85%

Signed-off-by: Paweł Dziepak <[email protected]>

src/system/libroot/posix/string/arch/x86_64/arch_string.cpp

@@ -4,20 +4,139 @@
  */
 
 
+#include <array>
+
 #include <cstddef>
 #include <cstdint>
 
 #include <x86intrin.h>
 
 
-extern "C" void*
-memcpy(void* destination, const void* source, size_t length)
+namespace {
+
+
+template<template<size_t N> class Generator, unsigned N, unsigned ...Index>
+struct GenerateTable : GenerateTable<Generator, N - 1,  N - 1, Index...> {
+};
+
+template<template<size_t N> class Generator, unsigned ...Index>
+struct GenerateTable<Generator, 0, Index...>
+	: std::array<decltype(Generator<0>::sValue), sizeof...(Index)> {
+	constexpr GenerateTable()
+	:
+	std::array<decltype(Generator<0>::sValue), sizeof...(Index)> {
+		{ Generator<Index>::sValue... }
+	}
+	{
+	}
+};
+
+
+static inline void memcpy_repmovs(uint8_t* destination, const uint8_t* source,
+	size_t length)
 {
-	auto returnValue = destination;
 	__asm__ __volatile__("rep movsb"
 		: "+D" (destination), "+S" (source), "+c" (length)
-		: : "memory");
-	return returnValue;
+		:
+		: "memory");
+}
+
+
+template<size_t N>
+inline void copy_small(uint8_t* destination, const uint8_t* source)
+{
+	struct data {
+		uint8_t x[N];
+	};
+	*reinterpret_cast<data*>(destination)
+		= *reinterpret_cast<const data*>(source);
+}
+
+
+template<size_t N>
+struct SmallGenerator {
+	constexpr static void (*sValue)(uint8_t*, const uint8_t*) = copy_small<N>;
+};
+constexpr static GenerateTable<SmallGenerator, 8> table_small;
+
+
+static inline void memcpy_small(uint8_t* destination, const uint8_t* source,
+	size_t length)
+{
+	if (length < 8) {
+		table_small[length](destination, source);
+	} else {
+		auto to = reinterpret_cast<uint64_t*>(destination);
+		auto from = reinterpret_cast<const uint64_t*>(source);
+		*to = *from;
+		to = reinterpret_cast<uint64_t*>(destination + length - 8);
+		from = reinterpret_cast<const uint64_t*>(source + length - 8);
+		*to = *from;
+	}
+}
+
+
+template<size_t N>
+inline void copy_sse(__m128i* destination, const __m128i* source)
+{
+	auto temp = _mm_loadu_si128(source);
+	_mm_storeu_si128(destination, temp);
+	copy_sse<N - 1>(destination + 1, source + 1);
+}
+
+
+template<>
+inline void copy_sse<0>(__m128i* destination, const __m128i* source)
+{
+}
+
+
+template<size_t N>
+struct SSEGenerator {
+	constexpr static void (*sValue)(__m128i*, const __m128i*) = copy_sse<N>;
+};
+constexpr static GenerateTable<SSEGenerator, 4> table_sse;
+
+
+static inline void memcpy_sse(uint8_t* destination, const uint8_t* source, size_t length)
+{
+	auto to = reinterpret_cast<__m128i*>(destination);
+	auto from = reinterpret_cast<const __m128i*>(source);
+	auto toEnd = reinterpret_cast<__m128i*>(destination + length - 16);
+	auto fromEnd = reinterpret_cast<const __m128i*>(source + length - 16);
+	while (length >= 64) {
+		copy_sse<4>(to, from);
+		to += 4;
+		from += 4;
+		length -= 64;
+	}
+	if (length >= 16) {
+		table_sse[length / 16](to, from);
+		length %= 16;
+	}
+	if (length) {
+		copy_sse<1>(toEnd, fromEnd);
+	}
+}
+
+
+}
+
+
+extern "C" void* memcpy(void* destination, const void* source, size_t length)
+{
+	auto to = static_cast<uint8_t*>(destination);
+	auto from = static_cast<const uint8_t*>(source);
+	if (length <= 16) {
+		memcpy_small(to, from, length);
+		return destination;
+	}
+	if (length < 2048) {
+		memcpy_sse(to, from, length);
+		return destination;
+	}
+	memcpy_repmovs(to, from, length);
+	return destination;
 }