x86, x86_64: Move (userland) memcpy, memset out of arch_string.S/.cpp.

Put the C++ versions in files with their own names, and
rename the assembly thunks to "commpage_string.S", as it
really just invokes the commpage versions.

The kernel x86 arch_string.S is left alone for the moment
as it will be replaced in the near future anyway.

src/system/boot/arch/x86/Jamfile

@@ -35,7 +35,7 @@ for platform in [ MultiBootSubDirSetup bios_ia32 efi pxe_ia32 ] {
 		local kernelLibArchSpecificSources ;
 		if $(TARGET_ARCH) = x86_64 && $(TARGET_BOOT_PLATFORM) = efi {
 			kernelArchSpecificSources = cpuid.cpp ;
-			kernelLibArchSpecificSources = arch_string.cpp ;
+			kernelLibArchSpecificSources = memcpy.cpp memset.cpp ;
 		} else {
 			kernelArchSpecificSources = cpuid.S ;
 			kernelLibArchSpecificSources = arch_string.S ;

src/system/kernel/lib/arch/x86_64/Jamfile

@@ -29,8 +29,8 @@ KernelMergeObject kernel_lib_posix_arch_$(TARGET_ARCH).o :
 	kernel_longjmp_return.c
 	kernel_setjmp_save_sigs.c
 
-	arch_string.cpp
+	memcpy.cpp
+	memset.cpp
 
 	: $(TARGET_KERNEL_PIC_CCFLAGS)
 ;
-

src/system/libroot/posix/string/arch/x86/Jamfile

@@ -14,7 +14,7 @@ for architectureObject in [ MultiArchSubDirSetup x86 x86_gcc2 ] {
 
 		MergeObject <$(architecture)>posix_string_arch_$(TARGET_ARCH).o :
 			[ MultiArchIfPrimary memcpy.c memset.c :
-				arch_string.S : x86_64 ]
+				commpage_string.S : x86_64 ]
 			;
 
 	}

src/system/libroot/posix/string/arch/x86_64/Jamfile

@@ -11,7 +11,8 @@ for architectureObject in [ MultiArchSubDirSetup x86_64 ] {
 		UsePrivateSystemHeaders ;
 
 		MergeObject <$(architecture)>posix_string_arch_$(TARGET_ARCH).o :
-			arch_string.cpp
+			memcpy.cpp
+			memset.cpp
 			;
 	}
 }

src/system/libroot/posix/string/arch/x86_64/memcpy.cpp

@@ -44,8 +44,8 @@ struct GenerateTable<Generator, 0, Index...>
 #pragma GCC diagnostic pop
 
 
-static inline void memcpy_repmovs(uint8_t* destination, const uint8_t* source,
+static inline void
-	size_t length)
+memcpy_repmovs(uint8_t* destination, const uint8_t* source, size_t length)
 {
 	__asm__ __volatile__("rep movsb"
 		: "+D" (destination), "+S" (source), "+c" (length)
@@ -72,8 +72,8 @@ struct SmallGenerator {
 constexpr static GenerateTable<SmallGenerator, 8> table_small;
 
 
-static inline void memcpy_small(uint8_t* destination, const uint8_t* source,
+static inline void
-	size_t length)
+memcpy_small(uint8_t* destination, const uint8_t* source, size_t length)
 {
 	if (length < 8) {
 		table_small[length](destination, source);
@@ -89,7 +89,8 @@ static inline void memcpy_small(uint8_t* destination, const uint8_t* source,
 
 
 template<size_t N>
-inline void copy_sse(__m128i* destination, const __m128i* source)
+inline void
+copy_sse(__m128i* destination, const __m128i* source)
 {
 	auto temp = _mm_loadu_si128(source);
 	_mm_storeu_si128(destination, temp);
@@ -98,7 +99,8 @@ inline void copy_sse(__m128i* destination, const __m128i* source)
 
 
 template<>
-inline void copy_sse<0>(__m128i* destination, const __m128i* source)
+inline void
+copy_sse<0>(__m128i* destination, const __m128i* source)
 {
 }
 
@@ -110,7 +112,8 @@ struct SSEGenerator {
 constexpr static GenerateTable<SSEGenerator, 4> table_sse;
 
 
-static inline void memcpy_sse(uint8_t* destination, const uint8_t* source, size_t length)
+static inline void
+memcpy_sse(uint8_t* destination, const uint8_t* source, size_t length)
 {
 	auto to = reinterpret_cast<__m128i*>(destination);
 	auto from = reinterpret_cast<const __m128i*>(source);
@@ -132,10 +135,11 @@ static inline void memcpy_sse(uint8_t* destination, const uint8_t* source, size_
 }
 
 
-}
+} // namespace
 
 
-extern "C" void* memcpy(void* destination, const void* source, size_t length)
+extern "C" void*
+memcpy(void* destination, const void* source, size_t length)
 {
 	auto to = static_cast<uint8_t*>(destination);
 	auto from = static_cast<const uint8_t*>(source);
@@ -150,79 +154,3 @@ extern "C" void* memcpy(void* destination, const void* source, size_t length)
 	memcpy_repmovs(to, from, length);
 	return destination;
 }
-
-
-static inline void
-memset_repstos(uint8_t* destination, uint8_t value, size_t length)
-{
-	__asm__ __volatile__("rep stosb"
-		: "+D" (destination), "+c" (length)
-		: "a" (value)
-		: "memory");
-}
-
-
-static inline void
-memset_sse(uint8_t* destination, uint8_t value, size_t length)
-{
-	__m128i packed = _mm_set1_epi8(value);
-	auto end = reinterpret_cast<__m128i*>(destination + length - 16);
-	auto diff = reinterpret_cast<uintptr_t>(destination) % 16;
-	if (diff) {
-		diff = 16 - diff;
-		length -= diff;
-		_mm_storeu_si128(reinterpret_cast<__m128i*>(destination), packed);
-	}
-	auto ptr = reinterpret_cast<__m128i*>(destination + diff);
-	while (length >= 64) {
-		_mm_store_si128(ptr++, packed);
-		_mm_store_si128(ptr++, packed);
-		_mm_store_si128(ptr++, packed);
-		_mm_store_si128(ptr++, packed);
-		length -= 64;
-	}
-	while (length >= 16) {
-		_mm_store_si128(ptr++, packed);
-		length -= 16;
-	}
-	_mm_storeu_si128(end, packed);
-}
-
-
-static inline void
-memset_small(uint8_t* destination, uint8_t value, size_t length)
-{
-	if (length >= 8) {
-		auto packed = value * 0x101010101010101ul;
-		auto ptr = reinterpret_cast<uint64_t*>(destination);
-		auto end = reinterpret_cast<uint64_t*>(destination + length - 8);
-		while (length >= 8) {
-			*ptr++ = packed;
-			length -= 8;
-		}
-		*end = packed;
-	} else {
-		while (length--) {
-			*destination++ = value;
-		}
-	}
-}
-
-
-extern "C" void*
-memset(void* ptr, int chr, size_t length)
-{
-	auto value = static_cast<unsigned char>(chr);
-	auto destination = static_cast<uint8_t*>(ptr);
-	if (length < 32) {
-		memset_small(destination, value, length);
-		return ptr;
-	}
-	if (length < 2048) {
-		memset_sse(destination, value, length);
-		return ptr;
-	}
-	memset_repstos(destination, value, length);
-	return ptr;
-}
-

src/system/libroot/posix/string/arch/x86_64/memset.cpp

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2014, Paweł Dziepak, [email protected].
+ * Distributed under the terms of the MIT License.
+ */
+
+
+#include <array>
+
+#include <cstddef>
+#include <cstdint>
+
+#include <emmintrin.h>
+
+
+static inline void
+memset_repstos(uint8_t* destination, uint8_t value, size_t length)
+{
+	__asm__ __volatile__("rep stosb"
+		: "+D" (destination), "+c" (length)
+		: "a" (value)
+		: "memory");
+}
+
+
+static inline void
+memset_sse(uint8_t* destination, uint8_t value, size_t length)
+{
+	__m128i packed = _mm_set1_epi8(value);
+	auto end = reinterpret_cast<__m128i*>(destination + length - 16);
+	auto diff = reinterpret_cast<uintptr_t>(destination) % 16;
+	if (diff) {
+		diff = 16 - diff;
+		length -= diff;
+		_mm_storeu_si128(reinterpret_cast<__m128i*>(destination), packed);
+	}
+	auto ptr = reinterpret_cast<__m128i*>(destination + diff);
+	while (length >= 64) {
+		_mm_store_si128(ptr++, packed);
+		_mm_store_si128(ptr++, packed);
+		_mm_store_si128(ptr++, packed);
+		_mm_store_si128(ptr++, packed);
+		length -= 64;
+	}
+	while (length >= 16) {
+		_mm_store_si128(ptr++, packed);
+		length -= 16;
+	}
+	_mm_storeu_si128(end, packed);
+}
+
+
+static inline void
+memset_small(uint8_t* destination, uint8_t value, size_t length)
+{
+	if (length >= 8) {
+		auto packed = value * 0x101010101010101ul;
+		auto ptr = reinterpret_cast<uint64_t*>(destination);
+		auto end = reinterpret_cast<uint64_t*>(destination + length - 8);
+		while (length >= 8) {
+			*ptr++ = packed;
+			length -= 8;
+		}
+		*end = packed;
+	} else {
+		while (length--) {
+			*destination++ = value;
+		}
+	}
+}
+
+
+extern "C" void*
+memset(void* ptr, int chr, size_t length)
+{
+	auto value = static_cast<unsigned char>(chr);
+	auto destination = static_cast<uint8_t*>(ptr);
+	if (length < 32) {
+		memset_small(destination, value, length);
+		return ptr;
+	}
+	if (length < 2048) {
+		memset_sse(destination, value, length);
+		return ptr;
+	}
+	memset_repstos(destination, value, length);
+	return ptr;
+}
+