runtime_loader: add support for ELF based TLS

This patch introduces support of ELF based TLS handling with lazy allocation
and initalization of TLS block for each DSO and thread. The implementation
generally follows the official ABI except that generation counter in dtv
is in fact a pointer to Generation object that contains both generation
counter and size of the dtv. That simplified the implementation a bit, but
could be changed later. The ABI requirements regariding in memory position
of TLS block is not honoured what results in static TLS model being
unsupported. However, that should not be a problem as long as
"executables" in Haiku are in fact shared objects and optimizations which
require specific TLS block in memory layout are not possible anyway.

headers/private/runtime_loader/runtime_loader.h

@@ -45,6 +45,8 @@ struct rld_export {
 		const char** _architecture);
 	status_t (*get_next_image_dependency)(image_id id, uint32 *cookie,
 		const char **_name);
+	void* (*get_tls_address)(unsigned dso, addr_t offset);
+	void (*destroy_thread_tls)();
 
 	status_t (*reinit_after_fork)();
 
@@ -114,6 +116,8 @@ typedef struct image_t {
 	elf_rel				*pltrel;
 	int					pltrel_len;
 
+	unsigned			dso_tls_id;
+
 	uint32				num_needed;
 	struct image_t		**needed;
 

headers/private/system/elf_common.h

@@ -158,6 +158,7 @@
 #define STT_FUNC 2
 #define STT_SECTION 3
 #define STT_FILE 4
+#define STT_TLS		6
 #define STT_LOPROC 13
 #define STT_HIPROC 15
 

headers/private/system/tls.h

@@ -19,6 +19,7 @@ enum {
 	TLS_ERRNO_SLOT,
 	TLS_ON_EXIT_THREAD_SLOT,
 	TLS_USER_THREAD_SLOT,
+	TLS_DYNAMIC_THREAD_VECTOR,
 
 	// Note: these entries can safely be changed between
 	// releases; 3rd party code always calls tls_allocate()

src/system/kernel/arch/x86/arch_thread.cpp

@@ -190,7 +190,7 @@ arch_team_init_team_struct(Team* p, bool kernel)
 status_t
 arch_thread_init_tls(Thread* thread)
 {
-	size_t tls[TLS_USER_THREAD_SLOT + 1];
+	addr_t tls[TLS_FIRST_FREE_SLOT];
 
 	thread->user_local_storage = thread->user_stack_base
 		+ thread->user_stack_size;

src/system/libroot/os/arch/x86/tls.c

@@ -1,4 +1,5 @@
 /*
+ * Copyright 2014, Paweł Dziepak, [email protected].
  * Copyright 2003, Axel Dörfler, [email protected]. All rights reserved.
  * Distributed under the terms of the MIT License.
  */
@@ -9,10 +10,25 @@
 #	define _NO_INLINE_ASM 1
 #endif
 
+#include <runtime_loader/runtime_loader.h>
+
 #include "support/TLS.h"
 #include "tls.h"
 
 
+#if !defined(__GNUC__) || (__GNUC__ > 2)
+
+struct tls_index {
+	unsigned long int	module;
+	unsigned long int	offset;
+};
+
+
+void* ___tls_get_addr(struct tls_index* ti) __attribute__((__regparm__(1)));
+
+#endif	// GCC2
+
+
 static int32 gNextSlot = TLS_FIRST_FREE_SLOT;
 
 
@@ -58,3 +74,16 @@ tls_set(int32 index, void *value)
 		: : "r" (index), "r" (value));
 }
 
+
+#if !defined(__GNUC__) || (__GNUC__ > 2)
+
+
+void* __attribute__((__regparm__(1)))
+___tls_get_addr(struct tls_index* ti)
+{
+	return __gRuntimeLoader->get_tls_address(ti->module, ti->offset);
+}
+
+
+#endif	// GCC2
+

src/system/libroot/os/arch/x86_64/tls.cpp

@@ -8,12 +8,20 @@
 #	define _NO_INLINE_ASM 1
 #endif
 
+#include <runtime_loader/runtime_loader.h>
+
 #include <support/TLS.h>
 #include <tls.h>
 
 #include <assert.h>
 
 
+struct tls_index {
+	unsigned long int	module;
+	unsigned long int	offset;
+};
+
+
 static int32 gNextSlot = TLS_FIRST_FREE_SLOT;
 
 
@@ -63,3 +71,11 @@ tls_set(int32 _index, void* value)
 		"movq	%1, %%fs:(, %0, 8)"
 		: : "r" (index), "r" (value));
 }
+
+
+extern "C" void*
+__tls_get_addr(tls_index* ti)
+{
+	return __gRuntimeLoader->get_tls_address(ti->module, ti->offset);
+}
+

src/system/libroot/os/thread.c

@@ -11,6 +11,7 @@
 
 #include <libroot_private.h>
 #include <pthread_private.h>
+#include <runtime_loader.h>
 #include <thread_defs.h>
 #include <tls.h>
 #include <syscalls.h>
@@ -70,6 +71,8 @@ _thread_do_exit_work(void)
 
 	tls_set(TLS_ON_EXIT_THREAD_SLOT, NULL);
 
+	__gRuntimeLoader->destroy_thread_tls();
+
 	__pthread_destroy_thread();
 }
 

src/system/libroot/posix/tls.cpp

@@ -0,0 +1,4 @@
+/*
+ * Copyright 2014, Paweł Dziepak, [email protected].
+ * Distributed under the terms of the MIT License.
+ */

src/system/runtime_loader/Jamfile

@@ -30,6 +30,7 @@ StaticLibrary libruntime_loader.a :
 	<src!system!libroot!os!$(architecture)>recursive_lock.o
 	<src!system!libroot!os!$(architecture)>syscalls.o
 	<src!system!libroot!os!$(architecture)>sem.o
+	<src!system!libroot!os!arch!$(TARGET_ARCH)!$(architecture)>tls.o
 
 	<src!system!libroot!posix!$(architecture)>errno.o
   	<src!system!libroot!posix!$(architecture)>fcntl.o
@@ -75,6 +76,7 @@ local sources =
 	elf_haiku_version.cpp
 	elf_load_image.cpp
 	elf_symbol_lookup.cpp
+	elf_tls.cpp
 	elf_versioning.cpp
 	pe.cpp
 	errors.cpp

src/system/runtime_loader/arch/x86/arch_relocate.cpp

@@ -38,6 +38,7 @@ relocate_rel(image_t *rootImage, image_t *image, struct Elf32_Rel *rel,
 			case R_386_GLOB_DAT:
 			case R_386_JMP_SLOT:
 			case R_386_GOTOFF:
+			case R_386_TLS_DTPOFF32:
 			{
 				struct Elf32_Sym *sym;
 				status_t status;
@@ -90,6 +91,12 @@ relocate_rel(image_t *rootImage, image_t *image, struct Elf32_Rel *rel,
 				final_val = GOT + A - P;
 				break;
 #endif
+			case R_386_TLS_DTPMOD32:
+				final_val = image->dso_tls_id;
+				break;
+			case R_386_TLS_DTPOFF32:
+				final_val = S;
+				break;
 			default:
 				TRACE(("unhandled relocation type %d\n", ELF32_R_TYPE(rel[i].r_info)));
 				return B_NOT_ALLOWED;

src/system/runtime_loader/arch/x86_64/arch_relocate.cpp

@@ -56,6 +56,13 @@ relocate_rela(image_t* rootImage, image_t* image, Elf64_Rela* rel,
 			case R_X86_64_RELATIVE:
 				relocValue = image->regions[0].delta + rel[i].r_addend;
 				break;
+			case R_X86_64_DTPMOD64:
+				relocValue = image->dso_tls_id;
+				break;
+			case R_X86_64_DTPOFF32:
+			case R_X86_64_DTPOFF64:
+				relocValue = symAddr;
+				break;
 			default:
 				TRACE(("unhandled relocation type %d\n", type));
 				return B_BAD_DATA;

src/system/runtime_loader/elf.cpp

@@ -26,6 +26,7 @@
 #include "add_ons.h"
 #include "elf_load_image.h"
 #include "elf_symbol_lookup.h"
+#include "elf_tls.h"
 #include "elf_versioning.h"
 #include "errors.h"
 #include "images.h"
@@ -635,6 +636,8 @@ unload_library(void* handle, image_id imageID, bool addOn)
 			if (image->term_routine)
 				((init_term_function)image->term_routine)(image->id);
 
+			TLSBlockTemplates::Get().Unregister(image->dso_tls_id);
+
 			dequeue_disposable_image(image);
 			unmap_image(image);
 

src/system/runtime_loader/elf_load_image.cpp

@@ -18,6 +18,7 @@
 #include "add_ons.h"
 #include "elf_haiku_version.h"
 #include "elf_symbol_lookup.h"
+#include "elf_tls.h"
 #include "elf_versioning.h"
 #include "images.h"
 #include "runtime_loader_private.h"
@@ -77,6 +78,9 @@ count_regions(const char* imagePath, char const* buff, int phnum, int phentsize)
 			case PT_STACK:
 				// we don't use it
 				break;
+			case PT_TLS:
+				// will be handled at some other place
+				break;
 			default:
 				FATAL("%s: Unhandled pheader type in count 0x%" B_PRIx32 "\n",
 					imagePath, pheaders->p_type);
@@ -95,6 +99,8 @@ parse_program_headers(image_t* image, char* buff, int phnum, int phentsize)
 	int regcount;
 	int i;
 
+	image->dso_tls_id = unsigned(-1);
+
 	regcount = 0;
 	for (i = 0; i < phnum; i++) {
 		pheader = (elf_phdr*)(buff + i * phentsize);
@@ -193,6 +199,12 @@ parse_program_headers(image_t* image, char* buff, int phnum, int phentsize)
 			case PT_STACK:
 				// we don't use it
 				break;
+			case PT_TLS:
+				image->dso_tls_id
+					= TLSBlockTemplates::Get().Register(
+						TLSBlockTemplate((void*)pheader->p_vaddr,
+							pheader->p_filesz, pheader->p_memsz));
+				break;
 			default:
 				FATAL("%s: Unhandled pheader type in parse 0x%" B_PRIx32 "\n",
 					image->path, pheader->p_type);
@@ -315,6 +327,10 @@ parse_dynamic_segment(image_t* image)
 				uint32 flags = d[i].d_un.d_val;
 				if ((flags & DF_SYMBOLIC) != 0)
 					image->flags |= RFLAG_SYMBOLIC;
+				if ((flags & DF_STATIC_TLS) != 0) {
+					FATAL("Static TLS model is not supported.\n");
+					return false;
+				}
 				break;
 			}
 			default:

src/system/runtime_loader/elf_symbol_lookup.cpp

@@ -514,6 +514,7 @@ resolve_symbol(image_t* rootImage, image_t* image, elf_sym* sym,
 	}
 
 	enum {
+		SUCCESS,
 		ERROR_NO_SYMBOL,
 		ERROR_WRONG_TYPE,
 		ERROR_NOT_EXPORTED,
@@ -521,6 +522,7 @@ resolve_symbol(image_t* rootImage, image_t* image, elf_sym* sym,
 	};
 	uint32 lookupError = ERROR_UNPATCHED;
 
+	bool tlsSymbol = sym->Type() == STT_TLS;
 	void* location = NULL;
 	if (sharedSym == NULL) {
 		// symbol not found at all
@@ -538,14 +540,20 @@ resolve_symbol(image_t* rootImage, image_t* image, elf_sym* sym,
 		sharedImage = NULL;
 	} else {
 		// symbol is fine, get its location
-		location = (void*)(sharedSym->st_value
+		location = (void*)sharedSym->st_value;
-			+ sharedImage->regions[0].delta);
+		if (!tlsSymbol) {
+			location
+				= (void*)((addr_t)location + sharedImage->regions[0].delta);
+		} else
+			lookupError = SUCCESS;
 	}
 
-	patch_undefined_symbol(rootImage, image, symName, &sharedImage, &location,
+	if (!tlsSymbol) {
-		&type);
+		patch_undefined_symbol(rootImage, image, symName, &sharedImage,
+			&location, &type);
+	}
 
-	if (location == NULL) {
+	if (location == NULL && lookupError != SUCCESS) {
 		switch (lookupError) {
 			case ERROR_NO_SYMBOL:
 				FATAL("%s: Could not resolve symbol '%s'\n",

src/system/runtime_loader/elf_tls.cpp

@@ -0,0 +1,304 @@
+/*
+ * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org.
+ * Distributed under the terms of the MIT License.
+ */
+
+#include "elf_tls.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <support/TLS.h>
+
+#include <tls.h>
+
+
+class TLSBlock {
+public:	
+	inline				TLSBlock();
+	inline				TLSBlock(void* pointer);
+
+	inline	status_t	Initialize(unsigned dso);
+
+			void		Destroy();
+
+			bool		IsInvalid() const	{ return fPointer == NULL; }
+
+			void*		operator+(addr_t offset) const
+							{ return (void*)((addr_t)fPointer + offset); }
+
+private:
+			void*		fPointer;
+};
+
+class Generation {
+public:
+	inline				Generation();
+
+			unsigned	Counter() const	{ return fCounter; }
+			unsigned	Size() const	{ return fSize; }
+
+			void		SetCounter(unsigned counter)	{ fCounter = counter; }
+			void		SetSize(unsigned size)	{ fSize = size; }
+
+private:
+			unsigned	fCounter;
+			unsigned	fSize;
+};
+
+class DynamicThreadVector {
+public:
+	inline				DynamicThreadVector();
+
+			void		DestroyAll();
+
+	inline	TLSBlock&	operator[](unsigned dso);
+
+private:
+			bool		_DoesExist() const	{ return *fVector != NULL; }
+			unsigned	_Size() const
+							{ return _DoesExist()
+									? fGeneration->Size() : 0; }
+
+			unsigned	_Generation() const;
+
+			status_t	_ResizeVector(unsigned minimumSize);
+
+			TLSBlock**	fVector;
+			Generation*	fGeneration;
+			TLSBlock	fNullBlock;
+};
+
+
+TLSBlockTemplates*	TLSBlockTemplates::fInstance;
+
+
+void
+TLSBlockTemplate::SetBaseAddress(addr_t baseAddress)
+{
+	fAddress = (void*)((addr_t)fAddress + baseAddress);
+}
+
+
+TLSBlock
+TLSBlockTemplate::CreateBlock()
+{
+	void* pointer = malloc(fMemorySize);
+	if (pointer == NULL)
+		return TLSBlock();
+	memcpy(pointer, fAddress, fFileSize);
+	if (fMemorySize > fFileSize)
+		memset((char*)pointer + fFileSize, 0, fMemorySize - fFileSize);
+	return TLSBlock(pointer);
+}
+
+
+TLSBlockTemplates&
+TLSBlockTemplates::Get()
+{
+	if (fInstance == NULL)
+		fInstance = new TLSBlockTemplates;
+	return *fInstance;
+}
+
+
+unsigned
+TLSBlockTemplates::Register(const TLSBlockTemplate& block)
+{
+	unsigned dso;
+
+	if (!fFreeDSOs.empty()) {
+		dso = fFreeDSOs.back();
+		fFreeDSOs.pop_back();
+		fTemplates[dso] = block;
+	} else {
+		dso = fTemplates.size();
+		fTemplates.push_back(block);
+	}
+
+	fTemplates[dso].SetGeneration(fGeneration++);
+	return dso;
+}
+
+
+void
+TLSBlockTemplates::Unregister(unsigned dso)
+{
+	if (dso == unsigned(-1))
+		return;
+
+	fFreeDSOs.push_back(dso);
+}
+
+
+void
+TLSBlockTemplates::SetBaseAddress(unsigned dso, addr_t baseAddress)
+{
+	if (dso != unsigned(-1))
+		fTemplates[dso].SetBaseAddress(baseAddress);
+}
+
+
+unsigned
+TLSBlockTemplates::GetGeneration(unsigned dso) const
+{
+	if (dso == unsigned(-1))
+		return fGeneration;
+	return fTemplates[dso].Generation();
+}
+
+
+TLSBlock
+TLSBlockTemplates::CreateBlock(unsigned dso)
+{
+	return fTemplates[dso].CreateBlock();
+}
+
+
+TLSBlock::TLSBlock()
+	:
+	fPointer(NULL)
+{
+}
+
+
+TLSBlock::TLSBlock(void* pointer)
+	:
+	fPointer(pointer)
+{
+}
+
+
+status_t
+TLSBlock::Initialize(unsigned dso)
+{
+	fPointer = TLSBlockTemplates::Get().CreateBlock(dso).fPointer;
+	return fPointer != NULL ? B_OK : B_NO_MEMORY;
+}
+
+
+void
+TLSBlock::Destroy()
+{
+	free(fPointer);
+	fPointer = NULL;
+}
+
+
+Generation::Generation()
+	:
+	fCounter(0),
+	fSize(0)
+{
+}
+
+
+DynamicThreadVector::DynamicThreadVector()
+	:
+	fVector((TLSBlock**)tls_address(TLS_DYNAMIC_THREAD_VECTOR)),
+	fGeneration(NULL)
+{
+	if (*fVector != NULL)
+		fGeneration = (Generation*)*(void**)*fVector;
+}
+
+
+void
+DynamicThreadVector::DestroyAll()
+{
+	for (unsigned i = 0; i < _Size(); i++) {
+		TLSBlock& block = (*fVector)[i + 1];
+		if (!block.IsInvalid())
+			block.Destroy();
+	}
+
+	free(*fVector);
+	*fVector = NULL;
+
+	delete fGeneration;
+}
+
+
+TLSBlock&
+DynamicThreadVector::operator[](unsigned dso)
+{
+	unsigned generation = TLSBlockTemplates::Get().GetGeneration(-1);
+	if (_Generation() < generation) {
+		for (unsigned i = 0; i < _Size(); i++) {
+			TLSBlock& block = (*fVector)[i + 1];
+			unsigned dsoGeneration
+				= TLSBlockTemplates::Get().GetGeneration(dso);
+			if (_Generation() < dsoGeneration && dsoGeneration <= generation)
+				block.Destroy();
+		}
+
+		fGeneration->SetCounter(generation);
+	}
+
+	if (_Size() <= dso) {
+		status_t result = _ResizeVector(dso + 1);
+		if (result != B_OK)
+			return fNullBlock;
+	}
+
+	TLSBlock& block = (*fVector)[dso + 1];
+	if (block.IsInvalid()) {
+		status_t result = block.Initialize(dso);
+		if (result != B_OK)
+			return fNullBlock;
+	};
+
+	return block;
+}
+
+
+unsigned
+DynamicThreadVector::_Generation() const
+{
+	if (fGeneration != NULL)
+		return fGeneration->Counter();
+	return unsigned(-1);
+}
+
+
+status_t
+DynamicThreadVector::_ResizeVector(unsigned minimumSize)
+{
+	static const unsigned kInitialSize = 4;
+	unsigned size = std::max(minimumSize, kInitialSize);
+	unsigned oldSize = _Size();
+	if (size <= oldSize)
+		return B_OK;
+
+	void* newVector = realloc(*fVector, (size + 1) * sizeof(TLSBlock));
+	if (newVector == NULL)
+		return B_NO_MEMORY;
+
+	*fVector = (TLSBlock*)newVector;
+	memset(*fVector + oldSize + 1, 0, (size - oldSize) * sizeof(TLSBlock));
+	if (fGeneration == NULL)
+		fGeneration = new Generation;
+	*(Generation**)*fVector = fGeneration;
+	fGeneration->SetSize(size);
+
+	return B_OK;
+}
+
+
+void*
+get_tls_address(unsigned dso, addr_t offset)
+{
+	DynamicThreadVector dynamicThreadVector;
+	TLSBlock& block = dynamicThreadVector[dso];
+	if (block.IsInvalid())
+		return NULL;
+	return block + offset;
+}
+
+
+void
+destroy_thread_tls()
+{
+	DynamicThreadVector().DestroyAll();
+}
+

src/system/runtime_loader/elf_tls.h

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef ELF_TLS_H
+#define ELF_TLS_H
+
+
+#include "runtime_loader_private.h"
+#include "utility.h"
+
+
+class TLSBlock;
+
+class TLSBlockTemplate {
+public:
+						TLSBlockTemplate() { }
+	inline				TLSBlockTemplate(void* address, size_t fileSize,
+							size_t memorySize);
+
+			void		SetGeneration(unsigned generation)
+							{ fGeneration = generation; }
+			unsigned	Generation() const	{ return fGeneration; }
+
+			void		SetBaseAddress(addr_t baseAddress);
+
+			TLSBlock	CreateBlock();
+
+private:
+			unsigned	fGeneration;
+
+			void*		fAddress;
+			size_t		fFileSize;
+			size_t		fMemorySize;
+};
+
+class TLSBlockTemplates {
+public:
+	static	TLSBlockTemplates&	Get();
+
+			unsigned	Register(const TLSBlockTemplate& block);
+			void		Unregister(unsigned dso);
+
+			void		SetBaseAddress(unsigned dso, addr_t baseAddress);
+
+			unsigned	GetGeneration(unsigned dso) const;
+
+			TLSBlock	CreateBlock(unsigned dso);
+
+private:
+						TLSBlockTemplates() { }
+
+	static	TLSBlockTemplates*	fInstance;
+
+			unsigned	fGeneration;
+
+			utility::vector<TLSBlockTemplate>	fTemplates;
+			utility::vector<unsigned>	fFreeDSOs;
+};
+
+
+TLSBlockTemplate::TLSBlockTemplate(void* address, size_t fileSize,
+	size_t memorySize)
+	:
+	fAddress(address),
+	fFileSize(fileSize),
+	fMemorySize(memorySize)
+{
+}
+
+
+void*	get_tls_address(unsigned dso, addr_t offset);
+void	destroy_thread_tls();
+
+
+#endif	// ELF_TLS_H

src/system/runtime_loader/export.cpp

@@ -8,6 +8,7 @@
 
 
 #include "runtime_loader_private.h"
+#include "elf_tls.h"
 
 
 // exported via the rld_export structure in user space program arguments
@@ -55,6 +56,8 @@ struct rld_export gRuntimeLoader = {
 	test_executable,
 	get_executable_architecture,
 	get_next_image_dependency,
+	get_tls_address,
+	destroy_thread_tls,
 
 	elf_reinit_after_fork,
 	NULL, // call_atexit_hooks_for_range
@@ -66,6 +69,8 @@ struct rld_export gRuntimeLoader = {
 	0		// ABI version
 };
 
+rld_export* __gRuntimeLoader = &gRuntimeLoader;
+
 
 void
 rldexport_init(void)

src/system/runtime_loader/images.cpp

@@ -20,6 +20,7 @@
 #include <vm_defs.h>
 
 #include "add_ons.h"
+#include "elf_tls.h"
 #include "runtime_loader_private.h"
 
 
@@ -396,6 +397,10 @@ map_image(int fd, char const* path, image_t* image)
 
 		image->regions[i].delta = loadAddress - image->regions[i].vmstart;
 		image->regions[i].vmstart = loadAddress;
+		if (i == 0) {
+			TLSBlockTemplates::Get().SetBaseAddress(image->dso_tls_id,
+				loadAddress);
+		}
 	}
 
 	if (image->dynamic_ptr != 0)

src/system/runtime_loader/runtime_loader_private.h

@@ -50,9 +50,7 @@ extern bool gProgramLoaded;
 extern image_t* gProgramImage;
 
 
-#ifdef __cplusplus
 extern "C" {
-#endif
 
 int runtime_loader(void* arg, void* commpage);
 int open_executable(char* name, image_type type, const char* rpath,
@@ -93,8 +91,6 @@ status_t heap_init(void);
 status_t arch_relocate_image(image_t* rootImage, image_t* image,
 	SymbolLookupCache* cache);
 
-#ifdef __cplusplus
 }
-#endif
 
 #endif	/* RUNTIME_LOADER_PRIVATE_H */