diff --git a/headers/private/runtime_loader/runtime_loader.h b/headers/private/runtime_loader/runtime_loader.h index 3772a09d03..6fa83bbc9d 100644 --- a/headers/private/runtime_loader/runtime_loader.h +++ b/headers/private/runtime_loader/runtime_loader.h @@ -45,6 +45,8 @@ struct rld_export { const char** _architecture); status_t (*get_next_image_dependency)(image_id id, uint32 *cookie, const char **_name); + void* (*get_tls_address)(unsigned dso, addr_t offset); + void (*destroy_thread_tls)(); status_t (*reinit_after_fork)(); @@ -114,6 +116,8 @@ typedef struct image_t { elf_rel *pltrel; int pltrel_len; + unsigned dso_tls_id; + uint32 num_needed; struct image_t **needed; diff --git a/headers/private/system/elf_common.h b/headers/private/system/elf_common.h index 73a7743eff..fbd202fe6b 100644 --- a/headers/private/system/elf_common.h +++ b/headers/private/system/elf_common.h @@ -158,6 +158,7 @@ #define STT_FUNC 2 #define STT_SECTION 3 #define STT_FILE 4 +#define STT_TLS 6 #define STT_LOPROC 13 #define STT_HIPROC 15 diff --git a/headers/private/system/tls.h b/headers/private/system/tls.h index f2105da1fb..751332e33f 100644 --- a/headers/private/system/tls.h +++ b/headers/private/system/tls.h @@ -19,6 +19,7 @@ enum { TLS_ERRNO_SLOT, TLS_ON_EXIT_THREAD_SLOT, TLS_USER_THREAD_SLOT, + TLS_DYNAMIC_THREAD_VECTOR, // Note: these entries can safely be changed between // releases; 3rd party code always calls tls_allocate() diff --git a/src/system/kernel/arch/x86/arch_thread.cpp b/src/system/kernel/arch/x86/arch_thread.cpp index 11031b5264..25cde443ed 100644 --- a/src/system/kernel/arch/x86/arch_thread.cpp +++ b/src/system/kernel/arch/x86/arch_thread.cpp @@ -190,7 +190,7 @@ arch_team_init_team_struct(Team* p, bool kernel) status_t arch_thread_init_tls(Thread* thread) { - size_t tls[TLS_USER_THREAD_SLOT + 1]; + addr_t tls[TLS_FIRST_FREE_SLOT]; thread->user_local_storage = thread->user_stack_base + thread->user_stack_size; diff --git a/src/system/libroot/os/arch/x86/tls.c b/src/system/libroot/os/arch/x86/tls.c index be36829640..e1ec0ad096 100644 --- a/src/system/libroot/os/arch/x86/tls.c +++ b/src/system/libroot/os/arch/x86/tls.c @@ -1,4 +1,5 @@ /* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2003, Axel Dörfler, axeld@pinc-software.de. All rights reserved. * Distributed under the terms of the MIT License. */ @@ -9,10 +10,25 @@ # define _NO_INLINE_ASM 1 #endif +#include + #include "support/TLS.h" #include "tls.h" +#if !defined(__GNUC__) || (__GNUC__ > 2) + +struct tls_index { + unsigned long int module; + unsigned long int offset; +}; + + +void* ___tls_get_addr(struct tls_index* ti) __attribute__((__regparm__(1))); + +#endif // GCC2 + + static int32 gNextSlot = TLS_FIRST_FREE_SLOT; @@ -58,3 +74,16 @@ tls_set(int32 index, void *value) : : "r" (index), "r" (value)); } + +#if !defined(__GNUC__) || (__GNUC__ > 2) + + +void* __attribute__((__regparm__(1))) +___tls_get_addr(struct tls_index* ti) +{ + return __gRuntimeLoader->get_tls_address(ti->module, ti->offset); +} + + +#endif // GCC2 + diff --git a/src/system/libroot/os/arch/x86_64/tls.cpp b/src/system/libroot/os/arch/x86_64/tls.cpp index 0cab26d198..15925c282f 100644 --- a/src/system/libroot/os/arch/x86_64/tls.cpp +++ b/src/system/libroot/os/arch/x86_64/tls.cpp @@ -8,12 +8,20 @@ # define _NO_INLINE_ASM 1 #endif +#include + #include #include #include +struct tls_index { + unsigned long int module; + unsigned long int offset; +}; + + static int32 gNextSlot = TLS_FIRST_FREE_SLOT; @@ -63,3 +71,11 @@ tls_set(int32 _index, void* value) "movq %1, %%fs:(, %0, 8)" : : "r" (index), "r" (value)); } + + +extern "C" void* +__tls_get_addr(tls_index* ti) +{ + return __gRuntimeLoader->get_tls_address(ti->module, ti->offset); +} + diff --git a/src/system/libroot/os/thread.c b/src/system/libroot/os/thread.c index 0f14f29d9d..41e105e4c5 100644 --- a/src/system/libroot/os/thread.c +++ b/src/system/libroot/os/thread.c @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -70,6 +71,8 @@ _thread_do_exit_work(void) tls_set(TLS_ON_EXIT_THREAD_SLOT, NULL); + __gRuntimeLoader->destroy_thread_tls(); + __pthread_destroy_thread(); } diff --git a/src/system/libroot/os/tls.cpp b/src/system/libroot/os/tls.cpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/system/libroot/posix/tls.cpp b/src/system/libroot/posix/tls.cpp new file mode 100644 index 0000000000..08d7f38ee6 --- /dev/null +++ b/src/system/libroot/posix/tls.cpp @@ -0,0 +1,4 @@ +/* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ diff --git a/src/system/runtime_loader/Jamfile b/src/system/runtime_loader/Jamfile index 3b4a802993..da25b1f0be 100644 --- a/src/system/runtime_loader/Jamfile +++ b/src/system/runtime_loader/Jamfile @@ -30,6 +30,7 @@ StaticLibrary libruntime_loader.a : recursive_lock.o syscalls.o sem.o + tls.o errno.o fcntl.o @@ -75,6 +76,7 @@ local sources = elf_haiku_version.cpp elf_load_image.cpp elf_symbol_lookup.cpp + elf_tls.cpp elf_versioning.cpp pe.cpp errors.cpp diff --git a/src/system/runtime_loader/arch/x86/arch_relocate.cpp b/src/system/runtime_loader/arch/x86/arch_relocate.cpp index 4e1f041612..e0090d02b6 100644 --- a/src/system/runtime_loader/arch/x86/arch_relocate.cpp +++ b/src/system/runtime_loader/arch/x86/arch_relocate.cpp @@ -38,6 +38,7 @@ relocate_rel(image_t *rootImage, image_t *image, struct Elf32_Rel *rel, case R_386_GLOB_DAT: case R_386_JMP_SLOT: case R_386_GOTOFF: + case R_386_TLS_DTPOFF32: { struct Elf32_Sym *sym; status_t status; @@ -90,6 +91,12 @@ relocate_rel(image_t *rootImage, image_t *image, struct Elf32_Rel *rel, final_val = GOT + A - P; break; #endif + case R_386_TLS_DTPMOD32: + final_val = image->dso_tls_id; + break; + case R_386_TLS_DTPOFF32: + final_val = S; + break; default: TRACE(("unhandled relocation type %d\n", ELF32_R_TYPE(rel[i].r_info))); return B_NOT_ALLOWED; diff --git a/src/system/runtime_loader/arch/x86_64/arch_relocate.cpp b/src/system/runtime_loader/arch/x86_64/arch_relocate.cpp index 7148c2e23f..b681a2164a 100644 --- a/src/system/runtime_loader/arch/x86_64/arch_relocate.cpp +++ b/src/system/runtime_loader/arch/x86_64/arch_relocate.cpp @@ -56,6 +56,13 @@ relocate_rela(image_t* rootImage, image_t* image, Elf64_Rela* rel, case R_X86_64_RELATIVE: relocValue = image->regions[0].delta + rel[i].r_addend; break; + case R_X86_64_DTPMOD64: + relocValue = image->dso_tls_id; + break; + case R_X86_64_DTPOFF32: + case R_X86_64_DTPOFF64: + relocValue = symAddr; + break; default: TRACE(("unhandled relocation type %d\n", type)); return B_BAD_DATA; diff --git a/src/system/runtime_loader/elf.cpp b/src/system/runtime_loader/elf.cpp index e7d751426a..37ccc5e239 100644 --- a/src/system/runtime_loader/elf.cpp +++ b/src/system/runtime_loader/elf.cpp @@ -26,6 +26,7 @@ #include "add_ons.h" #include "elf_load_image.h" #include "elf_symbol_lookup.h" +#include "elf_tls.h" #include "elf_versioning.h" #include "errors.h" #include "images.h" @@ -635,6 +636,8 @@ unload_library(void* handle, image_id imageID, bool addOn) if (image->term_routine) ((init_term_function)image->term_routine)(image->id); + TLSBlockTemplates::Get().Unregister(image->dso_tls_id); + dequeue_disposable_image(image); unmap_image(image); diff --git a/src/system/runtime_loader/elf_load_image.cpp b/src/system/runtime_loader/elf_load_image.cpp index f1c17b4c5d..8b698e98a2 100644 --- a/src/system/runtime_loader/elf_load_image.cpp +++ b/src/system/runtime_loader/elf_load_image.cpp @@ -18,6 +18,7 @@ #include "add_ons.h" #include "elf_haiku_version.h" #include "elf_symbol_lookup.h" +#include "elf_tls.h" #include "elf_versioning.h" #include "images.h" #include "runtime_loader_private.h" @@ -77,6 +78,9 @@ count_regions(const char* imagePath, char const* buff, int phnum, int phentsize) case PT_STACK: // we don't use it break; + case PT_TLS: + // will be handled at some other place + break; default: FATAL("%s: Unhandled pheader type in count 0x%" B_PRIx32 "\n", imagePath, pheaders->p_type); @@ -95,6 +99,8 @@ parse_program_headers(image_t* image, char* buff, int phnum, int phentsize) int regcount; int i; + image->dso_tls_id = unsigned(-1); + regcount = 0; for (i = 0; i < phnum; i++) { pheader = (elf_phdr*)(buff + i * phentsize); @@ -193,6 +199,12 @@ parse_program_headers(image_t* image, char* buff, int phnum, int phentsize) case PT_STACK: // we don't use it break; + case PT_TLS: + image->dso_tls_id + = TLSBlockTemplates::Get().Register( + TLSBlockTemplate((void*)pheader->p_vaddr, + pheader->p_filesz, pheader->p_memsz)); + break; default: FATAL("%s: Unhandled pheader type in parse 0x%" B_PRIx32 "\n", image->path, pheader->p_type); @@ -315,6 +327,10 @@ parse_dynamic_segment(image_t* image) uint32 flags = d[i].d_un.d_val; if ((flags & DF_SYMBOLIC) != 0) image->flags |= RFLAG_SYMBOLIC; + if ((flags & DF_STATIC_TLS) != 0) { + FATAL("Static TLS model is not supported.\n"); + return false; + } break; } default: diff --git a/src/system/runtime_loader/elf_symbol_lookup.cpp b/src/system/runtime_loader/elf_symbol_lookup.cpp index 1983197dc2..1a8ea17804 100644 --- a/src/system/runtime_loader/elf_symbol_lookup.cpp +++ b/src/system/runtime_loader/elf_symbol_lookup.cpp @@ -514,6 +514,7 @@ resolve_symbol(image_t* rootImage, image_t* image, elf_sym* sym, } enum { + SUCCESS, ERROR_NO_SYMBOL, ERROR_WRONG_TYPE, ERROR_NOT_EXPORTED, @@ -521,6 +522,7 @@ resolve_symbol(image_t* rootImage, image_t* image, elf_sym* sym, }; uint32 lookupError = ERROR_UNPATCHED; + bool tlsSymbol = sym->Type() == STT_TLS; void* location = NULL; if (sharedSym == NULL) { // symbol not found at all @@ -538,14 +540,20 @@ resolve_symbol(image_t* rootImage, image_t* image, elf_sym* sym, sharedImage = NULL; } else { // symbol is fine, get its location - location = (void*)(sharedSym->st_value - + sharedImage->regions[0].delta); + location = (void*)sharedSym->st_value; + if (!tlsSymbol) { + location + = (void*)((addr_t)location + sharedImage->regions[0].delta); + } else + lookupError = SUCCESS; } - patch_undefined_symbol(rootImage, image, symName, &sharedImage, &location, - &type); + if (!tlsSymbol) { + patch_undefined_symbol(rootImage, image, symName, &sharedImage, + &location, &type); + } - if (location == NULL) { + if (location == NULL && lookupError != SUCCESS) { switch (lookupError) { case ERROR_NO_SYMBOL: FATAL("%s: Could not resolve symbol '%s'\n", diff --git a/src/system/runtime_loader/elf_tls.cpp b/src/system/runtime_loader/elf_tls.cpp new file mode 100644 index 0000000000..3a17f731b4 --- /dev/null +++ b/src/system/runtime_loader/elf_tls.cpp @@ -0,0 +1,304 @@ +/* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ + +#include "elf_tls.h" + +#include +#include + +#include + +#include + + +class TLSBlock { +public: + inline TLSBlock(); + inline TLSBlock(void* pointer); + + inline status_t Initialize(unsigned dso); + + void Destroy(); + + bool IsInvalid() const { return fPointer == NULL; } + + void* operator+(addr_t offset) const + { return (void*)((addr_t)fPointer + offset); } + +private: + void* fPointer; +}; + +class Generation { +public: + inline Generation(); + + unsigned Counter() const { return fCounter; } + unsigned Size() const { return fSize; } + + void SetCounter(unsigned counter) { fCounter = counter; } + void SetSize(unsigned size) { fSize = size; } + +private: + unsigned fCounter; + unsigned fSize; +}; + +class DynamicThreadVector { +public: + inline DynamicThreadVector(); + + void DestroyAll(); + + inline TLSBlock& operator[](unsigned dso); + +private: + bool _DoesExist() const { return *fVector != NULL; } + unsigned _Size() const + { return _DoesExist() + ? fGeneration->Size() : 0; } + + unsigned _Generation() const; + + status_t _ResizeVector(unsigned minimumSize); + + TLSBlock** fVector; + Generation* fGeneration; + TLSBlock fNullBlock; +}; + + +TLSBlockTemplates* TLSBlockTemplates::fInstance; + + +void +TLSBlockTemplate::SetBaseAddress(addr_t baseAddress) +{ + fAddress = (void*)((addr_t)fAddress + baseAddress); +} + + +TLSBlock +TLSBlockTemplate::CreateBlock() +{ + void* pointer = malloc(fMemorySize); + if (pointer == NULL) + return TLSBlock(); + memcpy(pointer, fAddress, fFileSize); + if (fMemorySize > fFileSize) + memset((char*)pointer + fFileSize, 0, fMemorySize - fFileSize); + return TLSBlock(pointer); +} + + +TLSBlockTemplates& +TLSBlockTemplates::Get() +{ + if (fInstance == NULL) + fInstance = new TLSBlockTemplates; + return *fInstance; +} + + +unsigned +TLSBlockTemplates::Register(const TLSBlockTemplate& block) +{ + unsigned dso; + + if (!fFreeDSOs.empty()) { + dso = fFreeDSOs.back(); + fFreeDSOs.pop_back(); + fTemplates[dso] = block; + } else { + dso = fTemplates.size(); + fTemplates.push_back(block); + } + + fTemplates[dso].SetGeneration(fGeneration++); + return dso; +} + + +void +TLSBlockTemplates::Unregister(unsigned dso) +{ + if (dso == unsigned(-1)) + return; + + fFreeDSOs.push_back(dso); +} + + +void +TLSBlockTemplates::SetBaseAddress(unsigned dso, addr_t baseAddress) +{ + if (dso != unsigned(-1)) + fTemplates[dso].SetBaseAddress(baseAddress); +} + + +unsigned +TLSBlockTemplates::GetGeneration(unsigned dso) const +{ + if (dso == unsigned(-1)) + return fGeneration; + return fTemplates[dso].Generation(); +} + + +TLSBlock +TLSBlockTemplates::CreateBlock(unsigned dso) +{ + return fTemplates[dso].CreateBlock(); +} + + +TLSBlock::TLSBlock() + : + fPointer(NULL) +{ +} + + +TLSBlock::TLSBlock(void* pointer) + : + fPointer(pointer) +{ +} + + +status_t +TLSBlock::Initialize(unsigned dso) +{ + fPointer = TLSBlockTemplates::Get().CreateBlock(dso).fPointer; + return fPointer != NULL ? B_OK : B_NO_MEMORY; +} + + +void +TLSBlock::Destroy() +{ + free(fPointer); + fPointer = NULL; +} + + +Generation::Generation() + : + fCounter(0), + fSize(0) +{ +} + + +DynamicThreadVector::DynamicThreadVector() + : + fVector((TLSBlock**)tls_address(TLS_DYNAMIC_THREAD_VECTOR)), + fGeneration(NULL) +{ + if (*fVector != NULL) + fGeneration = (Generation*)*(void**)*fVector; +} + + +void +DynamicThreadVector::DestroyAll() +{ + for (unsigned i = 0; i < _Size(); i++) { + TLSBlock& block = (*fVector)[i + 1]; + if (!block.IsInvalid()) + block.Destroy(); + } + + free(*fVector); + *fVector = NULL; + + delete fGeneration; +} + + +TLSBlock& +DynamicThreadVector::operator[](unsigned dso) +{ + unsigned generation = TLSBlockTemplates::Get().GetGeneration(-1); + if (_Generation() < generation) { + for (unsigned i = 0; i < _Size(); i++) { + TLSBlock& block = (*fVector)[i + 1]; + unsigned dsoGeneration + = TLSBlockTemplates::Get().GetGeneration(dso); + if (_Generation() < dsoGeneration && dsoGeneration <= generation) + block.Destroy(); + } + + fGeneration->SetCounter(generation); + } + + if (_Size() <= dso) { + status_t result = _ResizeVector(dso + 1); + if (result != B_OK) + return fNullBlock; + } + + TLSBlock& block = (*fVector)[dso + 1]; + if (block.IsInvalid()) { + status_t result = block.Initialize(dso); + if (result != B_OK) + return fNullBlock; + }; + + return block; +} + + +unsigned +DynamicThreadVector::_Generation() const +{ + if (fGeneration != NULL) + return fGeneration->Counter(); + return unsigned(-1); +} + + +status_t +DynamicThreadVector::_ResizeVector(unsigned minimumSize) +{ + static const unsigned kInitialSize = 4; + unsigned size = std::max(minimumSize, kInitialSize); + unsigned oldSize = _Size(); + if (size <= oldSize) + return B_OK; + + void* newVector = realloc(*fVector, (size + 1) * sizeof(TLSBlock)); + if (newVector == NULL) + return B_NO_MEMORY; + + *fVector = (TLSBlock*)newVector; + memset(*fVector + oldSize + 1, 0, (size - oldSize) * sizeof(TLSBlock)); + if (fGeneration == NULL) + fGeneration = new Generation; + *(Generation**)*fVector = fGeneration; + fGeneration->SetSize(size); + + return B_OK; +} + + +void* +get_tls_address(unsigned dso, addr_t offset) +{ + DynamicThreadVector dynamicThreadVector; + TLSBlock& block = dynamicThreadVector[dso]; + if (block.IsInvalid()) + return NULL; + return block + offset; +} + + +void +destroy_thread_tls() +{ + DynamicThreadVector().DestroyAll(); +} + diff --git a/src/system/runtime_loader/elf_tls.h b/src/system/runtime_loader/elf_tls.h new file mode 100644 index 0000000000..0fa7a07497 --- /dev/null +++ b/src/system/runtime_loader/elf_tls.h @@ -0,0 +1,76 @@ +/* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef ELF_TLS_H +#define ELF_TLS_H + + +#include "runtime_loader_private.h" +#include "utility.h" + + +class TLSBlock; + +class TLSBlockTemplate { +public: + TLSBlockTemplate() { } + inline TLSBlockTemplate(void* address, size_t fileSize, + size_t memorySize); + + void SetGeneration(unsigned generation) + { fGeneration = generation; } + unsigned Generation() const { return fGeneration; } + + void SetBaseAddress(addr_t baseAddress); + + TLSBlock CreateBlock(); + +private: + unsigned fGeneration; + + void* fAddress; + size_t fFileSize; + size_t fMemorySize; +}; + +class TLSBlockTemplates { +public: + static TLSBlockTemplates& Get(); + + unsigned Register(const TLSBlockTemplate& block); + void Unregister(unsigned dso); + + void SetBaseAddress(unsigned dso, addr_t baseAddress); + + unsigned GetGeneration(unsigned dso) const; + + TLSBlock CreateBlock(unsigned dso); + +private: + TLSBlockTemplates() { } + + static TLSBlockTemplates* fInstance; + + unsigned fGeneration; + + utility::vector fTemplates; + utility::vector fFreeDSOs; +}; + + +TLSBlockTemplate::TLSBlockTemplate(void* address, size_t fileSize, + size_t memorySize) + : + fAddress(address), + fFileSize(fileSize), + fMemorySize(memorySize) +{ +} + + +void* get_tls_address(unsigned dso, addr_t offset); +void destroy_thread_tls(); + + +#endif // ELF_TLS_H diff --git a/src/system/runtime_loader/export.cpp b/src/system/runtime_loader/export.cpp index bae3006c47..a480210cbd 100644 --- a/src/system/runtime_loader/export.cpp +++ b/src/system/runtime_loader/export.cpp @@ -8,6 +8,7 @@ #include "runtime_loader_private.h" +#include "elf_tls.h" // exported via the rld_export structure in user space program arguments @@ -55,6 +56,8 @@ struct rld_export gRuntimeLoader = { test_executable, get_executable_architecture, get_next_image_dependency, + get_tls_address, + destroy_thread_tls, elf_reinit_after_fork, NULL, // call_atexit_hooks_for_range @@ -66,6 +69,8 @@ struct rld_export gRuntimeLoader = { 0 // ABI version }; +rld_export* __gRuntimeLoader = &gRuntimeLoader; + void rldexport_init(void) diff --git a/src/system/runtime_loader/images.cpp b/src/system/runtime_loader/images.cpp index 992dc15975..15f39d280a 100644 --- a/src/system/runtime_loader/images.cpp +++ b/src/system/runtime_loader/images.cpp @@ -20,6 +20,7 @@ #include #include "add_ons.h" +#include "elf_tls.h" #include "runtime_loader_private.h" @@ -396,6 +397,10 @@ map_image(int fd, char const* path, image_t* image) image->regions[i].delta = loadAddress - image->regions[i].vmstart; image->regions[i].vmstart = loadAddress; + if (i == 0) { + TLSBlockTemplates::Get().SetBaseAddress(image->dso_tls_id, + loadAddress); + } } if (image->dynamic_ptr != 0) diff --git a/src/system/runtime_loader/runtime_loader_private.h b/src/system/runtime_loader/runtime_loader_private.h index 036a46566e..d00eb5f4cd 100644 --- a/src/system/runtime_loader/runtime_loader_private.h +++ b/src/system/runtime_loader/runtime_loader_private.h @@ -50,9 +50,7 @@ extern bool gProgramLoaded; extern image_t* gProgramImage; -#ifdef __cplusplus extern "C" { -#endif int runtime_loader(void* arg, void* commpage); int open_executable(char* name, image_type type, const char* rpath, @@ -93,8 +91,6 @@ status_t heap_init(void); status_t arch_relocate_image(image_t* rootImage, image_t* image, SymbolLookupCache* cache); -#ifdef __cplusplus } -#endif #endif /* RUNTIME_LOADER_PRIVATE_H */