diff --git a/headers/private/system/tls.h b/headers/private/system/tls.h index e87ac4698b..7d828dc306 100644 --- a/headers/private/system/tls.h +++ b/headers/private/system/tls.h @@ -21,6 +21,7 @@ enum { TLS_ON_EXIT_THREAD_SLOT, TLS_USER_THREAD_SLOT, TLS_DYNAMIC_THREAD_VECTOR, + TLS_MALLOC_SLOT, TLS_LOCALE_SLOT, // Note: these entries can safely be changed between diff --git a/src/system/libroot/posix/malloc/openbsd/Jamfile b/src/system/libroot/posix/malloc/openbsd/Jamfile new file mode 100644 index 0000000000..a28446098d --- /dev/null +++ b/src/system/libroot/posix/malloc/openbsd/Jamfile @@ -0,0 +1,18 @@ +SubDir HAIKU_TOP src system libroot posix malloc openbsd ; + +UsePrivateHeaders kernel libroot shared ; +UseHeaders [ FDirName $(HAIKU_TOP) headers compatibility bsd ] : true ; + +local architectureObject ; +for architectureObject in [ MultiArchSubDirSetup ] { + on $(architectureObject) { + local architecture = $(TARGET_PACKAGING_ARCH) ; + + UsePrivateSystemHeaders ; + + MergeObject <$(architecture)>posix_malloc.o : + PagesAllocator.cpp + malloc.c + ; + } +} diff --git a/src/system/libroot/posix/malloc/openbsd/PagesAllocator.cpp b/src/system/libroot/posix/malloc/openbsd/PagesAllocator.cpp new file mode 100644 index 0000000000..fee394e531 --- /dev/null +++ b/src/system/libroot/posix/malloc/openbsd/PagesAllocator.cpp @@ -0,0 +1,551 @@ +/* + * Copyright 2025, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Augustin Cavalier + */ + +#include "PagesAllocator.h" + +#include +#include +#include +#include + +#include +#include +#include + + +/*! The local page size. Must be a multiple of the system page size. */ +static const size_t kPageSize = B_PAGE_SIZE; + +/*! The "largest useful" chunk size: any allocations larger than this + * will get their own area, rather than sharing the common one(s). */ +static const size_t kLargestUsefulChunk = 512 * kPageSize; + +/*! Amount of virtual address space to reserve when creating new areas. */ +static const size_t kReserveAddressSpace = 128 * 1024 * 1024; + +/*! Cache up to this many percentage points of free memory (compared to used.) */ +static const size_t kFreePercentage = 25; + +/*! Always allow this much free memory, even if it's above kFreePercentage. */ +static const size_t kFreeMinimum = 128 * kPageSize; + + +namespace { + +class PagesAllocator { + struct FreeChunk; + +public: + PagesAllocator() + { + mutex_init(&fLock, "PagesAllocator lock"); + fUsed = fFree = 0; + fLastArea = -1; + } + + ~PagesAllocator() + { + } + + void BeforeFork() + { + mutex_lock(&fLock); + } + + void AfterFork(bool parent) + { + if (parent) { + mutex_unlock(&fLock); + } else { + if (fLastArea >= 0) + fLastArea = area_for((void*)(fLastAreaTop - 1)); + + mutex_init(&fLock, "PagesAllocator lock"); + } + } + + status_t AllocatePages(void*& address, size_t allocate) + { + MutexLocker locker(fLock); + + if (allocate > kLargestUsefulChunk) { + // Create an area just for this allocation. + locker.Unlock(); + area_id area = create_area("heap large allocation", + &address, B_ANY_ADDRESS, allocate, + B_NO_LOCK, B_READ_AREA | B_WRITE_AREA); + if (area >= 0) { + locker.Lock(); + fUsed += allocate; + return B_OK; + } + return area; + } + + if (fFree >= allocate) { + // Try to use memory from the cache. + FreeChunk* chunk = fChunksBySizeTree.FindClosest(allocate, true, true); + if (chunk != NULL) { + address = _Use(chunk, allocate); + return B_OK; + } + } + + // Not enough memory in the cache. Allocate some more. + FreeChunk* chunk; + status_t status = _Map(allocate, chunk); + if (status != B_OK) + return status; + + address = _Use(chunk, allocate); + return B_OK; + } + + status_t AllocatePagesAt(void* _address, size_t allocate) + { + const addr_t address = (addr_t)_address; + MutexLocker locker(fLock); + + if (allocate <= kLargestUsefulChunk) { + FreeChunk* chunk = fChunksByAddressTree.FindClosest(address, false, true); + if (chunk != NULL && chunk->NextAddress() > address) { + // The address is in a free chunk. + size_t remainingAfter = chunk->size - (address - (addr_t)chunk); + if (remainingAfter < allocate) { + if (chunk->NextAddress() != fLastAreaTop) + return B_NO_MEMORY; + + size_t add = allocate - remainingAfter; + status_t status = _ResizeLastArea(add); + if (status != B_OK) + return status; + + chunk = _Insert((void*)chunk->NextAddress(), add); + } + + // Cut the beginning? + if (address == (addr_t)chunk) { + _Use(chunk, allocate); + return B_OK; + } + + // Why would anyone want to allocate such a specific address, anyway? + debugger("PagesAllocator: middle-chunk allocation not implemented!"); + return B_ERROR; + } + } + + if (address == fLastAreaTop) { + if (allocate > kLargestUsefulChunk) + return B_NO_MEMORY; + + status_t status = _ResizeLastArea(allocate); + if (status == B_OK) { + fUsed += allocate; + return B_OK; + } + return status; + } + if (address >= (fLastAreaTop - fLastAreaSize) && address < fLastAreaTop) { + // We didn't find a matching free chunk, so that isn't going to work. + return EEXIST; + } + + locker.Unlock(); + + // One last try: see if we can resize the area this belongs to. + area_info info; + info.area = area_for((void*)(address - 1)); + if (info.area < 0) + return info.area; + + status_t status = get_area_info(info.area, &info); + if (status != B_OK) + return status; + + if (((addr_t)info.address + info.size) != address) + return B_NO_MEMORY; + + status = resize_area(info.area, info.size + allocate); + if (status == B_OK) { + locker.Lock(); + fUsed += allocate; + return B_OK; + } + + // TODO: We could add a "resize allowing move" feature to resize_area, + // which would avoid having to memcpy() the data of large allocations. + return status; + } + + status_t FreePages(void* _address, size_t size) + { + MutexLocker locker(fLock); + + if (size > fUsed) + debugger("PagesAllocator: request to free more than allocated"); + fUsed -= size; + + FreeChunk* chunk = _Insert(_address, size); + + if (size > kLargestUsefulChunk) { + // TODO: This doesn't deal with a free of a smaller number of pages + // within a "large allocation" area. We should probably free those + // immediately also, or at least mark them specially in the trees + // (so they don't get reused for anything but the large allocation.) + + locker.Detach(); + return _UnmapLocked(chunk); + } + + if (fFree <= _FreeLimit()) { + // TODO: If not decommitting/unmapping, we might free the pages + // using MADV_FREE (perhaps on medium-sized chunks with others + // of equal sizes already present in the tree, at least.) + return B_OK; + } + + while (fFree > _FreeLimit()) { + FreeChunk* chunk = fChunksBySizeTree.FindMax(); + status_t status = _UnmapLocked(chunk); + if (status != B_OK) + return status; + mutex_lock(&fLock); + } + + if (fFree > kLargestUsefulChunk && fFree > (_FreeLimit() / 2) + && fChunksBySizeTree.FindMax()->size == kPageSize) { + // All the free chunks are single pages, and there's more of them + // than a "largest useful" chunk. Just evict them all at this point. + while (FreeChunk* chunk = fChunksBySizeTree.FindMin()) { + if (chunk->size != kPageSize) + break; + + status_t status = _UnmapLocked(chunk); + if (status != B_OK) + return status; + mutex_lock(&fLock); + } + } + + return B_OK; + } + +private: + void* _Use(FreeChunk* chunk, size_t amount) + { + fChunksBySizeTree.Remove(chunk); + fChunksByAddressTree.Remove(chunk); + + if (chunk->size == amount) { + // The whole chunk will be used. + // Nothing special to do in this case. + } else { + // Some will be left over. + // Break the remainder off and reinsert into the trees. + FreeChunk* newChunk = (FreeChunk*)((addr_t)chunk + amount); + newChunk->size = (chunk->size - amount); + + fChunksBySizeTree.Insert(newChunk); + fChunksByAddressTree.Insert(newChunk); + } + + fUsed += amount; + fFree -= amount; + return (void*)chunk; + } + + FreeChunk* _Insert(void* _address, size_t size) + { + fFree += size; + + const addr_t address = (addr_t)_address; + FreeChunk* chunk; + + FreeChunk* preceding = fChunksByAddressTree.FindClosest(address, false, false); + if (preceding != NULL && preceding->NextAddress() == address) { + fChunksBySizeTree.Remove(preceding); + chunk = preceding; + chunk->size += size; + } else { + chunk = (FreeChunk*)_address; + chunk->size = size; + fChunksByAddressTree.Insert(chunk); + } + + FreeChunk* following = chunk->address_tree_list_link; + if (following != NULL && chunk->NextAddress() == (addr_t)following) { + fChunksBySizeTree.Remove(following); + fChunksByAddressTree.Remove(following); + chunk->size += following->size; + } + fChunksBySizeTree.Insert(chunk); + + return chunk; + } + + size_t _FreeLimit() const + { + size_t freeLimit = ((fUsed * kFreePercentage) / 100); + if (freeLimit < kFreeMinimum) + freeLimit = kFreeMinimum; + else + freeLimit = (freeLimit + (kPageSize - 1)) & ~(kPageSize - 1); + return freeLimit; + } + +private: + status_t _Map(size_t allocate, FreeChunk*& allocated) + { + if (fLastArea >= 0) { + addr_t oldTop = fLastAreaTop; + status_t status = _ResizeLastArea(allocate); + if (status == B_OK) { + allocated = _Insert((void*)oldTop, allocate); + return B_OK; + } + } + + // Create a new area. + // TODO: We could use an inner lock here to avoid contention. + addr_t newAreaBase; + status_t status = _kern_reserve_address_range(&newAreaBase, + B_RANDOMIZED_ANY_ADDRESS, kReserveAddressSpace); + size_t newReservation = (status == B_OK) ? kReserveAddressSpace : 0; + + status = create_area("heap area", (void**)&newAreaBase, + (status == B_OK) ? B_EXACT_ADDRESS : B_RANDOMIZED_ANY_ADDRESS, + allocate, B_NO_LOCK, B_READ_AREA | B_WRITE_AREA); + if (status < B_OK) + return status; + + if (fLastAreaReservedTop > fLastAreaTop) + _kern_unreserve_address_range(fLastAreaTop, fLastAreaReservedTop - fLastAreaTop); + + fLastArea = status; + fLastAreaTop = newAreaBase + allocate; + fLastAreaSize = allocate; + fLastAreaReservedTop = newAreaBase + newReservation; + + allocated = _Insert((void*)newAreaBase, allocate); + return B_OK; + } + + status_t _ResizeLastArea(size_t amount) + { + // TODO: We could use an inner lock here to avoid contention. + status_t status = resize_area(fLastArea, fLastAreaSize + amount); + if (status == B_OK) { + fLastAreaTop += amount; + fLastAreaSize += amount; + } + return status; + } + + status_t _UnmapLocked(FreeChunk* chunk) + { + // TODO: We could use an inner lock here to avoid contention. + MutexLocker locker(fLock, true); + + fChunksByAddressTree.Remove(chunk); + fChunksBySizeTree.Remove(chunk); + fFree -= chunk->size; + + const size_t size = chunk->size; + const addr_t address = (addr_t)chunk; + const addr_t top = address + size; + + const addr_t lastAreaBase = (fLastAreaTop - fLastAreaSize); + if (address <= lastAreaBase && top >= fLastAreaTop) { + // The whole area is being deleted. + const addr_t lastTop = fLastAreaTop, reservedTop = fLastAreaReservedTop; + fLastArea = -1; + fLastAreaTop = fLastAreaSize = fLastAreaReservedTop = 0; + locker.Unlock(); + + if (reservedTop > lastTop) + _kern_unreserve_address_range(lastTop, reservedTop - lastTop); + } else if (top == fLastAreaTop) { + // Shrink the top. + status_t status = resize_area(fLastArea, fLastAreaSize - size); + if (status != B_OK) + return status; + + fLastAreaSize -= size; + fLastAreaTop -= size; + return B_OK; + } else if (address == lastAreaBase) { + // Shrink the bottom. + if (munmap(chunk, size) != 0) + return errno; + fLastAreaSize -= size; + return B_OK; + } else if (address >= lastAreaBase && address < fLastAreaTop) { + // Cut the middle and get the new ID. + if (munmap(chunk, size) != 0) + return errno; + + fLastAreaSize = fLastAreaTop - top; + fLastArea = area_for((void*)(fLastAreaTop - 1)); + return B_OK; + } else { + // Not in the last area. + locker.Unlock(); + } + + if (munmap(chunk, size) != 0) + return errno; + return B_OK; + } + +private: + struct FreeChunk { + SplayTreeLink address_tree_link; + SplayTreeLink size_tree_link; + FreeChunk* address_tree_list_link; + FreeChunk* size_tree_list_link; + + size_t size; + + public: + inline addr_t NextAddress() const { return ((addr_t)this + size); } + }; + + struct ChunksByAddressTreeDefinition { + typedef addr_t KeyType; + typedef FreeChunk NodeType; + + static addr_t GetKey(const FreeChunk* node) + { + return (addr_t)node; + } + + static SplayTreeLink* GetLink(FreeChunk* node) + { + return &node->address_tree_link; + } + + static int Compare(const addr_t& key, const FreeChunk* node) + { + if (key == (addr_t)node) + return 0; + return (key < (addr_t)node) ? -1 : 1; + } + + static FreeChunk** GetListLink(FreeChunk* node) + { + return &node->address_tree_list_link; + } + }; + typedef IteratableSplayTree ChunksByAddressTree; + + struct ChunksBySizeTreeDefinition { + struct KeyType { + size_t size; + addr_t address; + + public: + KeyType(size_t _size) : size(_size), address(0) {} + KeyType(const FreeChunk* chunk) : size(chunk->size), address((addr_t)chunk) {} + }; + typedef FreeChunk NodeType; + + static KeyType GetKey(const FreeChunk* node) + { + return KeyType(node); + } + + static SplayTreeLink* GetLink(FreeChunk* node) + { + return &node->size_tree_link; + } + + static int Compare(const KeyType& key, const FreeChunk* node) + { + if (key.size == node->size) + return ChunksByAddressTreeDefinition::Compare(key.address, node); + return (key.size < node->size) ? -1 : 1; + } + + static FreeChunk** GetListLink(FreeChunk* node) + { + return &node->size_tree_list_link; + } + }; + typedef IteratableSplayTree ChunksBySizeTree; + +private: + mutex fLock; + + size_t fUsed; + size_t fFree; + + ChunksByAddressTree fChunksByAddressTree; + ChunksBySizeTree fChunksBySizeTree; + + area_id fLastArea; + addr_t fLastAreaTop; + size_t fLastAreaSize; + size_t fLastAreaReservedTop; +}; + +} // namespace + + +static char sPagesAllocatorStorage[sizeof(PagesAllocator)] +#if defined(__GNUC__) && __GNUC__ >= 4 + __attribute__((__aligned__(alignof(PagesAllocator)))) +#endif + ; +static PagesAllocator* sPagesAllocator; + + +void +__init_pages_allocator() +{ + sPagesAllocator = new(sPagesAllocatorStorage) PagesAllocator; +} + + +void +__pages_allocator_before_fork() +{ + sPagesAllocator->BeforeFork(); +} + + +void +__pages_allocator_after_fork(int parent) +{ + sPagesAllocator->AfterFork(parent); +} + + +status_t +__allocate_pages(void** address, size_t length, int flags) +{ + if ((length % kPageSize) != 0) + debugger("PagesAllocator: incorrectly sized allocate"); + + if ((flags & MAP_FIXED) != 0) + return sPagesAllocator->AllocatePagesAt(*address, length); + + //fprintf(stderr, "AllocatePages! 0x%x\n", (int)length); + return sPagesAllocator->AllocatePages(*address, length); +} + + +status_t +__free_pages(void* address, size_t length) +{ + if ((length % kPageSize) != 0) + debugger("PagesAllocator: incorrectly sized free"); + + //fprintf(stderr, "FreePages! %p, 0x%x\n", address, (int)length); + return sPagesAllocator->FreePages(address, length); +} diff --git a/src/system/libroot/posix/malloc/openbsd/PagesAllocator.h b/src/system/libroot/posix/malloc/openbsd/PagesAllocator.h new file mode 100644 index 0000000000..a57b61b008 --- /dev/null +++ b/src/system/libroot/posix/malloc/openbsd/PagesAllocator.h @@ -0,0 +1,30 @@ +/* + * Copyright 2025, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + */ +#ifndef _PAGES_ALLOCATOR_H +#define _PAGES_ALLOCATOR_H + + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +void __init_pages_allocator(); +void __pages_allocator_before_fork(); +void __pages_allocator_after_fork(int parent); + + +status_t __allocate_pages(void** address, size_t length, int flags); +status_t __free_pages(void* address, size_t length); + + +#ifdef __cplusplus +} // extern "C" +#endif + + +#endif // _PAGES_ALLOCATOR_H diff --git a/src/system/libroot/posix/malloc/openbsd/malloc.c b/src/system/libroot/posix/malloc/openbsd/malloc.c index 17ba5fb127..47517d7397 100644 --- a/src/system/libroot/posix/malloc/openbsd/malloc.c +++ b/src/system/libroot/posix/malloc/openbsd/malloc.c @@ -23,15 +23,17 @@ * can buy me a beer in return. Poul-Henning Kamp */ -#ifndef MALLOC_SMALL +#if !defined(MALLOC_SMALL) && !defined(__HAIKU__) #define MALLOC_STATS #endif #include #include #include +#ifndef __HAIKU__ #include #include +#endif #include #include #include @@ -46,8 +48,14 @@ #include #endif +#ifdef __HAIKU__ +#include "wrapper.c" +#endif + +#ifndef __HAIKU__ #include "thread_private.h" #include +#endif #define MALLOC_PAGESHIFT _MAX_PAGE_SHIFT @@ -160,6 +168,9 @@ struct dir_info { int malloc_junk; /* junk fill? */ int mmap_flag; /* extra flag for mmap */ int mutex; +#ifdef __HAIKU__ /* cross-thread free optimization */ + int last_found_pool; +#endif int malloc_mt; /* multi-threaded mode? */ /* lists of free chunk info structs */ struct chunk_head chunk_info_list[BUCKETS + 1]; @@ -264,6 +275,13 @@ static union { __attribute__((section(".openbsd.mutable"))); #define mopts malloc_readonly.mopts +#ifdef __HAIKU__ +static inline u_int mopts_nmutexes() +{ + return mopts.malloc_pool[1]->malloc_mt ? mopts.malloc_mutexes : 2; +} +#endif + char *malloc_options; /* compile-time options */ static __dead void wrterror(struct dir_info *d, char *msg, ...) @@ -351,7 +369,11 @@ getpool(void) if (mopts.malloc_pool[1] == NULL || !mopts.malloc_pool[1]->malloc_mt) return mopts.malloc_pool[1]; else /* first one reserved for special pool */ +#ifdef __HAIKU__ + return mopts.malloc_pool[1 + get_thread_malloc_id() % +#else return mopts.malloc_pool[1 + TIB_GET()->tib_tid % +#endif (mopts.malloc_mutexes - 1)]; } @@ -361,6 +383,15 @@ wrterror(struct dir_info *d, char *msg, ...) int saved_errno = errno; va_list ap; +#ifdef __HAIKU__ + char msgBuf[1024]; + va_start(ap, msg); + vsnprintf(msgBuf, sizeof(msgBuf), msg, ap); + va_end(ap); + + debugger(msgBuf); + exit(-1); +#else dprintf(STDERR_FILENO, "%s(%d) in %s(): ", __progname, getpid(), (d != NULL && d->func) ? d->func : "unknown"); va_start(ap, msg); @@ -376,6 +407,7 @@ wrterror(struct dir_info *d, char *msg, ...) errno = saved_errno; abort(); +#endif } static void @@ -392,7 +424,11 @@ getrbyte(struct dir_info *d) u_char x; if (d->rbytesused >= sizeof(d->rbytes)) +#ifdef __HAIKU__ + d->rbytesused = 0; +#else rbytes_init(d); +#endif x = d->rbytes[d->rbytesused++]; return x; } @@ -503,16 +539,25 @@ omalloc_init(void) { char *p, *q, b[16]; int i, j; +#ifndef __HAIKU__ const int mib[2] = { CTL_VM, VM_MALLOC_CONF }; +#endif size_t sb; +#ifdef __HAIKU__ + memset(&mopts, 0, sizeof(mopts)); +#endif + /* * Default options */ mopts.malloc_mutexes = 8; +#ifndef __HAIKU__ mopts.def_malloc_junk = 1; +#endif mopts.def_maxcache = MALLOC_DEFAULT_CACHE; +#ifndef __HAIKU__ for (i = 0; i < 3; i++) { switch (i) { case 0: @@ -553,6 +598,7 @@ omalloc_init(void) } } } +#endif #ifdef MALLOC_STATS if (DO_STATS && (atexit(malloc_exit) == -1)) { @@ -615,6 +661,10 @@ omalloc_grow(struct dir_info *d) if (p == MAP_FAILED) return 1; +#ifdef __HAIKU__ + memset(p, 0, newsize); +#endif + STATS_ADD(d->malloc_used, newsize); STATS_ZERO(d->inserts); STATS_ZERO(d->insert_collisions); @@ -961,7 +1011,12 @@ map(struct dir_info *d, size_t sz, int zero_fill) mprotect(p, (cache->max - 1) * sz, PROT_NONE); p = (char*)p + (cache->max - 1) * sz; +#ifdef __HAIKU__ + if (zero_fill) + memset(p, 0, sz); +#else /* zero fill not needed, freshly mmapped */ +#endif return p; } } @@ -970,7 +1025,12 @@ map(struct dir_info *d, size_t sz, int zero_fill) p = MMAP(sz, d->mmap_flag); if (p != MAP_FAILED) STATS_ADD(d->malloc_used, sz); +#ifdef __HAIKU__ + if (zero_fill) + memset(p, 0, sz); +#else /* zero fill not needed */ +#endif return p; } @@ -1020,6 +1080,9 @@ alloc_chunk_info(struct dir_info *d, u_int bucket) q = MMAP(MALLOC_PAGESIZE * chunk_pages, d->mmap_flag); if (q == MAP_FAILED) return NULL; +#ifdef __HAIKU__ + memset(q, 0, MALLOC_PAGESIZE * chunk_pages); +#endif d->chunk_pages = q; d->chunk_pages_used = 0; STATS_ADD(d->malloc_used, MALLOC_PAGESIZE * @@ -1415,6 +1478,9 @@ malloc_recurse(struct dir_info *d) errno = EDEADLK; } +#ifdef __HAIKU__ +static +#endif void _malloc_init(int from_rthreads) { @@ -1491,6 +1557,9 @@ _malloc_init(int from_rthreads) sz += d->bigcache_size * sizeof(struct bigcache); if (sz > 0) { void *p = MMAP(sz, 0); +#ifdef __HAIKU__ + memset(p, 0, sz); +#endif if (p == MAP_FAILED) wrterror(NULL, "malloc_init mmap2 failed"); @@ -1506,6 +1575,9 @@ _malloc_init(int from_rthreads) } } d->mutex = i; +#ifdef __HAIKU__ /* cross-thread free optimization */ + d->last_found_pool = -1; +#endif } _MALLOC_UNLOCK(1); @@ -1515,7 +1587,7 @@ DEF_STRONG(_malloc_init); #define PROLOGUE(p, fn) \ d = (p); \ if (d == NULL) { \ - _malloc_init(0); \ + /* _malloc_init(0); */ \ d = (p); \ } \ _MALLOC_LOCK(d->mutex); \ @@ -1548,6 +1620,7 @@ malloc(size_t size) } DEF_STRONG(malloc); +#ifndef __HAIKU__ void * malloc_conceal(size_t size) { @@ -1562,6 +1635,7 @@ malloc_conceal(size_t size) return r; } DEF_WEAK(malloc_conceal); +#endif static struct region_info * findpool(void *p, struct dir_info *argpool, struct dir_info **foundpool, @@ -1572,11 +1646,30 @@ findpool(void *p, struct dir_info *argpool, struct dir_info **foundpool, if (r == NULL) { u_int i, nmutexes; + int first, skip; nmutexes = mopts.malloc_pool[1]->malloc_mt ? mopts.malloc_mutexes : 2; +#ifdef __HAIKU__ /* cross-thread free optimization */ + first = argpool->last_found_pool; + skip = -1; +#endif for (i = 1; i < nmutexes; i++) { +#ifdef __HAIKU__ /* cross-thread free optimization */ + u_int j; + if (first >= 0) { + j = first & (nmutexes - 1); + skip = j; + first = -1; + i--; + } else { + j = (argpool->mutex + i) & (nmutexes - 1); + if (skip >= 0 && j == skip) + continue; + } +#else u_int j = (argpool->mutex + i) & (nmutexes - 1); +#endif pool->active--; _MALLOC_UNLOCK(pool->mutex); @@ -1587,6 +1680,9 @@ findpool(void *p, struct dir_info *argpool, struct dir_info **foundpool, if (r != NULL) { *saved_function = pool->func; pool->func = argpool->func; +#ifdef __HAIKU__ /* cross-thread free optimization */ + argpool->last_found_pool = j; +#endif break; } } @@ -1758,6 +1854,9 @@ freezero_p(void *ptr, size_t sz) free(ptr); } +#ifdef __HAIKU__ +static +#endif void freezero(void *ptr, size_t sz) { @@ -1974,6 +2073,42 @@ realloc(void *ptr, size_t size) } DEF_STRONG(realloc); +#ifdef __HAIKU__ +static size_t +omalloc_usable_size(struct dir_info **argpool, void *p) +{ + struct region_info *r; + struct dir_info *pool; + const char *saved_function; + size_t sz; + + r = findpool(p, *argpool, &pool, &saved_function); + + REALSIZE(sz, r); + + if (*argpool != pool) { + pool->func = saved_function; + *argpool = pool; + } + return sz; +} + +size_t +malloc_usable_size(void *ptr) +{ + struct dir_info *d; + size_t r; + int saved_errno = errno; + + PROLOGUE(getpool(), "malloc_usable_size") + SET_CALLER(d, caller(d)); + r = omalloc_usable_size(&d, ptr); + EPILOGUE() + return r; +} +DEF_STRONG(malloc_usable_size); +#endif + /* * This is sqrt(SIZE_MAX+1), as s1*s2 <= SIZE_MAX * if both s1 < MUL_NO_OVERFLOW and s2 < MUL_NO_OVERFLOW @@ -2006,6 +2141,7 @@ calloc(size_t nmemb, size_t size) } DEF_STRONG(calloc); +#ifndef __HAIKU__ void * calloc_conceal(size_t nmemb, size_t size) { @@ -2189,6 +2325,7 @@ recallocarray(void *ptr, size_t oldnmemb, size_t newnmemb, size_t size) return r; } DEF_WEAK(recallocarray); +#endif static void * mapalign(struct dir_info *d, size_t alignment, size_t sz, int zero_fill) diff --git a/src/system/libroot/posix/malloc/openbsd/wrapper.c b/src/system/libroot/posix/malloc/openbsd/wrapper.c new file mode 100644 index 0000000000..bbfe925093 --- /dev/null +++ b/src/system/libroot/posix/malloc/openbsd/wrapper.c @@ -0,0 +1,278 @@ +/* + * Copyright 2024, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "PagesAllocator.h" + + +/* generic stuff */ +#if B_PAGE_SIZE == 4096 +#define _MAX_PAGE_SHIFT 12 +#endif + +extern char* __progname; + +#define MAP_CONCEAL (0) +#define __MAP_NOREPLACE (0) + +#define DEF_STRONG(X) +#define DEF_WEAK(x) + + +/* entropy routines, wrapped for malloc */ + +static uint32_t +malloc_arc4random() +{ + // TODO: Improve this? + const uintptr_t address = (uintptr_t)&address; + uint32_t random = (uint32_t)address * 1103515245; + random += (uint32_t)system_time(); + return random; +} +#define arc4random malloc_arc4random + + +static void +malloc_arc4random_buf(void *_buf, size_t nbytes) +{ + uint8* buf = (uint8*)_buf; + while (nbytes > 0) { + uint32_t value = malloc_arc4random(); + const size_t copy = (nbytes > 4) ? 4 : nbytes; + memcpy(buf, &value, copy); + nbytes -= copy; + buf += copy; + } +} +#define arc4random_buf malloc_arc4random_buf + + +static uint32_t +malloc_arc4random_uniform(uint32_t upper_bound) +{ + return (malloc_arc4random() % upper_bound); +} +#define arc4random_uniform malloc_arc4random_uniform + + +/* memory routines, wrapped for malloc */ + +static inline void +malloc_explicit_bzero(void* buf, size_t len) +{ + memset(buf, 0, len); +} +#define explicit_bzero malloc_explicit_bzero + + +static int +malloc_mprotect(void* address, size_t length, int protection) +{ + /* do nothing */ + return 0; +} +#define mprotect malloc_mprotect + + +static int +mimmutable(void* address, size_t length) +{ + /* do nothing */ + return 0; +} + + +/* memory mapping */ + + +static void* +malloc_mmap(void* address, size_t length, int protection, int flags, int fd, off_t offset) +{ + status_t status; + length = (length + (B_PAGE_SIZE - 1)) & ~(B_PAGE_SIZE - 1);; + + status = __allocate_pages(&address, length, flags); + if (status != B_OK) { + __set_errno(status); + return MAP_FAILED; + } + return address; +} +#define mmap malloc_mmap + + +static int +malloc_munmap(void* address, size_t length) +{ + status_t status = __free_pages(address, length); + if (status < 0) { + errno = status; + return -1; + } + return 0; +} +#define munmap malloc_munmap + + +/* public methods */ + +void* +memalign(size_t align, size_t len) +{ + void* result = NULL; + int status; + + if (align < sizeof(void*)) + align = sizeof(void*); + + status = posix_memalign(&result, align, len); + if (status != 0) { + errno = status; + return NULL; + } + return result; +} + + +void* +valloc(size_t size) +{ + return memalign(B_PAGE_SIZE, size); +} + + +/* malloc implementation */ + +#define _MALLOC_MUTEXES 32 +static mutex sMallocMutexes[_MALLOC_MUTEXES]; +static pthread_once_t sThreadedMallocInitOnce = PTHREAD_ONCE_INIT; + +static int32 sNextMallocThreadID = 1; + +static u_int mopts_nmutexes(); +static void _malloc_init(int from_rthreads); + + +static inline void +_MALLOC_LOCK(int32 index) +{ + mutex_lock(&sMallocMutexes[index]); +} + + +static inline void +_MALLOC_UNLOCK(int32 index) +{ + mutex_unlock(&sMallocMutexes[index]); +} + + +static void +init_threaded_malloc() +{ + u_int i; + for (i = 2; i < _MALLOC_MUTEXES; i++) + mutex_init(&sMallocMutexes[i], "heap mutex"); + + _MALLOC_LOCK(0); + _malloc_init(1); + _MALLOC_UNLOCK(0); +} + + +status_t +__init_heap() +{ + tls_set(TLS_MALLOC_SLOT, (void*)0); + __init_pages_allocator(); + mutex_init(&sMallocMutexes[0], "heap mutex"); + mutex_init(&sMallocMutexes[1], "heap mutex"); + _malloc_init(0); + return B_OK; +} + + +static int32 +get_thread_malloc_id() +{ + int32 result = (int32)(intptr_t)tls_get(TLS_MALLOC_SLOT); + if (result == -1) { + // thread has never called malloc() before; assign it an ID. + result = atomic_add(&sNextMallocThreadID, 1); + tls_set(TLS_MALLOC_SLOT, (void*)(intptr_t)result); + } + return result; +} + + +void +__heap_thread_init() +{ + pthread_once(&sThreadedMallocInitOnce, &init_threaded_malloc); + tls_set(TLS_MALLOC_SLOT, (void*)(intptr_t)-1); +} + + +void +__heap_thread_exit() +{ + const int32 id = (int32)(intptr_t)tls_get(TLS_MALLOC_SLOT); + if (id != -1 && id == (sNextMallocThreadID - 1)) { + // Try to "de-allocate" this thread's ID. + atomic_test_and_set(&sNextMallocThreadID, id, id + 1); + } +} + + +void +__heap_before_fork() +{ + u_int i; + u_int nmutexes = mopts_nmutexes(); + for (i = 0; i < nmutexes; i++) + _MALLOC_LOCK(i); + + __pages_allocator_before_fork(); +} + + +void +__heap_after_fork_child() +{ + u_int i; + u_int nmutexes = mopts_nmutexes(); + for (i = 0; i < nmutexes; i++) + mutex_init(&sMallocMutexes[i], "heap mutex"); + + __pages_allocator_after_fork(0); +} + + +void +__heap_after_fork_parent() +{ + u_int i; + u_int nmutexes = mopts_nmutexes(); + for (i = 0; i < nmutexes; i++) + _MALLOC_UNLOCK(i); + + __pages_allocator_after_fork(1); +} + + +void +__heap_terminate_after() +{ +}