From 69cb8e25ff5aa21f618d775e4e73ad150d4cd7de Mon Sep 17 00:00:00 2001 From: Augustin Cavalier Date: Wed, 12 Feb 2025 20:56:22 -0500 Subject: [PATCH] libroot/posix: Add glue code, global caching logic, and integrate OpenBSD malloc. * PagesAllocator: A process-global caching strategy for the allocator. It deals with allocating virtual addresses and memory, and gives us back some of the performance that's lost by having an actual decommitment strategy (which the hoard2 glue code doesn't.) It uses two SplayTrees to manage free lists, and resizes areas on allocate if they aren't next to a free chunk (which saves a lot of time for large reallocations.) There's still room for improvement here, see inline TODOs. But overall we get pretty good performance with it. * Add a TLS slot for the allocator glue to use. Right now it just puts integers in there (since thread IDs are not evenly distributed), but we could put a data structure pointer in there as well, potentially. Change-Id: I56ddb0b022a468dc04275075ed7e174b339c8ca4 Reviewed-on: https://review.haiku-os.org/c/haiku/+/8335 Reviewed-by: waddlesplash Tested-by: Commit checker robot --- headers/private/system/tls.h | 1 + .../libroot/posix/malloc/openbsd/Jamfile | 18 + .../posix/malloc/openbsd/PagesAllocator.cpp | 551 ++++++++++++++++++ .../posix/malloc/openbsd/PagesAllocator.h | 30 + .../libroot/posix/malloc/openbsd/malloc.c | 141 ++++- .../libroot/posix/malloc/openbsd/wrapper.c | 278 +++++++++ 6 files changed, 1017 insertions(+), 2 deletions(-) create mode 100644 src/system/libroot/posix/malloc/openbsd/Jamfile create mode 100644 src/system/libroot/posix/malloc/openbsd/PagesAllocator.cpp create mode 100644 src/system/libroot/posix/malloc/openbsd/PagesAllocator.h create mode 100644 src/system/libroot/posix/malloc/openbsd/wrapper.c diff --git a/headers/private/system/tls.h b/headers/private/system/tls.h index e87ac4698b..7d828dc306 100644 --- a/headers/private/system/tls.h +++ b/headers/private/system/tls.h @@ -21,6 +21,7 @@ enum { TLS_ON_EXIT_THREAD_SLOT, TLS_USER_THREAD_SLOT, TLS_DYNAMIC_THREAD_VECTOR, + TLS_MALLOC_SLOT, TLS_LOCALE_SLOT, // Note: these entries can safely be changed between diff --git a/src/system/libroot/posix/malloc/openbsd/Jamfile b/src/system/libroot/posix/malloc/openbsd/Jamfile new file mode 100644 index 0000000000..a28446098d --- /dev/null +++ b/src/system/libroot/posix/malloc/openbsd/Jamfile @@ -0,0 +1,18 @@ +SubDir HAIKU_TOP src system libroot posix malloc openbsd ; + +UsePrivateHeaders kernel libroot shared ; +UseHeaders [ FDirName $(HAIKU_TOP) headers compatibility bsd ] : true ; + +local architectureObject ; +for architectureObject in [ MultiArchSubDirSetup ] { + on $(architectureObject) { + local architecture = $(TARGET_PACKAGING_ARCH) ; + + UsePrivateSystemHeaders ; + + MergeObject <$(architecture)>posix_malloc.o : + PagesAllocator.cpp + malloc.c + ; + } +} diff --git a/src/system/libroot/posix/malloc/openbsd/PagesAllocator.cpp b/src/system/libroot/posix/malloc/openbsd/PagesAllocator.cpp new file mode 100644 index 0000000000..fee394e531 --- /dev/null +++ b/src/system/libroot/posix/malloc/openbsd/PagesAllocator.cpp @@ -0,0 +1,551 @@ +/* + * Copyright 2025, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Augustin Cavalier + */ + +#include "PagesAllocator.h" + +#include +#include +#include +#include + +#include +#include +#include + + +/*! The local page size. Must be a multiple of the system page size. */ +static const size_t kPageSize = B_PAGE_SIZE; + +/*! The "largest useful" chunk size: any allocations larger than this + * will get their own area, rather than sharing the common one(s). */ +static const size_t kLargestUsefulChunk = 512 * kPageSize; + +/*! Amount of virtual address space to reserve when creating new areas. */ +static const size_t kReserveAddressSpace = 128 * 1024 * 1024; + +/*! Cache up to this many percentage points of free memory (compared to used.) */ +static const size_t kFreePercentage = 25; + +/*! Always allow this much free memory, even if it's above kFreePercentage. */ +static const size_t kFreeMinimum = 128 * kPageSize; + + +namespace { + +class PagesAllocator { + struct FreeChunk; + +public: + PagesAllocator() + { + mutex_init(&fLock, "PagesAllocator lock"); + fUsed = fFree = 0; + fLastArea = -1; + } + + ~PagesAllocator() + { + } + + void BeforeFork() + { + mutex_lock(&fLock); + } + + void AfterFork(bool parent) + { + if (parent) { + mutex_unlock(&fLock); + } else { + if (fLastArea >= 0) + fLastArea = area_for((void*)(fLastAreaTop - 1)); + + mutex_init(&fLock, "PagesAllocator lock"); + } + } + + status_t AllocatePages(void*& address, size_t allocate) + { + MutexLocker locker(fLock); + + if (allocate > kLargestUsefulChunk) { + // Create an area just for this allocation. + locker.Unlock(); + area_id area = create_area("heap large allocation", + &address, B_ANY_ADDRESS, allocate, + B_NO_LOCK, B_READ_AREA | B_WRITE_AREA); + if (area >= 0) { + locker.Lock(); + fUsed += allocate; + return B_OK; + } + return area; + } + + if (fFree >= allocate) { + // Try to use memory from the cache. + FreeChunk* chunk = fChunksBySizeTree.FindClosest(allocate, true, true); + if (chunk != NULL) { + address = _Use(chunk, allocate); + return B_OK; + } + } + + // Not enough memory in the cache. Allocate some more. + FreeChunk* chunk; + status_t status = _Map(allocate, chunk); + if (status != B_OK) + return status; + + address = _Use(chunk, allocate); + return B_OK; + } + + status_t AllocatePagesAt(void* _address, size_t allocate) + { + const addr_t address = (addr_t)_address; + MutexLocker locker(fLock); + + if (allocate <= kLargestUsefulChunk) { + FreeChunk* chunk = fChunksByAddressTree.FindClosest(address, false, true); + if (chunk != NULL && chunk->NextAddress() > address) { + // The address is in a free chunk. + size_t remainingAfter = chunk->size - (address - (addr_t)chunk); + if (remainingAfter < allocate) { + if (chunk->NextAddress() != fLastAreaTop) + return B_NO_MEMORY; + + size_t add = allocate - remainingAfter; + status_t status = _ResizeLastArea(add); + if (status != B_OK) + return status; + + chunk = _Insert((void*)chunk->NextAddress(), add); + } + + // Cut the beginning? + if (address == (addr_t)chunk) { + _Use(chunk, allocate); + return B_OK; + } + + // Why would anyone want to allocate such a specific address, anyway? + debugger("PagesAllocator: middle-chunk allocation not implemented!"); + return B_ERROR; + } + } + + if (address == fLastAreaTop) { + if (allocate > kLargestUsefulChunk) + return B_NO_MEMORY; + + status_t status = _ResizeLastArea(allocate); + if (status == B_OK) { + fUsed += allocate; + return B_OK; + } + return status; + } + if (address >= (fLastAreaTop - fLastAreaSize) && address < fLastAreaTop) { + // We didn't find a matching free chunk, so that isn't going to work. + return EEXIST; + } + + locker.Unlock(); + + // One last try: see if we can resize the area this belongs to. + area_info info; + info.area = area_for((void*)(address - 1)); + if (info.area < 0) + return info.area; + + status_t status = get_area_info(info.area, &info); + if (status != B_OK) + return status; + + if (((addr_t)info.address + info.size) != address) + return B_NO_MEMORY; + + status = resize_area(info.area, info.size + allocate); + if (status == B_OK) { + locker.Lock(); + fUsed += allocate; + return B_OK; + } + + // TODO: We could add a "resize allowing move" feature to resize_area, + // which would avoid having to memcpy() the data of large allocations. + return status; + } + + status_t FreePages(void* _address, size_t size) + { + MutexLocker locker(fLock); + + if (size > fUsed) + debugger("PagesAllocator: request to free more than allocated"); + fUsed -= size; + + FreeChunk* chunk = _Insert(_address, size); + + if (size > kLargestUsefulChunk) { + // TODO: This doesn't deal with a free of a smaller number of pages + // within a "large allocation" area. We should probably free those + // immediately also, or at least mark them specially in the trees + // (so they don't get reused for anything but the large allocation.) + + locker.Detach(); + return _UnmapLocked(chunk); + } + + if (fFree <= _FreeLimit()) { + // TODO: If not decommitting/unmapping, we might free the pages + // using MADV_FREE (perhaps on medium-sized chunks with others + // of equal sizes already present in the tree, at least.) + return B_OK; + } + + while (fFree > _FreeLimit()) { + FreeChunk* chunk = fChunksBySizeTree.FindMax(); + status_t status = _UnmapLocked(chunk); + if (status != B_OK) + return status; + mutex_lock(&fLock); + } + + if (fFree > kLargestUsefulChunk && fFree > (_FreeLimit() / 2) + && fChunksBySizeTree.FindMax()->size == kPageSize) { + // All the free chunks are single pages, and there's more of them + // than a "largest useful" chunk. Just evict them all at this point. + while (FreeChunk* chunk = fChunksBySizeTree.FindMin()) { + if (chunk->size != kPageSize) + break; + + status_t status = _UnmapLocked(chunk); + if (status != B_OK) + return status; + mutex_lock(&fLock); + } + } + + return B_OK; + } + +private: + void* _Use(FreeChunk* chunk, size_t amount) + { + fChunksBySizeTree.Remove(chunk); + fChunksByAddressTree.Remove(chunk); + + if (chunk->size == amount) { + // The whole chunk will be used. + // Nothing special to do in this case. + } else { + // Some will be left over. + // Break the remainder off and reinsert into the trees. + FreeChunk* newChunk = (FreeChunk*)((addr_t)chunk + amount); + newChunk->size = (chunk->size - amount); + + fChunksBySizeTree.Insert(newChunk); + fChunksByAddressTree.Insert(newChunk); + } + + fUsed += amount; + fFree -= amount; + return (void*)chunk; + } + + FreeChunk* _Insert(void* _address, size_t size) + { + fFree += size; + + const addr_t address = (addr_t)_address; + FreeChunk* chunk; + + FreeChunk* preceding = fChunksByAddressTree.FindClosest(address, false, false); + if (preceding != NULL && preceding->NextAddress() == address) { + fChunksBySizeTree.Remove(preceding); + chunk = preceding; + chunk->size += size; + } else { + chunk = (FreeChunk*)_address; + chunk->size = size; + fChunksByAddressTree.Insert(chunk); + } + + FreeChunk* following = chunk->address_tree_list_link; + if (following != NULL && chunk->NextAddress() == (addr_t)following) { + fChunksBySizeTree.Remove(following); + fChunksByAddressTree.Remove(following); + chunk->size += following->size; + } + fChunksBySizeTree.Insert(chunk); + + return chunk; + } + + size_t _FreeLimit() const + { + size_t freeLimit = ((fUsed * kFreePercentage) / 100); + if (freeLimit < kFreeMinimum) + freeLimit = kFreeMinimum; + else + freeLimit = (freeLimit + (kPageSize - 1)) & ~(kPageSize - 1); + return freeLimit; + } + +private: + status_t _Map(size_t allocate, FreeChunk*& allocated) + { + if (fLastArea >= 0) { + addr_t oldTop = fLastAreaTop; + status_t status = _ResizeLastArea(allocate); + if (status == B_OK) { + allocated = _Insert((void*)oldTop, allocate); + return B_OK; + } + } + + // Create a new area. + // TODO: We could use an inner lock here to avoid contention. + addr_t newAreaBase; + status_t status = _kern_reserve_address_range(&newAreaBase, + B_RANDOMIZED_ANY_ADDRESS, kReserveAddressSpace); + size_t newReservation = (status == B_OK) ? kReserveAddressSpace : 0; + + status = create_area("heap area", (void**)&newAreaBase, + (status == B_OK) ? B_EXACT_ADDRESS : B_RANDOMIZED_ANY_ADDRESS, + allocate, B_NO_LOCK, B_READ_AREA | B_WRITE_AREA); + if (status < B_OK) + return status; + + if (fLastAreaReservedTop > fLastAreaTop) + _kern_unreserve_address_range(fLastAreaTop, fLastAreaReservedTop - fLastAreaTop); + + fLastArea = status; + fLastAreaTop = newAreaBase + allocate; + fLastAreaSize = allocate; + fLastAreaReservedTop = newAreaBase + newReservation; + + allocated = _Insert((void*)newAreaBase, allocate); + return B_OK; + } + + status_t _ResizeLastArea(size_t amount) + { + // TODO: We could use an inner lock here to avoid contention. + status_t status = resize_area(fLastArea, fLastAreaSize + amount); + if (status == B_OK) { + fLastAreaTop += amount; + fLastAreaSize += amount; + } + return status; + } + + status_t _UnmapLocked(FreeChunk* chunk) + { + // TODO: We could use an inner lock here to avoid contention. + MutexLocker locker(fLock, true); + + fChunksByAddressTree.Remove(chunk); + fChunksBySizeTree.Remove(chunk); + fFree -= chunk->size; + + const size_t size = chunk->size; + const addr_t address = (addr_t)chunk; + const addr_t top = address + size; + + const addr_t lastAreaBase = (fLastAreaTop - fLastAreaSize); + if (address <= lastAreaBase && top >= fLastAreaTop) { + // The whole area is being deleted. + const addr_t lastTop = fLastAreaTop, reservedTop = fLastAreaReservedTop; + fLastArea = -1; + fLastAreaTop = fLastAreaSize = fLastAreaReservedTop = 0; + locker.Unlock(); + + if (reservedTop > lastTop) + _kern_unreserve_address_range(lastTop, reservedTop - lastTop); + } else if (top == fLastAreaTop) { + // Shrink the top. + status_t status = resize_area(fLastArea, fLastAreaSize - size); + if (status != B_OK) + return status; + + fLastAreaSize -= size; + fLastAreaTop -= size; + return B_OK; + } else if (address == lastAreaBase) { + // Shrink the bottom. + if (munmap(chunk, size) != 0) + return errno; + fLastAreaSize -= size; + return B_OK; + } else if (address >= lastAreaBase && address < fLastAreaTop) { + // Cut the middle and get the new ID. + if (munmap(chunk, size) != 0) + return errno; + + fLastAreaSize = fLastAreaTop - top; + fLastArea = area_for((void*)(fLastAreaTop - 1)); + return B_OK; + } else { + // Not in the last area. + locker.Unlock(); + } + + if (munmap(chunk, size) != 0) + return errno; + return B_OK; + } + +private: + struct FreeChunk { + SplayTreeLink address_tree_link; + SplayTreeLink size_tree_link; + FreeChunk* address_tree_list_link; + FreeChunk* size_tree_list_link; + + size_t size; + + public: + inline addr_t NextAddress() const { return ((addr_t)this + size); } + }; + + struct ChunksByAddressTreeDefinition { + typedef addr_t KeyType; + typedef FreeChunk NodeType; + + static addr_t GetKey(const FreeChunk* node) + { + return (addr_t)node; + } + + static SplayTreeLink* GetLink(FreeChunk* node) + { + return &node->address_tree_link; + } + + static int Compare(const addr_t& key, const FreeChunk* node) + { + if (key == (addr_t)node) + return 0; + return (key < (addr_t)node) ? -1 : 1; + } + + static FreeChunk** GetListLink(FreeChunk* node) + { + return &node->address_tree_list_link; + } + }; + typedef IteratableSplayTree ChunksByAddressTree; + + struct ChunksBySizeTreeDefinition { + struct KeyType { + size_t size; + addr_t address; + + public: + KeyType(size_t _size) : size(_size), address(0) {} + KeyType(const FreeChunk* chunk) : size(chunk->size), address((addr_t)chunk) {} + }; + typedef FreeChunk NodeType; + + static KeyType GetKey(const FreeChunk* node) + { + return KeyType(node); + } + + static SplayTreeLink* GetLink(FreeChunk* node) + { + return &node->size_tree_link; + } + + static int Compare(const KeyType& key, const FreeChunk* node) + { + if (key.size == node->size) + return ChunksByAddressTreeDefinition::Compare(key.address, node); + return (key.size < node->size) ? -1 : 1; + } + + static FreeChunk** GetListLink(FreeChunk* node) + { + return &node->size_tree_list_link; + } + }; + typedef IteratableSplayTree ChunksBySizeTree; + +private: + mutex fLock; + + size_t fUsed; + size_t fFree; + + ChunksByAddressTree fChunksByAddressTree; + ChunksBySizeTree fChunksBySizeTree; + + area_id fLastArea; + addr_t fLastAreaTop; + size_t fLastAreaSize; + size_t fLastAreaReservedTop; +}; + +} // namespace + + +static char sPagesAllocatorStorage[sizeof(PagesAllocator)] +#if defined(__GNUC__) && __GNUC__ >= 4 + __attribute__((__aligned__(alignof(PagesAllocator)))) +#endif + ; +static PagesAllocator* sPagesAllocator; + + +void +__init_pages_allocator() +{ + sPagesAllocator = new(sPagesAllocatorStorage) PagesAllocator; +} + + +void +__pages_allocator_before_fork() +{ + sPagesAllocator->BeforeFork(); +} + + +void +__pages_allocator_after_fork(int parent) +{ + sPagesAllocator->AfterFork(parent); +} + + +status_t +__allocate_pages(void** address, size_t length, int flags) +{ + if ((length % kPageSize) != 0) + debugger("PagesAllocator: incorrectly sized allocate"); + + if ((flags & MAP_FIXED) != 0) + return sPagesAllocator->AllocatePagesAt(*address, length); + + //fprintf(stderr, "AllocatePages! 0x%x\n", (int)length); + return sPagesAllocator->AllocatePages(*address, length); +} + + +status_t +__free_pages(void* address, size_t length) +{ + if ((length % kPageSize) != 0) + debugger("PagesAllocator: incorrectly sized free"); + + //fprintf(stderr, "FreePages! %p, 0x%x\n", address, (int)length); + return sPagesAllocator->FreePages(address, length); +} diff --git a/src/system/libroot/posix/malloc/openbsd/PagesAllocator.h b/src/system/libroot/posix/malloc/openbsd/PagesAllocator.h new file mode 100644 index 0000000000..a57b61b008 --- /dev/null +++ b/src/system/libroot/posix/malloc/openbsd/PagesAllocator.h @@ -0,0 +1,30 @@ +/* + * Copyright 2025, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + */ +#ifndef _PAGES_ALLOCATOR_H +#define _PAGES_ALLOCATOR_H + + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +void __init_pages_allocator(); +void __pages_allocator_before_fork(); +void __pages_allocator_after_fork(int parent); + + +status_t __allocate_pages(void** address, size_t length, int flags); +status_t __free_pages(void* address, size_t length); + + +#ifdef __cplusplus +} // extern "C" +#endif + + +#endif // _PAGES_ALLOCATOR_H diff --git a/src/system/libroot/posix/malloc/openbsd/malloc.c b/src/system/libroot/posix/malloc/openbsd/malloc.c index 17ba5fb127..47517d7397 100644 --- a/src/system/libroot/posix/malloc/openbsd/malloc.c +++ b/src/system/libroot/posix/malloc/openbsd/malloc.c @@ -23,15 +23,17 @@ * can buy me a beer in return. Poul-Henning Kamp */ -#ifndef MALLOC_SMALL +#if !defined(MALLOC_SMALL) && !defined(__HAIKU__) #define MALLOC_STATS #endif #include #include #include +#ifndef __HAIKU__ #include #include +#endif #include #include #include @@ -46,8 +48,14 @@ #include #endif +#ifdef __HAIKU__ +#include "wrapper.c" +#endif + +#ifndef __HAIKU__ #include "thread_private.h" #include +#endif #define MALLOC_PAGESHIFT _MAX_PAGE_SHIFT @@ -160,6 +168,9 @@ struct dir_info { int malloc_junk; /* junk fill? */ int mmap_flag; /* extra flag for mmap */ int mutex; +#ifdef __HAIKU__ /* cross-thread free optimization */ + int last_found_pool; +#endif int malloc_mt; /* multi-threaded mode? */ /* lists of free chunk info structs */ struct chunk_head chunk_info_list[BUCKETS + 1]; @@ -264,6 +275,13 @@ static union { __attribute__((section(".openbsd.mutable"))); #define mopts malloc_readonly.mopts +#ifdef __HAIKU__ +static inline u_int mopts_nmutexes() +{ + return mopts.malloc_pool[1]->malloc_mt ? mopts.malloc_mutexes : 2; +} +#endif + char *malloc_options; /* compile-time options */ static __dead void wrterror(struct dir_info *d, char *msg, ...) @@ -351,7 +369,11 @@ getpool(void) if (mopts.malloc_pool[1] == NULL || !mopts.malloc_pool[1]->malloc_mt) return mopts.malloc_pool[1]; else /* first one reserved for special pool */ +#ifdef __HAIKU__ + return mopts.malloc_pool[1 + get_thread_malloc_id() % +#else return mopts.malloc_pool[1 + TIB_GET()->tib_tid % +#endif (mopts.malloc_mutexes - 1)]; } @@ -361,6 +383,15 @@ wrterror(struct dir_info *d, char *msg, ...) int saved_errno = errno; va_list ap; +#ifdef __HAIKU__ + char msgBuf[1024]; + va_start(ap, msg); + vsnprintf(msgBuf, sizeof(msgBuf), msg, ap); + va_end(ap); + + debugger(msgBuf); + exit(-1); +#else dprintf(STDERR_FILENO, "%s(%d) in %s(): ", __progname, getpid(), (d != NULL && d->func) ? d->func : "unknown"); va_start(ap, msg); @@ -376,6 +407,7 @@ wrterror(struct dir_info *d, char *msg, ...) errno = saved_errno; abort(); +#endif } static void @@ -392,7 +424,11 @@ getrbyte(struct dir_info *d) u_char x; if (d->rbytesused >= sizeof(d->rbytes)) +#ifdef __HAIKU__ + d->rbytesused = 0; +#else rbytes_init(d); +#endif x = d->rbytes[d->rbytesused++]; return x; } @@ -503,16 +539,25 @@ omalloc_init(void) { char *p, *q, b[16]; int i, j; +#ifndef __HAIKU__ const int mib[2] = { CTL_VM, VM_MALLOC_CONF }; +#endif size_t sb; +#ifdef __HAIKU__ + memset(&mopts, 0, sizeof(mopts)); +#endif + /* * Default options */ mopts.malloc_mutexes = 8; +#ifndef __HAIKU__ mopts.def_malloc_junk = 1; +#endif mopts.def_maxcache = MALLOC_DEFAULT_CACHE; +#ifndef __HAIKU__ for (i = 0; i < 3; i++) { switch (i) { case 0: @@ -553,6 +598,7 @@ omalloc_init(void) } } } +#endif #ifdef MALLOC_STATS if (DO_STATS && (atexit(malloc_exit) == -1)) { @@ -615,6 +661,10 @@ omalloc_grow(struct dir_info *d) if (p == MAP_FAILED) return 1; +#ifdef __HAIKU__ + memset(p, 0, newsize); +#endif + STATS_ADD(d->malloc_used, newsize); STATS_ZERO(d->inserts); STATS_ZERO(d->insert_collisions); @@ -961,7 +1011,12 @@ map(struct dir_info *d, size_t sz, int zero_fill) mprotect(p, (cache->max - 1) * sz, PROT_NONE); p = (char*)p + (cache->max - 1) * sz; +#ifdef __HAIKU__ + if (zero_fill) + memset(p, 0, sz); +#else /* zero fill not needed, freshly mmapped */ +#endif return p; } } @@ -970,7 +1025,12 @@ map(struct dir_info *d, size_t sz, int zero_fill) p = MMAP(sz, d->mmap_flag); if (p != MAP_FAILED) STATS_ADD(d->malloc_used, sz); +#ifdef __HAIKU__ + if (zero_fill) + memset(p, 0, sz); +#else /* zero fill not needed */ +#endif return p; } @@ -1020,6 +1080,9 @@ alloc_chunk_info(struct dir_info *d, u_int bucket) q = MMAP(MALLOC_PAGESIZE * chunk_pages, d->mmap_flag); if (q == MAP_FAILED) return NULL; +#ifdef __HAIKU__ + memset(q, 0, MALLOC_PAGESIZE * chunk_pages); +#endif d->chunk_pages = q; d->chunk_pages_used = 0; STATS_ADD(d->malloc_used, MALLOC_PAGESIZE * @@ -1415,6 +1478,9 @@ malloc_recurse(struct dir_info *d) errno = EDEADLK; } +#ifdef __HAIKU__ +static +#endif void _malloc_init(int from_rthreads) { @@ -1491,6 +1557,9 @@ _malloc_init(int from_rthreads) sz += d->bigcache_size * sizeof(struct bigcache); if (sz > 0) { void *p = MMAP(sz, 0); +#ifdef __HAIKU__ + memset(p, 0, sz); +#endif if (p == MAP_FAILED) wrterror(NULL, "malloc_init mmap2 failed"); @@ -1506,6 +1575,9 @@ _malloc_init(int from_rthreads) } } d->mutex = i; +#ifdef __HAIKU__ /* cross-thread free optimization */ + d->last_found_pool = -1; +#endif } _MALLOC_UNLOCK(1); @@ -1515,7 +1587,7 @@ DEF_STRONG(_malloc_init); #define PROLOGUE(p, fn) \ d = (p); \ if (d == NULL) { \ - _malloc_init(0); \ + /* _malloc_init(0); */ \ d = (p); \ } \ _MALLOC_LOCK(d->mutex); \ @@ -1548,6 +1620,7 @@ malloc(size_t size) } DEF_STRONG(malloc); +#ifndef __HAIKU__ void * malloc_conceal(size_t size) { @@ -1562,6 +1635,7 @@ malloc_conceal(size_t size) return r; } DEF_WEAK(malloc_conceal); +#endif static struct region_info * findpool(void *p, struct dir_info *argpool, struct dir_info **foundpool, @@ -1572,11 +1646,30 @@ findpool(void *p, struct dir_info *argpool, struct dir_info **foundpool, if (r == NULL) { u_int i, nmutexes; + int first, skip; nmutexes = mopts.malloc_pool[1]->malloc_mt ? mopts.malloc_mutexes : 2; +#ifdef __HAIKU__ /* cross-thread free optimization */ + first = argpool->last_found_pool; + skip = -1; +#endif for (i = 1; i < nmutexes; i++) { +#ifdef __HAIKU__ /* cross-thread free optimization */ + u_int j; + if (first >= 0) { + j = first & (nmutexes - 1); + skip = j; + first = -1; + i--; + } else { + j = (argpool->mutex + i) & (nmutexes - 1); + if (skip >= 0 && j == skip) + continue; + } +#else u_int j = (argpool->mutex + i) & (nmutexes - 1); +#endif pool->active--; _MALLOC_UNLOCK(pool->mutex); @@ -1587,6 +1680,9 @@ findpool(void *p, struct dir_info *argpool, struct dir_info **foundpool, if (r != NULL) { *saved_function = pool->func; pool->func = argpool->func; +#ifdef __HAIKU__ /* cross-thread free optimization */ + argpool->last_found_pool = j; +#endif break; } } @@ -1758,6 +1854,9 @@ freezero_p(void *ptr, size_t sz) free(ptr); } +#ifdef __HAIKU__ +static +#endif void freezero(void *ptr, size_t sz) { @@ -1974,6 +2073,42 @@ realloc(void *ptr, size_t size) } DEF_STRONG(realloc); +#ifdef __HAIKU__ +static size_t +omalloc_usable_size(struct dir_info **argpool, void *p) +{ + struct region_info *r; + struct dir_info *pool; + const char *saved_function; + size_t sz; + + r = findpool(p, *argpool, &pool, &saved_function); + + REALSIZE(sz, r); + + if (*argpool != pool) { + pool->func = saved_function; + *argpool = pool; + } + return sz; +} + +size_t +malloc_usable_size(void *ptr) +{ + struct dir_info *d; + size_t r; + int saved_errno = errno; + + PROLOGUE(getpool(), "malloc_usable_size") + SET_CALLER(d, caller(d)); + r = omalloc_usable_size(&d, ptr); + EPILOGUE() + return r; +} +DEF_STRONG(malloc_usable_size); +#endif + /* * This is sqrt(SIZE_MAX+1), as s1*s2 <= SIZE_MAX * if both s1 < MUL_NO_OVERFLOW and s2 < MUL_NO_OVERFLOW @@ -2006,6 +2141,7 @@ calloc(size_t nmemb, size_t size) } DEF_STRONG(calloc); +#ifndef __HAIKU__ void * calloc_conceal(size_t nmemb, size_t size) { @@ -2189,6 +2325,7 @@ recallocarray(void *ptr, size_t oldnmemb, size_t newnmemb, size_t size) return r; } DEF_WEAK(recallocarray); +#endif static void * mapalign(struct dir_info *d, size_t alignment, size_t sz, int zero_fill) diff --git a/src/system/libroot/posix/malloc/openbsd/wrapper.c b/src/system/libroot/posix/malloc/openbsd/wrapper.c new file mode 100644 index 0000000000..bbfe925093 --- /dev/null +++ b/src/system/libroot/posix/malloc/openbsd/wrapper.c @@ -0,0 +1,278 @@ +/* + * Copyright 2024, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "PagesAllocator.h" + + +/* generic stuff */ +#if B_PAGE_SIZE == 4096 +#define _MAX_PAGE_SHIFT 12 +#endif + +extern char* __progname; + +#define MAP_CONCEAL (0) +#define __MAP_NOREPLACE (0) + +#define DEF_STRONG(X) +#define DEF_WEAK(x) + + +/* entropy routines, wrapped for malloc */ + +static uint32_t +malloc_arc4random() +{ + // TODO: Improve this? + const uintptr_t address = (uintptr_t)&address; + uint32_t random = (uint32_t)address * 1103515245; + random += (uint32_t)system_time(); + return random; +} +#define arc4random malloc_arc4random + + +static void +malloc_arc4random_buf(void *_buf, size_t nbytes) +{ + uint8* buf = (uint8*)_buf; + while (nbytes > 0) { + uint32_t value = malloc_arc4random(); + const size_t copy = (nbytes > 4) ? 4 : nbytes; + memcpy(buf, &value, copy); + nbytes -= copy; + buf += copy; + } +} +#define arc4random_buf malloc_arc4random_buf + + +static uint32_t +malloc_arc4random_uniform(uint32_t upper_bound) +{ + return (malloc_arc4random() % upper_bound); +} +#define arc4random_uniform malloc_arc4random_uniform + + +/* memory routines, wrapped for malloc */ + +static inline void +malloc_explicit_bzero(void* buf, size_t len) +{ + memset(buf, 0, len); +} +#define explicit_bzero malloc_explicit_bzero + + +static int +malloc_mprotect(void* address, size_t length, int protection) +{ + /* do nothing */ + return 0; +} +#define mprotect malloc_mprotect + + +static int +mimmutable(void* address, size_t length) +{ + /* do nothing */ + return 0; +} + + +/* memory mapping */ + + +static void* +malloc_mmap(void* address, size_t length, int protection, int flags, int fd, off_t offset) +{ + status_t status; + length = (length + (B_PAGE_SIZE - 1)) & ~(B_PAGE_SIZE - 1);; + + status = __allocate_pages(&address, length, flags); + if (status != B_OK) { + __set_errno(status); + return MAP_FAILED; + } + return address; +} +#define mmap malloc_mmap + + +static int +malloc_munmap(void* address, size_t length) +{ + status_t status = __free_pages(address, length); + if (status < 0) { + errno = status; + return -1; + } + return 0; +} +#define munmap malloc_munmap + + +/* public methods */ + +void* +memalign(size_t align, size_t len) +{ + void* result = NULL; + int status; + + if (align < sizeof(void*)) + align = sizeof(void*); + + status = posix_memalign(&result, align, len); + if (status != 0) { + errno = status; + return NULL; + } + return result; +} + + +void* +valloc(size_t size) +{ + return memalign(B_PAGE_SIZE, size); +} + + +/* malloc implementation */ + +#define _MALLOC_MUTEXES 32 +static mutex sMallocMutexes[_MALLOC_MUTEXES]; +static pthread_once_t sThreadedMallocInitOnce = PTHREAD_ONCE_INIT; + +static int32 sNextMallocThreadID = 1; + +static u_int mopts_nmutexes(); +static void _malloc_init(int from_rthreads); + + +static inline void +_MALLOC_LOCK(int32 index) +{ + mutex_lock(&sMallocMutexes[index]); +} + + +static inline void +_MALLOC_UNLOCK(int32 index) +{ + mutex_unlock(&sMallocMutexes[index]); +} + + +static void +init_threaded_malloc() +{ + u_int i; + for (i = 2; i < _MALLOC_MUTEXES; i++) + mutex_init(&sMallocMutexes[i], "heap mutex"); + + _MALLOC_LOCK(0); + _malloc_init(1); + _MALLOC_UNLOCK(0); +} + + +status_t +__init_heap() +{ + tls_set(TLS_MALLOC_SLOT, (void*)0); + __init_pages_allocator(); + mutex_init(&sMallocMutexes[0], "heap mutex"); + mutex_init(&sMallocMutexes[1], "heap mutex"); + _malloc_init(0); + return B_OK; +} + + +static int32 +get_thread_malloc_id() +{ + int32 result = (int32)(intptr_t)tls_get(TLS_MALLOC_SLOT); + if (result == -1) { + // thread has never called malloc() before; assign it an ID. + result = atomic_add(&sNextMallocThreadID, 1); + tls_set(TLS_MALLOC_SLOT, (void*)(intptr_t)result); + } + return result; +} + + +void +__heap_thread_init() +{ + pthread_once(&sThreadedMallocInitOnce, &init_threaded_malloc); + tls_set(TLS_MALLOC_SLOT, (void*)(intptr_t)-1); +} + + +void +__heap_thread_exit() +{ + const int32 id = (int32)(intptr_t)tls_get(TLS_MALLOC_SLOT); + if (id != -1 && id == (sNextMallocThreadID - 1)) { + // Try to "de-allocate" this thread's ID. + atomic_test_and_set(&sNextMallocThreadID, id, id + 1); + } +} + + +void +__heap_before_fork() +{ + u_int i; + u_int nmutexes = mopts_nmutexes(); + for (i = 0; i < nmutexes; i++) + _MALLOC_LOCK(i); + + __pages_allocator_before_fork(); +} + + +void +__heap_after_fork_child() +{ + u_int i; + u_int nmutexes = mopts_nmutexes(); + for (i = 0; i < nmutexes; i++) + mutex_init(&sMallocMutexes[i], "heap mutex"); + + __pages_allocator_after_fork(0); +} + + +void +__heap_after_fork_parent() +{ + u_int i; + u_int nmutexes = mopts_nmutexes(); + for (i = 0; i < nmutexes; i++) + _MALLOC_UNLOCK(i); + + __pages_allocator_after_fork(1); +} + + +void +__heap_terminate_after() +{ +}