diff --git a/src/system/kernel/slab/MemoryManager.cpp b/src/system/kernel/slab/MemoryManager.cpp index fece68df08..7b0e335a60 100644 --- a/src/system/kernel/slab/MemoryManager.cpp +++ b/src/system/kernel/slab/MemoryManager.cpp @@ -422,6 +422,28 @@ private: #endif // SLAB_MEMORY_MANAGER_TRACING +// #pragma mark - utility methods + + +template +static inline Type* +_pop(Type*& head) +{ + Type* oldHead = head; + head = head->next; + return oldHead; +} + + +template +static inline void +_push(Type*& head, Type* object) +{ + object->next = head; + head = object; +} + + // #pragma mark - MemoryManager @@ -1441,6 +1463,25 @@ MemoryManager::_FreeArea(Area* area, bool areaRemoved, uint32 flags) } +/*static*/ inline void +MemoryManager::_PushFreeArea(Area* area) +{ + _push(sFreeAreas, area); + sFreeAreaCount++; +} + + +/*static*/ inline MemoryManager::Area* +MemoryManager::_PopFreeArea() +{ + if (sFreeAreaCount == 0) + return NULL; + + sFreeAreaCount--; + return _pop(sFreeAreas); +} + + /*static*/ status_t MemoryManager::_MapChunk(VMArea* vmArea, addr_t address, size_t size, size_t reserveAdditionalMemory, uint32 flags) diff --git a/src/system/kernel/slab/MemoryManager.h b/src/system/kernel/slab/MemoryManager.h index f46fe14e0f..dbf8384ddd 100644 --- a/src/system/kernel/slab/MemoryManager.h +++ b/src/system/kernel/slab/MemoryManager.h @@ -239,25 +239,6 @@ MemoryManager::MaintenanceNeeded() } -/*static*/ inline void -MemoryManager::_PushFreeArea(Area* area) -{ - _push(sFreeAreas, area); - sFreeAreaCount++; -} - - -/*static*/ inline MemoryManager::Area* -MemoryManager::_PopFreeArea() -{ - if (sFreeAreaCount == 0) - return NULL; - - sFreeAreaCount--; - return _pop(sFreeAreas); -} - - /*static*/ inline addr_t MemoryManager::_AreaBaseAddressForAddress(addr_t address) { diff --git a/src/system/kernel/slab/ObjectCache.cpp b/src/system/kernel/slab/ObjectCache.cpp index 0ff1195bd4..1ef7625613 100644 --- a/src/system/kernel/slab/ObjectCache.cpp +++ b/src/system/kernel/slab/ObjectCache.cpp @@ -51,8 +51,8 @@ ObjectCache::Init(const char* name, size_t objectSize, size_t alignment, mutex_init(&lock, this->name); - if (objectSize < sizeof(object_link)) - objectSize = sizeof(object_link); + if (objectSize < sizeof(slab_queue_link)) + objectSize = sizeof(slab_queue_link); if (alignment < kMinObjectAlignment) alignment = kMinObjectAlignment; @@ -125,7 +125,7 @@ ObjectCache::InitSlab(slab* slab, void* pages, size_t byteCount, uint32 flags) slab->pages = pages; slab->count = slab->size = byteCount / object_size; - slab->free = NULL; + slab->free.Init(); size_t spareBytes = byteCount - (slab->size * object_size); @@ -142,7 +142,6 @@ ObjectCache::InitSlab(slab* slab, void* pages, size_t byteCount, uint32 flags) CREATE_PARANOIA_CHECK_SET(slab, "slab"); - for (size_t i = 0; i < slab->size; i++) { status_t status = B_OK; if (constructor) @@ -157,11 +156,10 @@ ObjectCache::InitSlab(slab* slab, void* pages, size_t byteCount, uint32 flags) } DELETE_PARANOIA_CHECK_SET(slab); - return NULL; } - _push(slab->free, object_to_link(data, object_size)); + slab->free.Push(object_to_link(data, object_size)); ADD_PARANOIA_CHECK(PARANOIA_SUSPICIOUS, slab, &object_to_link(data, object_size)->next, sizeof(void*)); @@ -216,13 +214,13 @@ ObjectCache::ReturnObjectToSlab(slab* source, void* object, uint32 flags) } #endif // KDEBUG - object_link* link = object_to_link(object, object_size); + slab_queue_link* link = object_to_link(object, object_size); TRACE_CACHE(this, "returning %p (%p) to %p, %lu used (%lu empty slabs).", object, link, source, source->size - source->count, empty_count); - _push(source->free, link); + source->free.Push(link); source->count++; used_count--; @@ -267,8 +265,8 @@ ObjectCache::AssertObjectNotFreed(void* object) return false; } - object_link* link = object_to_link(object, object_size); - for (object_link* freeLink = source->free; freeLink != NULL; + slab_queue_link* link = object_to_link(object, object_size); + for (slab_queue_link* freeLink = source->free.head; freeLink != NULL; freeLink = freeLink->next) { if (freeLink == link) { panic("object_cache: double free of %p (slab %p, cache %p)", diff --git a/src/system/kernel/slab/ObjectCache.h b/src/system/kernel/slab/ObjectCache.h index 1dac79088b..84ba05ab28 100644 --- a/src/system/kernel/slab/ObjectCache.h +++ b/src/system/kernel/slab/ObjectCache.h @@ -10,10 +10,12 @@ #include #include -#include #include #include +#include "ObjectDepot.h" +#include "slab_queue.h" + #include "kernel_debug_config.h" #include "slab_debug.h" @@ -21,16 +23,12 @@ struct ResizeRequest; -struct object_link { - struct object_link* next; -}; - struct slab : DoublyLinkedListLinkImpl { void* pages; size_t size; // total number of objects size_t count; // free objects size_t offset; - object_link* free; + slab_queue free; #if SLAB_OBJECT_CACHE_ALLOCATION_TRACKING AllocationTrackingInfo* tracking; #endif @@ -131,17 +129,17 @@ public: static inline void* -link_to_object(object_link* link, size_t objectSize) +link_to_object(slab_queue_link* link, size_t objectSize) { - return ((uint8*)link) - (objectSize - sizeof(object_link)); + return ((uint8*)link) - (objectSize - sizeof(slab_queue_link)); } -static inline object_link* +static inline slab_queue_link* object_to_link(void* object, size_t objectSize) { - return (object_link*)(((uint8*)object) - + (objectSize - sizeof(object_link))); + return (slab_queue_link*)(((uint8*)object) + + (objectSize - sizeof(slab_queue_link))); } diff --git a/src/system/kernel/slab/ObjectDepot.cpp b/src/system/kernel/slab/ObjectDepot.cpp index 716dafda9d..e49540b68c 100644 --- a/src/system/kernel/slab/ObjectDepot.cpp +++ b/src/system/kernel/slab/ObjectDepot.cpp @@ -7,7 +7,7 @@ */ -#include +#include "ObjectDepot.h" #include @@ -18,10 +18,10 @@ #include "slab_debug.h" #include "slab_private.h" +#include "slab_queue.h" -struct DepotMagazine { - DepotMagazine* next; +struct DepotMagazine : public slab_queue_link { uint16 current_round; uint16 round_count; void* rounds[0]; @@ -138,14 +138,14 @@ exchange_with_full(object_depot* depot, DepotMagazine*& magazine) SpinLocker _(depot->inner_lock); - if (depot->full == NULL) + if (depot->full.head == NULL) return false; depot->full_count--; depot->empty_count++; - _push(depot->empty, magazine); - magazine = _pop(depot->full); + depot->empty.Push(magazine); + magazine = (DepotMagazine*)depot->full.Pop(); return true; } @@ -158,21 +158,21 @@ exchange_with_empty(object_depot* depot, DepotMagazine*& magazine, SpinLocker _(depot->inner_lock); - if (depot->empty == NULL) + if (depot->empty.head == NULL) return false; depot->empty_count--; if (magazine != NULL) { if (depot->full_count < depot->max_count) { - _push(depot->full, magazine); + depot->full.Push(magazine); depot->full_count++; freeMagazine = NULL; } else freeMagazine = magazine; } - magazine = _pop(depot->empty); + magazine = (DepotMagazine*)depot->empty.Pop(); return true; } @@ -182,7 +182,7 @@ push_empty_magazine(object_depot* depot, DepotMagazine* magazine) { SpinLocker _(depot->inner_lock); - _push(depot->empty, magazine); + depot->empty.Push(magazine); depot->empty_count++; } @@ -202,8 +202,8 @@ object_depot_init(object_depot* depot, size_t capacity, size_t maxCount, uint32 flags, void* cookie, void (*return_object)(object_depot* depot, void* cookie, void* object, uint32 flags)) { - depot->full = NULL; - depot->empty = NULL; + depot->full.Init(); + depot->empty.Init(); depot->full_count = depot->empty_count = 0; depot->max_count = maxCount; depot->magazine_capacity = capacity; @@ -336,43 +336,44 @@ object_depot_make_empty(object_depot* depot, uint32 flags) // collect the store magazines - DepotMagazine* storeMagazines = NULL; + slab_queue storeMagazines; + storeMagazines.Init(); int cpuCount = smp_get_num_cpus(); for (int i = 0; i < cpuCount; i++) { depot_cpu_store& store = depot->stores[i]; - if (store.loaded) { - _push(storeMagazines, store.loaded); + if (store.loaded != NULL) { + storeMagazines.Push(store.loaded); store.loaded = NULL; } - if (store.previous) { - _push(storeMagazines, store.previous); + if (store.previous != NULL) { + storeMagazines.Push(store.previous); store.previous = NULL; } } // detach the depot's full and empty magazines - DepotMagazine* fullMagazines = depot->full; - depot->full = NULL; + slab_queue fullMagazines = depot->full; + depot->full.head = depot->full.tail = NULL; - DepotMagazine* emptyMagazines = depot->empty; - depot->empty = NULL; + slab_queue emptyMagazines = depot->empty; + depot->empty.head = depot->empty.tail = NULL; writeLocker.Unlock(); // free all magazines - while (storeMagazines != NULL) - empty_magazine(depot, _pop(storeMagazines), flags); + while (storeMagazines.head != NULL) + empty_magazine(depot, (DepotMagazine*)storeMagazines.Pop(), flags); - while (fullMagazines != NULL) - empty_magazine(depot, _pop(fullMagazines), flags); + while (fullMagazines.head != NULL) + empty_magazine(depot, (DepotMagazine*)fullMagazines.Pop(), flags); - while (emptyMagazines) - free_magazine(_pop(emptyMagazines), flags); + while (emptyMagazines.head != NULL) + free_magazine((DepotMagazine*)emptyMagazines.Pop(), flags); } @@ -398,8 +399,8 @@ object_depot_contains_object(object_depot* depot, void* object) } } - for (DepotMagazine* magazine = depot->full; magazine != NULL; - magazine = magazine->next) { + for (DepotMagazine* magazine = (DepotMagazine*)depot->full.head; magazine != NULL; + magazine = (DepotMagazine*)magazine->next) { if (magazine->ContainsObject(object)) return true; } @@ -416,8 +417,8 @@ object_depot_contains_object(object_depot* depot, void* object) void dump_object_depot(object_depot* depot) { - kprintf(" full: %p, count %lu\n", depot->full, depot->full_count); - kprintf(" empty: %p, count %lu\n", depot->empty, depot->empty_count); + kprintf(" full: %p, count %lu\n", depot->full.head, depot->full_count); + kprintf(" empty: %p, count %lu\n", depot->empty.head, depot->empty_count); kprintf(" max full: %lu\n", depot->max_count); kprintf(" capacity: %lu\n", depot->magazine_capacity); kprintf(" stores:\n"); diff --git a/headers/private/kernel/slab/ObjectDepot.h b/src/system/kernel/slab/ObjectDepot.h similarity index 95% rename from headers/private/kernel/slab/ObjectDepot.h rename to src/system/kernel/slab/ObjectDepot.h index 456038aed2..ebfbddf3c6 100644 --- a/headers/private/kernel/slab/ObjectDepot.h +++ b/src/system/kernel/slab/ObjectDepot.h @@ -10,14 +10,16 @@ #include #include +#include "slab_queue.h" + struct DepotMagazine; typedef struct object_depot { rw_lock outer_lock; spinlock inner_lock; - DepotMagazine* full; - DepotMagazine* empty; + slab_queue full; + slab_queue empty; size_t full_count; size_t empty_count; size_t max_count; diff --git a/src/system/kernel/slab/Slab.cpp b/src/system/kernel/slab/Slab.cpp index c4d4e9fae4..d4c57cd249 100644 --- a/src/system/kernel/slab/Slab.cpp +++ b/src/system/kernel/slab/Slab.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -28,11 +27,12 @@ #include #include +#include "SmallObjectCache.h" #include "HashedObjectCache.h" +#include "ObjectDepot.h" #include "MemoryManager.h" #include "slab_debug.h" #include "slab_private.h" -#include "SmallObjectCache.h" #if !USE_GUARDED_HEAP_FOR_OBJECT_CACHE @@ -251,7 +251,7 @@ static void dump_slab(::slab* slab) { kprintf(" %p %p %6" B_PRIuSIZE " %6" B_PRIuSIZE " %6" B_PRIuSIZE " %p\n", - slab, slab->pages, slab->size, slab->count, slab->offset, slab->free); + slab, slab->pages, slab->size, slab->count, slab->offset, slab->free.head); } @@ -1273,7 +1273,7 @@ object_cache_alloc(object_cache* cache, uint32 flags) ParanoiaChecker _2(source); - object_link* link = _pop(source->free); + slab_queue_link* link = source->free.Pop(); source->count--; cache->used_count++; diff --git a/src/system/kernel/slab/slab_private.h b/src/system/kernel/slab/slab_private.h index 985c5ecc2d..7531088406 100644 --- a/src/system/kernel/slab/slab_private.h +++ b/src/system/kernel/slab/slab_private.h @@ -24,25 +24,6 @@ void block_allocator_init_boot(); void block_allocator_init_rest(); -template -static inline Type* -_pop(Type*& head) -{ - Type* oldHead = head; - head = head->next; - return oldHead; -} - - -template -static inline void -_push(Type*& head, Type* object) -{ - object->next = head; - head = object; -} - - static inline void* slab_internal_alloc(size_t size, uint32 flags) { diff --git a/src/system/kernel/slab/slab_queue.h b/src/system/kernel/slab/slab_queue.h new file mode 100644 index 0000000000..c3f333a9f6 --- /dev/null +++ b/src/system/kernel/slab/slab_queue.h @@ -0,0 +1,49 @@ +/* + * Copyright 2025, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + */ +#ifndef SLAB_QUEUE_H +#define SLAB_QUEUE_H + + +#include + + +struct slab_queue_link { + slab_queue_link* next; +}; + +struct slab_queue { + slab_queue_link* head; + slab_queue_link* tail; + + void Init() + { + head = tail = NULL; + } + + void Push(slab_queue_link* item) + { + item->next = NULL; + + if (tail == NULL) { + head = tail = item; + return; + } + + tail->next = item; + tail = item; + } + + slab_queue_link* Pop() + { + slab_queue_link* item = head; + head = item->next; + if (head == NULL) + tail = NULL; + return item; + } +}; + + +#endif // SLAB_QUEUE_H