diff --git a/headers/private/kernel/slab/Slab.h b/headers/private/kernel/slab/Slab.h index e422b73246..4f1f8ceea2 100644 --- a/headers/private/kernel/slab/Slab.h +++ b/headers/private/kernel/slab/Slab.h @@ -46,6 +46,9 @@ object_cache *create_object_cache_etc(const char *name, size_t object_size, void delete_object_cache(object_cache *cache); +status_t object_cache_set_minimum_reserve(object_cache *cache, + size_t objectCount); + void *object_cache_alloc(object_cache *cache, uint32 flags); void object_cache_free(object_cache *cache, void *object); diff --git a/headers/private/kernel/vm.h b/headers/private/kernel/vm.h index 1a8756fc77..7fcaab3372 100644 --- a/headers/private/kernel/vm.h +++ b/headers/private/kernel/vm.h @@ -40,6 +40,7 @@ addr_t vm_allocate_early(struct kernel_args *args, size_t virtualSize, void slab_init(struct kernel_args *args, addr_t initialBase, size_t initialSize); void slab_init_post_sem(); +void slab_init_post_thread(); // to protect code regions with interrupts turned on void permit_page_faults(void); diff --git a/src/system/kernel/slab/Slab.cpp b/src/system/kernel/slab/Slab.cpp index b86869ba9a..bf99f1a999 100644 --- a/src/system/kernel/slab/Slab.cpp +++ b/src/system/kernel/slab/Slab.cpp @@ -11,11 +11,13 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -54,12 +56,14 @@ struct object_link { struct slab : DoublyLinkedListLinkImpl { void *pages; - size_t count, size; + size_t size; // total number of objects + size_t count; // free objects size_t offset; object_link *free; }; typedef DoublyLinkedList SlabList; +struct ResizeRequest; struct object_cache : DoublyLinkedListLinkImpl { char name[32]; @@ -67,24 +71,30 @@ struct object_cache : DoublyLinkedListLinkImpl { size_t object_size; size_t cache_color_cycle; SlabList empty, partial, full; - size_t used_count, empty_count, pressure; + size_t total_objects; // total number of objects + size_t used_count; // used objects + size_t empty_count; // empty slabs + size_t pressure; + size_t min_object_reserve; // minimum number of free objects size_t slab_size; size_t usage, maximum; uint32 flags; + ResizeRequest *resize_request; + void *cookie; object_cache_constructor constructor; object_cache_destructor destructor; object_cache_reclaimer reclaimer; status_t (*allocate_pages)(object_cache *cache, void **pages, - uint32 flags); + uint32 flags, bool unlockWhileAllocating); void (*free_pages)(object_cache *cache, void *pages); object_depot depot; - virtual slab *CreateSlab(uint32 flags) = 0; + virtual slab *CreateSlab(uint32 flags, bool unlockWhileAllocating) = 0; virtual void ReturnSlab(slab *slab) = 0; virtual slab *ObjectSlab(void *object) const = 0; @@ -95,12 +105,15 @@ struct object_cache : DoublyLinkedListLinkImpl { virtual void UnprepareObject(slab *source, void *object) {} virtual ~object_cache() {} + + bool Lock() { return mutex_lock(&lock) == B_OK; } + void Unlock() { mutex_unlock(&lock); } }; typedef DoublyLinkedList ObjectCacheList; struct SmallObjectCache : object_cache { - slab *CreateSlab(uint32 flags); + slab *CreateSlab(uint32 flags, bool unlockWhileAllocating); void ReturnSlab(slab *slab); slab *ObjectSlab(void *object) const; }; @@ -139,7 +152,7 @@ struct HashedObjectCache : object_cache { HashedObjectCache() : hash_table(this) {} - slab *CreateSlab(uint32 flags); + slab *CreateSlab(uint32 flags, bool unlockWhileAllocating); void ReturnSlab(slab *slab); slab *ObjectSlab(void *object) const; status_t PrepareObject(slab *source, void *object); @@ -162,6 +175,22 @@ struct depot_cpu_store { struct depot_magazine *loaded, *previous; }; +struct ResizeRequest : DoublyLinkedListLinkImpl { + ResizeRequest(object_cache* cache) + : + cache(cache), + pending(false), + delete_when_done(false) + { + } + + object_cache* cache; + bool pending; + bool delete_when_done; +}; + +typedef DoublyLinkedList ResizeRequestQueue; + static ObjectCacheList sObjectCaches; static mutex sObjectCacheListLock = MUTEX_INITIALIZER("object cache list"); @@ -171,10 +200,15 @@ static kernel_args *sKernelArgs; static status_t object_cache_reserve_internal(object_cache *cache, - size_t object_count, uint32 flags); + size_t object_count, uint32 flags, bool unlockWhileAllocating); static depot_magazine *alloc_magazine(); static void free_magazine(depot_magazine *magazine); +static mutex sResizeRequestsLock + = MUTEX_INITIALIZER("object cache resize requests"); +static ResizeRequestQueue sResizeRequests; +static ConditionVariable sResizeRequestsCondition; + #if OBJECT_CACHE_TRACING @@ -405,7 +439,8 @@ internal_free(void *_buffer) static status_t -area_allocate_pages(object_cache *cache, void **pages, uint32 flags) +area_allocate_pages(object_cache *cache, void **pages, uint32 flags, + bool unlockWhileAllocating) { TRACE_CACHE(cache, "allocate pages (%lu, 0x0%lx)", cache->slab_size, flags); @@ -418,11 +453,18 @@ area_allocate_pages(object_cache *cache, void **pages, uint32 flags) && cache->slab_size != B_PAGE_SIZE) addressSpec = B_ANY_KERNEL_BLOCK_ADDRESS; + if (unlockWhileAllocating) + cache->Unlock(); + // if we are allocating, it is because we need the pages immediatly // so we lock them. when moving the slab to the empty list we should // unlock them, and lock them again when getting one from the empty list. area_id areaId = create_area(cache->name, pages, addressSpec, cache->slab_size, lock, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + + if (unlockWhileAllocating) + cache->Lock(); + if (areaId < 0) return areaId; @@ -451,14 +493,21 @@ area_free_pages(object_cache *cache, void *pages) static status_t -early_allocate_pages(object_cache *cache, void **pages, uint32 flags) +early_allocate_pages(object_cache *cache, void **pages, uint32 flags, + bool unlockWhileAllocating) { TRACE_CACHE(cache, "early allocate pages (%lu, 0x0%lx)", cache->slab_size, flags); + if (unlockWhileAllocating) + cache->Unlock(); + addr_t base = vm_allocate_early(sKernelArgs, cache->slab_size, cache->slab_size, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + if (unlockWhileAllocating) + cache->Lock(); + *pages = (void *)base; cache->usage += cache->slab_size; @@ -558,15 +607,19 @@ object_cache_init(object_cache *cache, const char *name, size_t objectSize, cache->object_size); cache->cache_color_cycle = 0; + cache->total_objects = 0; cache->used_count = 0; cache->empty_count = 0; cache->pressure = 0; + cache->min_object_reserve = 0; cache->usage = 0; cache->maximum = maximum; cache->flags = flags; + cache->resize_request = NULL; + // TODO: depot destruction is obviously broken // no gain in using the depot in single cpu setups //if (smp_get_num_cpus() == 1) @@ -634,6 +687,67 @@ object_cache_commit_pre_pages(object_cache *cache) } +static status_t +object_cache_resizer(void*) +{ + while (true) { + MutexLocker locker(sResizeRequestsLock); + + // wait for the next request + while (sResizeRequests.IsEmpty()) { + ConditionVariableEntry entry; + sResizeRequestsCondition.Add(&entry); + locker.Unlock(); + entry.Wait(); + locker.Lock(); + } + + ResizeRequest* request = sResizeRequests.RemoveHead(); + + locker.Unlock(); + + // resize the cache, if necessary + + object_cache* cache = request->cache; + + MutexLocker cacheLocker(cache->lock); + + size_t freeObjects = cache->total_objects - cache->used_count; + + while (freeObjects < cache->min_object_reserve) { + status_t error = object_cache_reserve_internal(cache, + cache->min_object_reserve - freeObjects, 0, true); + if (error != B_OK) { + dprintf("object cache resizer: Failed to resize object cache " + "%p!\n", cache); + break; + } + + freeObjects = cache->total_objects - cache->used_count; + } + + request->pending = false; + + if (request->delete_when_done) + delete request; + } +} + + +static void +increase_object_reserve(object_cache* cache) +{ + if (cache->resize_request->pending) + return; + + cache->resize_request->pending = true; + + MutexLocker locker(sResizeRequestsLock); + sResizeRequests.Add(cache->resize_request); + sResizeRequestsCondition.NotifyAll(); +} + + static void delete_cache(object_cache *cache) { @@ -762,6 +876,35 @@ delete_object_cache(object_cache *cache) } +status_t +object_cache_set_minimum_reserve(object_cache *cache, size_t objectCount) +{ + MutexLocker _(cache->lock); + + if (cache->min_object_reserve == objectCount) + return B_OK; + + if (cache->min_object_reserve == 0) { + cache->resize_request = new(std::nothrow) ResizeRequest(cache); + if (cache->resize_request == NULL) + return B_NO_MEMORY; + } else if (cache->min_object_reserve == 0) { + if (cache->resize_request->pending) + cache->resize_request->delete_when_done = true; + else + delete cache->resize_request; + + cache->resize_request = NULL; + } + + cache->min_object_reserve = objectCount; + + increase_object_reserve(cache); + + return B_OK; +} + + void * object_cache_alloc(object_cache *cache, uint32 flags) { @@ -778,7 +921,7 @@ object_cache_alloc(object_cache *cache, uint32 flags) if (cache->partial.IsEmpty()) { if (cache->empty.IsEmpty()) { - if (object_cache_reserve_internal(cache, 1, flags) < B_OK) { + if (object_cache_reserve_internal(cache, 1, flags, false) < B_OK) { T(Alloc(cache, flags, NULL)); return NULL; } @@ -799,6 +942,9 @@ object_cache_alloc(object_cache *cache, uint32 flags) source->count--; cache->used_count++; + if (cache->total_objects - cache->used_count < cache->min_object_reserve) + increase_object_reserve(cache); + REMOVE_PARANOIA_CHECK(PARANOIA_SUSPICIOUS, source, &link->next, sizeof(void*)); @@ -839,7 +985,9 @@ object_cache_return_to_slab(object_cache *cache, slab *source, void *object) if (source->count == source->size) { cache->partial.Remove(source); - if (cache->empty_count < cache->pressure) { + if (cache->empty_count < cache->pressure + && cache->total_objects - cache->used_count - source->size + >= cache->min_object_reserve) { cache->empty_count++; cache->empty.Add(source); } else { @@ -871,14 +1019,16 @@ object_cache_free(object_cache *cache, void *object) static status_t -object_cache_reserve_internal(object_cache *cache, size_t object_count, - uint32 flags) +object_cache_reserve_internal(object_cache *cache, size_t objectCount, + uint32 flags, bool unlockWhileAllocating) { - size_t numBytes = object_count * cache->object_size; + size_t numBytes = objectCount * cache->object_size; size_t slabCount = ((numBytes - 1) / cache->slab_size) + 1; + // TODO: This also counts the unusable space of each slab, which can + // sum up. while (slabCount > 0) { - slab *newSlab = cache->CreateSlab(flags); + slab *newSlab = cache->CreateSlab(flags, unlockWhileAllocating); if (newSlab == NULL) return B_NO_MEMORY; @@ -900,7 +1050,7 @@ object_cache_reserve(object_cache *cache, size_t objectCount, uint32 flags) T(Reserve(cache, objectCount, flags)); MutexLocker _(cache->lock); - return object_cache_reserve_internal(cache, objectCount, flags); + return object_cache_reserve_internal(cache, objectCount, flags, false); } @@ -921,6 +1071,7 @@ object_cache::InitSlab(slab *slab, void *pages, size_t byteCount) slab->pages = pages; slab->count = slab->size = byteCount / object_size; slab->free = NULL; + total_objects += slab->size; size_t spareBytes = byteCount - (slab->size * object_size); slab->offset = cache_color_cycle; @@ -983,6 +1134,8 @@ object_cache::UninitSlab(slab *slab) if (slab->count != slab->size) panic("cache: destroying a slab which isn't empty."); + total_objects -= slab->size; + DELETE_PARANOIA_CHECK_SET(slab); uint8 *data = ((uint8 *)slab->pages) + slab->offset; @@ -1022,14 +1175,14 @@ check_cache_quota(object_cache *cache) slab * -SmallObjectCache::CreateSlab(uint32 flags) +SmallObjectCache::CreateSlab(uint32 flags, bool unlockWhileAllocating) { if (!check_cache_quota(this)) return NULL; void *pages; - if (allocate_pages(this, &pages, flags) < B_OK) + if (allocate_pages(this, &pages, flags, unlockWhileAllocating) < B_OK) return NULL; return InitSlab(slab_in_pages(pages, slab_size), pages, @@ -1082,18 +1235,25 @@ free_link(HashedObjectCache::Link *link) slab * -HashedObjectCache::CreateSlab(uint32 flags) +HashedObjectCache::CreateSlab(uint32 flags, bool unlockWhileAllocating) { if (!check_cache_quota(this)) return NULL; + if (unlockWhileAllocating) + Unlock(); + slab *slab = allocate_slab(flags); + + if (unlockWhileAllocating) + Lock(); + if (slab == NULL) return NULL; void *pages; - if (allocate_pages(this, &pages, flags) == B_OK) { + if (allocate_pages(this, &pages, flags, unlockWhileAllocating) == B_OK) { if (InitSlab(slab, pages, slab_size)) return slab; @@ -1474,3 +1634,20 @@ slab_init_post_sem() block_allocator_init_rest(); } + +void +slab_init_post_thread() +{ + new(&sResizeRequests) ResizeRequestQueue; + sResizeRequestsCondition.Init(&sResizeRequests, "object cache resizer"); + + thread_id objectCacheResizer = spawn_kernel_thread(object_cache_resizer, + "object cache resizer", B_URGENT_PRIORITY, NULL); + if (objectCacheResizer < 0) { + panic("slab_init_post_thread(): failed to spawn object cache resizer " + "thread\n"); + return; + } + + send_signal_etc(objectCacheResizer, SIGCONT, B_DO_NOT_RESCHEDULE); +} diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index b20b66d81d..5e78c7a735 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -4049,6 +4049,7 @@ vm_init_post_thread(kernel_args *args) { vm_page_init_post_thread(args); vm_daemon_init(); + slab_init_post_thread(); return heap_init_post_thread(); }