From 11b5020f2fa2dcc510704cf3fb8dc9c3d3ea1f71 Mon Sep 17 00:00:00 2001 From: Hugo Santos Date: Sat, 28 Apr 2007 18:53:58 +0000 Subject: [PATCH] rewrote the object cache (slab) implementation a bit, preparing for further integration. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@20887 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/private/kernel/slab/Backend.h | 38 - headers/private/kernel/slab/Base.h | 168 +--- headers/private/kernel/slab/Depot.h | 25 +- headers/private/kernel/slab/HashStrategy.h | 172 ---- headers/private/kernel/slab/MergedStrategy.h | 81 -- headers/private/kernel/slab/Slab.h | 85 +- headers/private/kernel/slab/Strategy.h | 56 -- headers/private/kernel/slab/Utilities.h | 45 - headers/private/kernel/vm.h | 1 + .../kernel/network/stack/net_buffer.cpp | 43 +- src/system/kernel/main.c | 2 + src/system/kernel/slab/Jamfile | 1 - src/system/kernel/slab/Slab.cpp | 773 ++++++++++++++---- src/system/kernel/slab/interface.cpp | 100 --- 14 files changed, 647 insertions(+), 943 deletions(-) delete mode 100644 headers/private/kernel/slab/Backend.h delete mode 100644 headers/private/kernel/slab/HashStrategy.h delete mode 100644 headers/private/kernel/slab/MergedStrategy.h delete mode 100644 headers/private/kernel/slab/Strategy.h delete mode 100644 headers/private/kernel/slab/Utilities.h delete mode 100644 src/system/kernel/slab/interface.cpp diff --git a/headers/private/kernel/slab/Backend.h b/headers/private/kernel/slab/Backend.h deleted file mode 100644 index 39848f94ce..0000000000 --- a/headers/private/kernel/slab/Backend.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2007, Hugo Santos. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Hugo Santos, hugosantos@gmail.com - */ - -#ifndef _SLAB_BACKEND_H_ -#define _SLAB_BACKEND_H_ - -#include - -extern "C" { - status_t slab_area_backend_allocate(base_cache *cache, area_id *id, - void **pages, size_t byte_count, uint32_t flags); - void slab_area_backend_free(base_cache *cache, area_id id); -} - -struct AreaBackend { - typedef area_id AllocationID; - - static const size_t kPageSize = B_PAGE_SIZE; - static const size_t kMaximumAlignedLength = B_PAGE_SIZE; - - static status_t AllocatePages(base_cache *cache, area_id *id, void **pages, - size_t byteCount, uint32_t flags) - { - return slab_area_backend_allocate(cache, id, pages, byteCount, flags); - } - - static void FreePages(base_cache *cache, area_id id) - { - return slab_area_backend_free(cache, id); - } -}; - -#endif diff --git a/headers/private/kernel/slab/Base.h b/headers/private/kernel/slab/Base.h index ad862f82ee..b40294d332 100644 --- a/headers/private/kernel/slab/Base.h +++ b/headers/private/kernel/slab/Base.h @@ -9,170 +9,44 @@ #ifndef _SLAB_BASE_SLAB_H_ #define _SLAB_BASE_SLAB_H_ -#include - #include #include -#include -#include -#include -#include - #ifdef __cplusplus -#include // pair<> - extern "C" { #endif +/* create_object_cache_etc flags */ enum { - CACHE_DONT_SLEEP = 1 << 0, - CACHE_ALIGN_TO_TOTAL = 1 << 16, + CACHE_NO_DEPOT = 1 << 0, }; -static const int kMinimumSlabItems = 32; +/* object_cache_alloc flags */ +enum { + CACHE_DONT_SLEEP = 1 << 0, +}; -typedef status_t (*base_cache_constructor)(void *cookie, void *object); -typedef void (*base_cache_destructor)(void *cookie, void *object); +typedef struct object_cache object_cache; -/* base Slab implementation, opaque to the backend used. - * - * NOTE: the caller is responsible for the Cache's locking. - * Cache<> below handles it as well. */ +typedef status_t (*object_cache_constructor)(void *cookie, void *object); +typedef void (*object_cache_destructor)(void *cookie, void *object); +typedef void (*object_cache_reclaimer)(void *cookie, void *object); -typedef struct base_cache { - char name[32]; - size_t object_size; - size_t cache_color_cycle; - struct list empty, partial, full; - size_t empty_count, pressure; - base_cache_constructor constructor; - base_cache_destructor destructor; - void *cookie; -} base_cache; +object_cache *create_object_cache(const char *name, size_t object_size, + size_t alignment, void *cookie, object_cache_constructor constructor, + object_cache_destructor); +object_cache *create_object_cache_etc(const char *name, size_t object_size, + size_t alignment, size_t max_byte_usage, uint32 flags, void *cookie, + object_cache_constructor constructor, object_cache_destructor destructor, + object_cache_reclaimer reclaimer); -typedef struct cache_slab { - void *pages; - size_t count, size; - size_t offset; - struct cache_object_link *free; - struct list_link link; -} cache_slab; +void delete_object_cache(object_cache *cache); -// TODO add reclaim method to base_cache to be called under severe memory -// pressure so the slab owner can free as much buffers as possible. - -void base_cache_init(base_cache *cache, const char *name, size_t object_size, - size_t alignment, base_cache_constructor constructor, - base_cache_destructor destructor, void *cookie); -void base_cache_destroy(base_cache *cache, - void (*return_slab)(base_cache *, cache_slab *)); -void base_cache_low_memory(base_cache *cache, int32 level, - void (*return_slab)(base_cache *, cache_slab *)); - -void *base_cache_allocate_object(base_cache *cache); -void *base_cache_allocate_object_with_new_slab(base_cache *cache, - cache_slab *slab); -int base_cache_return_object(base_cache *cache, cache_slab *slab, - void *object); - -typedef status_t (*base_cache_owner_prepare)(void *parent, - cache_slab *slab, void *object); -typedef void (*base_cache_owner_unprepare)(void *parent, cache_slab *slab, - void *object); - -cache_slab *base_cache_construct_slab(base_cache *cache, cache_slab *slab, - void *pages, size_t byte_count, void *parent, - base_cache_owner_prepare prepare, base_cache_owner_unprepare unprepare); -void base_cache_destruct_slab(base_cache *cache, cache_slab *slab, - void *parent, base_cache_owner_unprepare unprepare); +void *object_cache_alloc(object_cache *cache, uint32 flags); +void object_cache_free(object_cache *cache, void *object); #ifdef __cplusplus } - -// Slab implementation, glues together the frontend, backend as -// well as the Slab strategy used. -template -class Cache : protected base_cache { -public: - typedef Cache ThisCache; - typedef base_cache_constructor Constructor; - typedef base_cache_destructor Destructor; - - Cache(const char *name, size_t objectSize, size_t alignment, - Constructor constructor, Destructor destructor, void *cookie) - : fStrategy(this) - { - if (benaphore_init(&fLock, name) >= B_OK) { - base_cache_init(this, name, objectSize, alignment, constructor, - destructor, cookie); - register_low_memory_handler(_LowMemory, this, 0); - } - } - - ~Cache() - { - if (fLock.sem >= B_OK) { - benaphore_lock(&fLock); - unregister_low_memory_handler(_LowMemory, this); - base_cache_destroy(this, _ReturnSlab); - benaphore_destroy(&fLock); - } - } - - status_t InitCheck() const { return fLock.sem; } - - void *AllocateObject(uint32_t flags) - { - BenaphoreLocker _(fLock); - - void *object = base_cache_allocate_object(this); - - // if the cache is returning NULL it is because it ran out of slabs - if (object == NULL) { - cache_slab *newSlab = fStrategy.NewSlab(flags); - if (newSlab == NULL) - return NULL; - object = base_cache_allocate_object_with_new_slab(this, newSlab); - if (object == NULL) - panic("cache: failed to allocate with an empty slab"); - } - - return object; - } - - void ReturnObject(void *object) - { - BenaphoreLocker _(fLock); - - cache_slab *slab = fStrategy.ObjectSlab(object); - - if (base_cache_return_object(this, slab, object)) - fStrategy.ReturnSlab(slab); - } - -private: - static void _ReturnSlab(base_cache *self, cache_slab *slab) - { - // Already locked, ~Cache() -> base_cache_destroy -> _ReturnSlab - ((ThisCache *)self)->fStrategy.ReturnSlab(slab); - } - - static void _LowMemory(void *_self, int32 level) - { - if (level == B_NO_LOW_MEMORY) - return; - - ThisCache *self = (ThisCache *)_self; - - BenaphoreLocker _(self->fLock); - base_cache_low_memory(self, level, _ReturnSlab); - } - - benaphore fLock; - Strategy fStrategy; -}; - -#endif /* __cplusplus */ +#endif #endif diff --git a/headers/private/kernel/slab/Depot.h b/headers/private/kernel/slab/Depot.h index fccd113f82..8de9b4dab8 100644 --- a/headers/private/kernel/slab/Depot.h +++ b/headers/private/kernel/slab/Depot.h @@ -18,29 +18,22 @@ extern "C" { #endif -typedef struct depot_magazine { - struct depot_magazine *next; - uint16_t current_round, round_count; - void *rounds[0]; -} depot_magazine; +typedef struct base_depot { + benaphore lock; + struct depot_magazine *full, *empty; + size_t full_count, empty_count; + struct depot_cpu_store *stores; + + void (*return_object)(struct base_depot *depot, void *object); +} base_depot; typedef struct depot_cpu_store { benaphore lock; - depot_magazine *loaded, *previous; + struct depot_magazine *loaded, *previous; } depot_cpu_store; -typedef struct base_depot { - benaphore lock; - depot_magazine *full, *empty; - size_t full_count, empty_count; - depot_cpu_store *stores; - - void (*return_object)(base_depot *depot, void *object); -} base_depot; - - static inline depot_cpu_store * base_depot_cpu(base_depot *depot) { diff --git a/headers/private/kernel/slab/HashStrategy.h b/headers/private/kernel/slab/HashStrategy.h deleted file mode 100644 index b28ae25ce9..0000000000 --- a/headers/private/kernel/slab/HashStrategy.h +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2007, Hugo Santos. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Hugo Santos, hugosantos@gmail.com - */ - -#ifndef _SLAB_HASH_STRATEGY_H_ -#define _SLAB_HASH_STRATEGY_H_ - -#include -#include // for TypedCache - -#include -#include - - -struct BaseHashCacheStrategy { - struct Link : HashTableLink { - const void *buffer; - cache_slab *slab; - }; - - struct HashTableDefinition { - typedef BaseHashCacheStrategy ParentType; - typedef const void * KeyType; - typedef Link ValueType; - - HashTableDefinition(BaseHashCacheStrategy *_parent) : parent(_parent) {} - - size_t HashKey(const void *key) const - { - return (((const uint8_t *)key) - - ((const uint8_t *)0)) >> parent->fLowerBoundary; - } - - size_t Hash(Link *value) const { return HashKey(value->buffer); } - - bool Compare(const void *key, Link *value) const - { - return value->buffer == key; - } - - HashTableLink *GetLink(Link *value) const { return value; } - - BaseHashCacheStrategy *parent; - }; - - // for g++ 2.95 - friend class HashTableDefinition; - - typedef OpenHashTable HashTable; - - static inline int - __Fls0(size_t value) - { - if (value == 0) - return -1; - - int bit; - for (bit = 0; value != 1; bit++) - value >>= 1; - return bit; - } - - BaseHashCacheStrategy(base_cache *parent) - : fHashTable(this), fLowerBoundary(__Fls0(parent->object_size)) {} - - cache_slab *ObjectSlab(void *object) const - { - return _Linkage(object)->slab; - } - -protected: - Link *_Linkage(void *object) const - { - Link *link = fHashTable.Lookup(object); - if (link == NULL) - panic("slab: missing buffer link from hash table."); - return link; - } - - HashTable fHashTable; - const size_t fLowerBoundary; -}; - - -template -struct HashCacheStrategy : BaseCacheStrategy, BaseHashCacheStrategy { - typedef typename BaseCacheStrategy::BaseSlab BaseSlab; - typedef typename BaseCacheStrategy::Slab Slab; - typedef HashCacheStrategy Strategy; - - HashCacheStrategy(base_cache *parent) - : BaseCacheStrategy(parent), BaseHashCacheStrategy(parent), - fSlabCache("slab cache", 0), fLinkCache("link cache", 0) {} - - BaseSlab *NewSlab(uint32_t flags) - { - size_t byteCount = _SlabSize(); - - Slab *slab = fSlabCache.Alloc(flags); - if (slab == NULL) - return NULL; - - void *pages; - if (Backend::AllocatePages(Parent(), &slab->id, &pages, byteCount, - flags) < B_OK) { - fSlabCache.Free(slab); - return NULL; - } - - // it's very important that we cast this to BaseHashCacheStrategy - // so we get the proper instance offset through void * - cache_slab *result = BaseCacheStrategy::_ConstructSlab(slab, - pages, _SlabSize(), this, _PrepareObject, _UnprepareObject); - if (result == NULL) { - Backend::FreePages(Parent(), slab->id); - fSlabCache.Free(slab); - } - - return result; - } - - void ReturnSlab(BaseSlab *slab) - { - BaseCacheStrategy::_DestructSlab(slab, this, _UnprepareObject); - fSlabCache.Free((Slab *)slab); - } - -private: - size_t _SlabSize() const - { - return BaseCacheStrategy::SlabSize(0); - } - - base_cache *Parent() const { return BaseCacheStrategy::Parent(); } - - static status_t _PrepareObject(void *_self, cache_slab *slab, void *object) - { - Strategy *self = (Strategy *)_self; - - Link *link = self->fLinkCache.Alloc(CACHE_DONT_SLEEP); - if (link == NULL) - return B_NO_MEMORY; - - link->slab = slab; - link->buffer = object; - - self->fHashTable.Insert(link); - - return B_OK; - } - - static void _UnprepareObject(void *_self, cache_slab *slab, void *object) - { - ((Strategy *)_self)->_UnprepareObject(object); - } - - void _UnprepareObject(void *object) - { - Link *link = _Linkage(object); - fHashTable.Remove(link); - fLinkCache.Free(link); - } - - TypedCache fSlabCache; - TypedCache fLinkCache; -}; - -#endif diff --git a/headers/private/kernel/slab/MergedStrategy.h b/headers/private/kernel/slab/MergedStrategy.h deleted file mode 100644 index 4dcf587e0d..0000000000 --- a/headers/private/kernel/slab/MergedStrategy.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2007, Hugo Santos. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Hugo Santos, hugosantos@gmail.com - */ - -#ifndef _SLAB_MERGED_STRATEGY_H_ -#define _SLAB_MERGED_STRATEGY_H_ - -#include - - -// This slab strategy includes the ObjectLink at the end of each object and the -// slab at the end of the allocated pages. It uses aligned allocations to -// provide object to slab mapping with zero storage, thus there is only one -// word of overhead per object. This is optimized for small objects. -template -class MergedLinkCacheStrategy : public BaseCacheStrategy { -public: - typedef typename BaseCacheStrategy::BaseSlab BaseSlab; - typedef typename BaseCacheStrategy::Slab Slab; - - MergedLinkCacheStrategy(base_cache *parent) - : BaseCacheStrategy(parent) {} - - static inline const void * - LowerBoundary(void *object, size_t byteCount) - { - const uint8_t *null = (uint8_t *)NULL; - return null + ((((uint8_t *)object) - null) & ~(byteCount - 1)); - } - - BaseSlab *ObjectSlab(void *object) const - { - return _SlabInPages(LowerBoundary(object, _SlabSize())); - } - - BaseSlab *NewSlab(uint32_t flags) - { - typename Backend::AllocationID id; - void *pages; - - size_t byteCount = _SlabSize(); - // in order to save a pointer per object or a hash table to - // map objects to slabs we required this set of pages to be - // aligned in a (pageCount * PAGE_SIZE) boundary. - if (Backend::AllocatePages(Parent(), &id, &pages, byteCount, - CACHE_ALIGN_TO_TOTAL | flags) < B_OK) - return NULL; - - _SlabInPages(pages)->id = id; - - return BaseCacheStrategy::_ConstructSlab(_SlabInPages(pages), - pages, byteCount - sizeof(Slab), this, NULL, NULL); - } - - void ReturnSlab(BaseSlab *slab) - { - BaseCacheStrategy::_DestructSlab(slab, NULL, NULL); - } - -private: - size_t _SlabSize() const - { - size_t byteCount = BaseCacheStrategy::SlabSize(sizeof(Slab)); - if (byteCount > Backend::kMaximumAlignedLength) - byteCount = Backend::kMaximumAlignedLength; - return byteCount; - } - - base_cache *Parent() const { return BaseCacheStrategy::Parent(); } - - Slab *_SlabInPages(const void *pages) const - { - return (Slab *)(((uint8_t *)pages) + _SlabSize() - sizeof(Slab)); - } -}; - -#endif diff --git a/headers/private/kernel/slab/Slab.h b/headers/private/kernel/slab/Slab.h index f375fb17c4..aa66cb702f 100644 --- a/headers/private/kernel/slab/Slab.h +++ b/headers/private/kernel/slab/Slab.h @@ -9,90 +9,7 @@ #ifndef _SLAB_SLAB_H_ #define _SLAB_SLAB_H_ +#include #include -#ifdef __cplusplus -#include -#include -#include -#include -#include - - -extern "C" { -#endif - -typedef void *object_cache_t; - - -object_cache_t -object_cache_create(const char *name, size_t object_size, size_t alignment, - base_cache_constructor constructor, base_cache_destructor destructor, - void *cookie); -void *object_cache_alloc(object_cache_t cache); -void *object_cache_alloc_etc(object_cache_t cache, uint32_t flags); -void object_cache_free(object_cache_t cache, void *object); -void object_cache_destroy(object_cache_t cache); - -#ifdef __cplusplus -} - - -template -class LocalCache : public CacheType, protected base_depot { -public: - typedef LocalCache ThisType; - typedef typename CacheType::Constructor Constructor; - typedef typename CacheType::Destructor Destructor; - - LocalCache(const char *name, size_t objectSize, size_t alignment, - Constructor _constructor, Destructor _destructor, void *_cookie) - : CacheType(name, objectSize, alignment, _constructor, _destructor, - _cookie) - { - fStatus = base_depot_init(this, _ReturnObject); - } - - ~LocalCache() - { - base_depot_destroy(this); - } - - status_t InitCheck() const { return fStatus; } - - void *Alloc(uint32_t flags) - { - void *object = base_depot_obtain_from_store(this, base_depot_cpu(this)); - if (object == NULL) - object = CacheType::AllocateObject(flags); - return object; - } - - void Free(void *object) - { - if (!base_depot_return_to_store(this, base_depot_cpu(this), object)) - CacheType::ReturnObject(object); - } - - void Destroy() - { - base_depot_make_empty(this); - } - -private: - void ReturnObject(void *object) - { - CacheType::ReturnObject(object); - } - - static void _ReturnObject(base_depot *self, void *object) - { - static_cast(self)->ReturnObject(object); - } - - status_t fStatus; -}; - -#endif /* __cplusplus */ - #endif diff --git a/headers/private/kernel/slab/Strategy.h b/headers/private/kernel/slab/Strategy.h deleted file mode 100644 index 1929aa1693..0000000000 --- a/headers/private/kernel/slab/Strategy.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2007, Hugo Santos. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Hugo Santos, hugosantos@gmail.com - */ - -#ifndef _SLAB_STRATEGY_H_ -#define _SLAB_STRATEGY_H_ - -#include - - -template -class BaseCacheStrategy { -protected: - typedef cache_slab BaseSlab; - - BaseCacheStrategy(base_cache *parent) - : fParent(parent) {} - - size_t SlabSize(size_t tailSpace) const - { - size_t pageCount = ((kMinimumSlabItems * fParent->object_size - + tailSpace) + Backend::kPageSize / 2) / Backend::kPageSize; - if (pageCount < 1) - pageCount = 1; - return pageCount * Backend::kPageSize; - } - - struct Slab : BaseSlab { - typename Backend::AllocationID id; - }; - - BaseSlab *_ConstructSlab(Slab *slab, void *pages, size_t byteCount, - void *parent, base_cache_owner_prepare prepare, - base_cache_owner_unprepare unprepare) - { - return base_cache_construct_slab(fParent, slab, pages, byteCount, - parent, prepare, unprepare); - } - - void _DestructSlab(BaseSlab *slab, void *parent, - base_cache_owner_unprepare unprepare) - { - base_cache_destruct_slab(fParent, slab, parent, unprepare); - Backend::FreePages(fParent, ((Slab *)slab)->id); - } - - base_cache *Parent() const { return fParent; } - - base_cache *fParent; -}; - -#endif diff --git a/headers/private/kernel/slab/Utilities.h b/headers/private/kernel/slab/Utilities.h deleted file mode 100644 index 450adcbc62..0000000000 --- a/headers/private/kernel/slab/Utilities.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2007, Hugo Santos. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Hugo Santos, hugosantos@gmail.com - */ - -#ifndef _SLAB_UTILITIES_H_ -#define _SLAB_UTILITIES_H_ - -#include -#include - - -template -class TypedCache : public Cache< MergedLinkCacheStrategy > { -public: - typedef MergedLinkCacheStrategy Strategy; - typedef Cache BaseType; - - TypedCache(const char *name, size_t alignment) - : BaseType(name, sizeof(Type), alignment, _ConstructObject, - _DestructObject, this) {} - virtual ~TypedCache() {} - - Type *Alloc(uint32_t flags) { return (Type *)BaseType::AllocateObject(flags); } - void Free(Type *object) { BaseType::ReturnObject(object); } - -private: - static status_t _ConstructObject(void *cookie, void *object) - { - return ((TypedCache *)cookie)->ConstructObject((Type *)object); - } - - static void _DestructObject(void *cookie, void *object) - { - ((TypedCache *)cookie)->DestructObject((Type *)object); - } - - virtual status_t ConstructObject(Type *object) { return B_OK; } - virtual void DestructObject(Type *object) {} -}; - -#endif diff --git a/headers/private/kernel/vm.h b/headers/private/kernel/vm.h index 04bdcb6c2e..b660dea3f2 100644 --- a/headers/private/kernel/vm.h +++ b/headers/private/kernel/vm.h @@ -23,6 +23,7 @@ extern "C" { // startup only status_t vm_init(kernel_args *args); +status_t slab_init(); status_t vm_init_post_sem(struct kernel_args *args); status_t vm_init_post_thread(struct kernel_args *args); status_t vm_init_post_modules(struct kernel_args *args); diff --git a/src/add-ons/kernel/network/stack/net_buffer.cpp b/src/add-ons/kernel/network/stack/net_buffer.cpp index a3659b1945..4447850e74 100644 --- a/src/add-ons/kernel/network/stack/net_buffer.cpp +++ b/src/add-ons/kernel/network/stack/net_buffer.cpp @@ -65,14 +65,8 @@ struct net_buffer_private : net_buffer { }; -typedef MergedLinkCacheStrategy AreaMergedCacheStrategy; -typedef HashCacheStrategy AreaHashCacheStrategy; - -typedef Cache NetBufferCache; -typedef Cache DataNodeCache; - -static NetBufferCache *sNetBufferCache; -static DataNodeCache *sDataNodeCache; +static object_cache *sNetBufferCache; +static object_cache *sDataNodeCache; static status_t append_data(net_buffer *buffer, const void *data, size_t size); @@ -102,14 +96,14 @@ dump_buffer(net_buffer *_buffer) static inline data_header * allocate_data_header() { - return (data_header *)sDataNodeCache->AllocateObject(CACHE_DONT_SLEEP); + return (data_header *)object_cache_alloc(sDataNodeCache, CACHE_DONT_SLEEP); } static inline net_buffer_private * allocate_net_buffer() { - return (net_buffer_private *)sNetBufferCache->AllocateObject( + return (net_buffer_private *)object_cache_alloc(sNetBufferCache, CACHE_DONT_SLEEP); } @@ -117,14 +111,14 @@ allocate_net_buffer() static inline void free_data_header(data_header *header) { - sDataNodeCache->ReturnObject(header); + object_cache_free(sDataNodeCache, header); } static inline void free_net_buffer(net_buffer_private *buffer) { - sNetBufferCache->ReturnObject(buffer); + object_cache_free(sNetBufferCache, buffer); } @@ -1180,31 +1174,18 @@ init_net_buffers() // TODO improve our code a bit so we can add constructors // and keep around half-constructed buffers in the slab - sNetBufferCache = new (std::nothrow) NetBufferCache("net buffer cache", + sNetBufferCache = create_object_cache("net buffer cache", sizeof(net_buffer_private), 8, NULL, NULL, NULL); if (sNetBufferCache == NULL) return B_NO_MEMORY; - status_t status = sNetBufferCache->InitCheck(); - if (status < B_OK) { - delete sNetBufferCache; - return status; - } - - sDataNodeCache = new (std::nothrow) DataNodeCache("data node cache", - BUFFER_SIZE, 0, NULL, NULL, NULL); + sDataNodeCache = create_object_cache("data node cache", BUFFER_SIZE, 0, + NULL, NULL, NULL); if (sDataNodeCache == NULL) { - delete sNetBufferCache; + delete_object_cache(sNetBufferCache); return B_NO_MEMORY; } - status = sDataNodeCache->InitCheck(); - if (status < B_OK) { - delete sDataNodeCache; - delete sNetBufferCache; - return status; - } - return B_OK; } @@ -1212,8 +1193,8 @@ init_net_buffers() status_t uninit_net_buffers() { - delete sNetBufferCache; - delete sDataNodeCache; + delete_object_cache(sNetBufferCache); + delete_object_cache(sDataNodeCache); return B_OK; } diff --git a/src/system/kernel/main.c b/src/system/kernel/main.c index 4156294164..2c7905fb7f 100644 --- a/src/system/kernel/main.c +++ b/src/system/kernel/main.c @@ -137,6 +137,8 @@ _start(kernel_args *bootKernelArgs, int currentCPU) generic_syscall_init(); TRACE("init cbuf\n"); cbuf_init(); + TRACE("init slab\n"); + slab_init(); TRACE("init teams\n"); team_init(&sKernelArgs); TRACE("init threads\n"); diff --git a/src/system/kernel/slab/Jamfile b/src/system/kernel/slab/Jamfile index 6a04fa1f1a..b9aebd4cf1 100644 --- a/src/system/kernel/slab/Jamfile +++ b/src/system/kernel/slab/Jamfile @@ -3,7 +3,6 @@ SubDir HAIKU_TOP src system kernel slab ; UsePrivateHeaders [ FDirName kernel slab ] ; KernelMergeObject kernel_slab.o : - interface.cpp Slab.cpp : $(TARGET_KERNEL_PIC_CCFLAGS) -Wno-unused diff --git a/src/system/kernel/slab/Slab.cpp b/src/system/kernel/slab/Slab.cpp index bdd1db221d..a434808db5 100644 --- a/src/system/kernel/slab/Slab.cpp +++ b/src/system/kernel/slab/Slab.cpp @@ -12,16 +12,23 @@ #include #include +#include +#include + +#include #include // swap #include + // TODO all of the small allocations we perform here will fallback // to the internal allocator which in the future will use this // same code. We'll have to resolve all of the dependencies // then, for now, it is still not required. +// TODO kMagazineCapacity should be dynamically tuned per cache. -//#define TRACE_SLAB + +#define TRACE_SLAB #ifdef TRACE_SLAB #define TRACE_CACHE(cache, format, args...) \ @@ -31,14 +38,119 @@ #endif -// TODO this value should be dynamically tuned per cache. -static const int kMagazineCapacity = 32; +extern "C" status_t slab_init(); + +static const int kMagazineCapacity = 32; static const size_t kCacheColorPeriod = 8; -typedef struct cache_object_link { - struct cache_object_link *next; -} cache_object_link; +struct object_link { + struct object_link *next; +}; + +struct slab : DoublyLinkedListLinkImpl { + void *pages; + size_t count, size; + size_t offset; + object_link *free; +}; + +typedef DoublyLinkedList SlabList; + +struct object_cache : DoublyLinkedListLinkImpl { + char name[32]; + benaphore lock; + size_t object_size; + size_t cache_color_cycle; + SlabList empty, partial, full; + size_t empty_count, pressure; + + size_t slab_size; + size_t usage, maximum; + uint32 flags; + + void *cookie; + object_cache_constructor constructor; + object_cache_destructor destructor; + object_cache_reclaimer reclaimer; + + base_depot depot; + + virtual slab *CreateSlab(uint32 flags) = 0; + virtual void ReturnSlab(slab *slab) = 0; + virtual slab *ObjectSlab(void *object) const = 0; + + slab *InitSlab(slab *slab, void *pages, size_t byteCount); + void UninitSlab(slab *slab); + + virtual status_t PrepareObject(slab *source, void *object) { return B_OK; } + virtual void UnprepareObject(slab *source, void *object) {} + + virtual ~object_cache() {} +}; + +typedef DoublyLinkedList ObjectCacheList; + +struct SmallObjectCache : object_cache { + slab *CreateSlab(uint32 flags); + void ReturnSlab(slab *slab); + slab *ObjectSlab(void *object) const; +}; + + +struct HashedObjectCache : object_cache { + struct Link : HashTableLink { + const void *buffer; + slab *parent; + }; + + struct Definition { + typedef HashedObjectCache ParentType; + typedef const void * KeyType; + typedef Link ValueType; + + Definition(HashedObjectCache *_parent) : parent(_parent) {} + + size_t HashKey(const void *key) const + { + return (((const uint8 *)key) - ((const uint8 *)0)) + >> parent->lower_boundary; + } + + size_t Hash(Link *value) const { return HashKey(value->buffer); } + bool Compare(const void *key, Link *value) const + { return value->buffer == key; } + HashTableLink *GetLink(Link *value) const { return value; } + + HashedObjectCache *parent; + }; + + typedef OpenHashTable HashTable; + + HashedObjectCache() + : hash_table(this) {} + + slab *CreateSlab(uint32 flags); + void ReturnSlab(slab *slab); + slab *ObjectSlab(void *object) const; + status_t PrepareObject(slab *source, void *object); + void UnprepareObject(slab *source, void *object); + + HashTable hash_table; + size_t lower_boundary; +}; + + +struct depot_magazine { + struct depot_magazine *next; + uint16_t current_round, round_count; + void *rounds[0]; +}; + + +static object_cache *sSlabCache, *sLinkCache; +static ObjectCacheList sObjectCaches; +static benaphore sObjectCacheListLock; static depot_magazine *alloc_magazine(); @@ -62,110 +174,80 @@ _push(Type* &head, Type *object) static inline void * -link_to_object(cache_object_link *link, size_t objectSize) +link_to_object(object_link *link, size_t objectSize) { - return ((uint8_t *)link) - (objectSize - sizeof(cache_object_link)); + return ((uint8_t *)link) - (objectSize - sizeof(object_link)); } -static inline cache_object_link * +static inline object_link * object_to_link(void *object, size_t objectSize) { - return (cache_object_link *)(((uint8_t *)object) - + (objectSize - sizeof(cache_object_link))); + return (object_link *)(((uint8_t *)object) + + (objectSize - sizeof(object_link))); } -status_t -slab_area_backend_allocate(base_cache *cache, area_id *id, void **pages, - size_t byteCount, uint32_t flags) +static inline int +__fls0(size_t value) { - if (flags & CACHE_ALIGN_TO_TOTAL && byteCount > B_PAGE_SIZE) - return NULL; + if (value == 0) + return -1; - TRACE_CACHE(cache, "allocate pages (%lu, 0x0%lx)", byteCount, flags); + int bit; + for (bit = 0; value != 1; bit++) + value >>= 1; + return bit; +} - area_id areaId = create_area(cache->name, pages, - B_ANY_KERNEL_ADDRESS, byteCount, B_NO_LOCK, B_READ_AREA | B_WRITE_AREA); + +static status_t +allocate_pages(object_cache *cache, void **pages, uint32 flags) +{ + TRACE_CACHE(cache, "allocate pages (%lu, 0x0%lx)", cache->slab_size, flags); + + area_id areaId = create_area(cache->name, pages, B_ANY_KERNEL_ADDRESS, + cache->slab_size, B_NO_LOCK, B_READ_AREA | B_WRITE_AREA); if (areaId < 0) return areaId; + cache->usage += cache->slab_size; + TRACE_CACHE(cache, " ... = { %ld, %p }", areaId, *pages); - *id = areaId; return B_OK; } -void -slab_area_backend_free(base_cache *cache, area_id area) + +static void +free_pages(object_cache *cache, void *pages) { - TRACE_CACHE(cache, "delete pages %ld", area); - delete_area(area); + area_id id = area_for(pages); + + TRACE_CACHE(cache, "delete pages %p (%ld)", pages, id); + + if (id < B_OK) + panic("object cache: freeing unknown area"); + + delete_area(id); + + cache->usage -= cache->slab_size; } -void -base_cache_init(base_cache *cache, const char *name, size_t objectSize, - size_t alignment, base_cache_constructor constructor, - base_cache_destructor destructor, void *cookie) +static void +object_cache_low_memory(void *_self, int32 level) { - strlcpy(cache->name, name, sizeof(cache->name)); + if (level == B_NO_LOW_MEMORY) + return; - if (objectSize < sizeof(void *) && alignment < sizeof(void *)) - objectSize = sizeof(void *); + object_cache *cache = (object_cache *)_self; - if (alignment > 0 && (objectSize & (alignment - 1))) - cache->object_size = objectSize + alignment - - (objectSize & (alignment - 1)); - else - cache->object_size = objectSize; + BenaphoreLocker _(cache->lock); - TRACE_CACHE(cache, "init %lu, %lu -> %lu", objectSize, alignment, - cache->object_size); - - cache->cache_color_cycle = 0; - - list_init_etc(&cache->empty, offsetof(cache_slab, link)); - list_init_etc(&cache->partial, offsetof(cache_slab, link)); - list_init_etc(&cache->full, offsetof(cache_slab, link)); - - cache->empty_count = 0; - - // pressure is increased whenever we need a slab and don't have one - cache->pressure = 0; - - cache->constructor = constructor; - cache->destructor = destructor; - cache->cookie = cookie; -} - - -void -base_cache_destroy(base_cache *cache, - void (*return_slab)(base_cache *, cache_slab *)) -{ - if (!list_is_empty(&cache->full)) - panic("cache destroy: still has full slabs"); - - if (!list_is_empty(&cache->partial)) - panic("cache destroy: still has partial slabs"); - - while (!list_is_empty(&cache->empty)) { - cache_slab *slab = (cache_slab *)list_remove_head_item(&cache->empty); - return_slab(cache, slab); - } - - cache->empty_count = 0; -} - - -void -base_cache_low_memory(base_cache *cache, int32 level, - void (*return_slab)(base_cache *, cache_slab *)) -{ size_t minimumAllowed; - // only thing we can do right now is free up empty slabs + // TODO: call reclaim switch (level) { case B_LOW_MEMORY_NOTE: @@ -190,139 +272,287 @@ base_cache_low_memory(base_cache *cache, int32 level, minimumAllowed); while (cache->empty_count > minimumAllowed) { - cache_slab *slab = (cache_slab *)list_remove_head_item(&cache->empty); - return_slab(cache, slab); + cache->ReturnSlab(cache->empty.RemoveHead()); cache->empty_count--; } } -void * -base_cache_allocate_object(base_cache *cache) +static status_t +object_cache_init(object_cache *cache, const char *name, size_t objectSize, + size_t alignment, size_t maximum, uint32 flags, void *cookie, + object_cache_constructor constructor, object_cache_destructor destructor, + object_cache_reclaimer reclaimer) { - cache_slab *slab; + status_t status = benaphore_init(&cache->lock, name); + if (status < B_OK) + return status; + + strlcpy(cache->name, name, sizeof(cache->name)); + + if (objectSize < sizeof(object_link)) + objectSize = sizeof(object_link); + + if (alignment > 0 && (objectSize & (alignment - 1))) + cache->object_size = objectSize + alignment + - (objectSize & (alignment - 1)); + else + cache->object_size = objectSize; + + TRACE_CACHE(cache, "init %lu, %lu -> %lu", objectSize, alignment, + cache->object_size); + + cache->cache_color_cycle = 0; + cache->empty_count = 0; + cache->pressure = 0; + + cache->usage = 0; + cache->maximum = maximum; + + cache->flags = flags; + + if (!(flags & CACHE_NO_DEPOT)) { + // TODO init depot + } + + cache->cookie = cookie; + cache->constructor = constructor; + cache->destructor = destructor; + cache->reclaimer = reclaimer; + + register_low_memory_handler(object_cache_low_memory, cache, 0); + + BenaphoreLocker _(sObjectCacheListLock); + sObjectCaches.Add(cache); + + return B_OK; +} + + +static SmallObjectCache * +create_small_object_cache(const char *name, size_t object_size, + size_t alignment, size_t maximum, uint32 flags, void *cookie, + object_cache_constructor constructor, object_cache_destructor destructor, + object_cache_reclaimer reclaimer) +{ + SmallObjectCache *cache = new (std::nothrow) SmallObjectCache(); + if (cache == NULL) + return NULL; + + if (object_cache_init(cache, name, object_size, alignment, maximum, flags, + cookie, constructor, destructor, reclaimer) < B_OK) { + delete cache; + return NULL; + } + + cache->slab_size = B_PAGE_SIZE; + + return cache; +} + + +static HashedObjectCache * +create_hashed_object_cache(const char *name, size_t object_size, + size_t alignment, size_t maximum, uint32 flags, void *cookie, + object_cache_constructor constructor, object_cache_destructor destructor, + object_cache_reclaimer reclaimer) +{ + HashedObjectCache *cache = new (std::nothrow) HashedObjectCache(); + if (cache == NULL) + return NULL; + + if (object_cache_init(cache, name, object_size, alignment, maximum, flags, + cookie, constructor, destructor, reclaimer) < B_OK) { + delete cache; + return NULL; + } + + cache->slab_size = 16 * B_PAGE_SIZE; + cache->lower_boundary = __fls0(cache->object_size); + + return cache; +} + + +object_cache * +create_object_cache(const char *name, size_t object_size, size_t alignment, + void *cookie, object_cache_constructor constructor, + object_cache_destructor destructor) +{ + return create_object_cache_etc(name, object_size, alignment, 0, 0, cookie, + constructor, destructor, NULL); +} + + +object_cache * +create_object_cache_etc(const char *name, size_t object_size, size_t alignment, + size_t maximum, uint32 flags, void *cookie, + object_cache_constructor constructor, object_cache_destructor destructor, + object_cache_reclaimer reclaimer) +{ + if (object_size == 0) + return NULL; + else if (object_size <= 256) + return create_small_object_cache(name, object_size, alignment, + maximum, flags, cookie, constructor, destructor, reclaimer); + + return create_hashed_object_cache(name, object_size, alignment, + maximum, flags, cookie, constructor, destructor, reclaimer); +} + + +void +delete_object_cache(object_cache *cache) +{ + { + BenaphoreLocker _(sObjectCacheListLock); + sObjectCaches.Remove(cache); + } + + benaphore_lock(&cache->lock); + + unregister_low_memory_handler(object_cache_low_memory, cache); + + if (!cache->full.IsEmpty()) + panic("cache destroy: still has full slabs"); + + if (!cache->partial.IsEmpty()) + panic("cache destroy: still has partial slabs"); + + while (!cache->empty.IsEmpty()) + cache->ReturnSlab(cache->empty.RemoveHead()); + + cache->empty_count = 0; + + benaphore_destroy(&cache->lock); + delete cache; +} + + +void * +object_cache_alloc(object_cache *cache, uint32 flags) +{ + BenaphoreLocker _(cache->lock); + + slab *source = NULL; + + if (cache->partial.IsEmpty()) { + if (cache->empty.IsEmpty()) { + source = cache->CreateSlab(flags); + if (source == NULL) + return NULL; - if (list_is_empty(&cache->partial)) { - if (list_is_empty(&cache->empty)) { cache->pressure++; - return NULL; + } else { + cache->empty_count--; + source = cache->empty.RemoveHead(); } - cache->empty_count--; - slab = (cache_slab *)list_remove_head_item(&cache->empty); - list_add_item(&cache->partial, slab); - } else - slab = (cache_slab *)list_get_first_item(&cache->partial); + cache->partial.Add(source); + } else { + source = cache->partial.Head(); + } - cache_object_link *link = _pop(slab->free); - slab->count--; + object_link *link = _pop(source->free); + source->count--; - TRACE_CACHE(cache, "allocate %p from %p, %lu remaining.", link, slab, - slab->count); + TRACE_CACHE(cache, "allocate %p from %p, %lu remaining.", link, source, + source->count); - if (slab->count == 0) { - // move the partial slab to the full list - list_remove_item(&cache->partial, slab); - list_add_item(&cache->full, slab); + if (source->count == 0) { + cache->partial.Remove(source); + cache->full.Add(source); } return link_to_object(link, cache->object_size); } -void * -base_cache_allocate_object_with_new_slab(base_cache *cache, - cache_slab *newSlab) +static void +object_cache_return_to_slab(object_cache *cache, slab *source, void *object) { - list_add_item(&cache->partial, newSlab); - return base_cache_allocate_object(cache); -} + if (source == NULL) + panic("object_cache: free'd object has no slab"); - -int -base_cache_return_object(base_cache *cache, cache_slab *slab, - void *object) -{ - // We return true if the slab is completely unused. - cache_object_link *link = object_to_link(object, cache->object_size); + object_link *link = object_to_link(object, cache->object_size); TRACE_CACHE(cache, "returning %p to %p, %lu used (%lu empty slabs).", - link, slab, slab->size - slab->count, cache->empty_count); + link, source, source->size - source->count, cache->empty_count); - _push(slab->free, link); - slab->count++; - if (slab->count == slab->size) { - list_remove_item(&cache->partial, slab); + _push(source->free, link); + source->count++; + if (source->count == source->size) { + cache->partial.Remove(source); - if (cache->empty_count >= cache->pressure) - return 1; - - cache->empty_count++; - list_add_item(&cache->empty, slab); - } else if (slab->count == 1) { - list_remove_item(&cache->full, slab); - list_add_item(&cache->partial, slab); + if (cache->empty_count < cache->pressure) { + cache->empty_count++; + cache->empty.Add(source); + } else { + cache->ReturnSlab(source); + } + } else if (source->count == 1) { + cache->full.Remove(source); + cache->partial.Add(source); } - - return 0; } -cache_slab * -base_cache_construct_slab(base_cache *cache, cache_slab *slab, void *pages, - size_t byteCount, void *parent, - base_cache_owner_prepare prepare, base_cache_owner_unprepare unprepare) +void +object_cache_free(object_cache *cache, void *object) { - TRACE_CACHE(cache, "construct (%p, %p, %lu)", slab, pages, byteCount); + BenaphoreLocker _(cache->lock); + object_cache_return_to_slab(cache, cache->ObjectSlab(object), object); +} + + +slab * +object_cache::InitSlab(slab *slab, void *pages, size_t byteCount) +{ + TRACE_CACHE(this, "construct (%p, %p, %lu)", slab, pages, byteCount); slab->pages = pages; - slab->count = slab->size = byteCount / cache->object_size; + slab->count = slab->size = byteCount / object_size; slab->free = NULL; - size_t spareBytes = byteCount - (slab->size * cache->object_size); - slab->offset = cache->cache_color_cycle; + size_t spareBytes = byteCount - (slab->size * object_size); + slab->offset = cache_color_cycle; if (slab->offset > spareBytes) - cache->cache_color_cycle = slab->offset = 0; + cache_color_cycle = slab->offset = 0; else - cache->cache_color_cycle += kCacheColorPeriod; + cache_color_cycle += kCacheColorPeriod; - TRACE_CACHE(cache, " %lu objects, %lu spare bytes, offset %lu", + TRACE_CACHE(this, " %lu objects, %lu spare bytes, offset %lu", slab->size, spareBytes, slab->offset); uint8_t *data = ((uint8_t *)pages) + slab->offset; for (size_t i = 0; i < slab->size; i++) { - if (prepare || cache->constructor) { - bool failedOnFirst = false; - status_t status = B_OK; + bool failedOnFirst = false; - if (prepare) - status = prepare(parent, slab, data); - if (status < B_OK) - failedOnFirst = true; - else if (cache->constructor) - status = cache->constructor(cache->cookie, data); + status_t status = PrepareObject(slab, data); + if (status < B_OK) + failedOnFirst = true; + else if (constructor) + status = constructor(cookie, data); - if (status < B_OK) { - if (!failedOnFirst && unprepare) - unprepare(parent, slab, data); + if (status < B_OK) { + if (!failedOnFirst) + UnprepareObject(slab, data); - data = ((uint8_t *)pages) + slab->offset; - for (size_t j = 0; j < i; j++) { - if (cache->destructor) - cache->destructor(cache->cookie, data); - if (unprepare) - unprepare(parent, slab, data); - data += cache->object_size; - } - - return NULL; + data = ((uint8_t *)pages) + slab->offset; + for (size_t j = 0; j < i; j++) { + if (destructor) + destructor(cookie, data); + UnprepareObject(slab, data); + data += object_size; } + + return NULL; } - _push(slab->free, object_to_link(data, cache->object_size)); - data += cache->object_size; + _push(slab->free, object_to_link(data, object_size)); + data += object_size; } return slab; @@ -330,10 +560,9 @@ base_cache_construct_slab(base_cache *cache, cache_slab *slab, void *pages, void -base_cache_destruct_slab(base_cache *cache, cache_slab *slab, void *parent, - base_cache_owner_unprepare unprepare) +object_cache::UninitSlab(slab *slab) { - TRACE_CACHE(cache, "destruct %p", slab); + TRACE_CACHE(this, "destruct %p", slab); if (slab->count != slab->size) panic("cache: destroying a slab which isn't empty."); @@ -341,15 +570,171 @@ base_cache_destruct_slab(base_cache *cache, cache_slab *slab, void *parent, uint8_t *data = ((uint8_t *)slab->pages) + slab->offset; for (size_t i = 0; i < slab->size; i++) { - if (cache->destructor) - cache->destructor(cache->cookie, data); - if (unprepare) - unprepare(parent, slab, data); - data += cache->object_size; + if (destructor) + destructor(cookie, data); + UnprepareObject(slab, data); + data += object_size; } } +static inline slab * +slab_in_pages(const void *pages, size_t slab_size) +{ + return (slab *)(((uint8_t *)pages) + slab_size - sizeof(slab)); +} + + +static inline const void * +lower_boundary(void *object, size_t byteCount) +{ + const uint8_t *null = (uint8_t *)NULL; + return null + ((((uint8_t *)object) - null) & ~(byteCount - 1)); +} + + +static inline bool +check_cache_quota(object_cache *cache) +{ + if (cache->maximum == 0) + return true; + + return (cache->usage + cache->slab_size) <= cache->maximum; +} + + +slab * +SmallObjectCache::CreateSlab(uint32 flags) +{ + if (!check_cache_quota(this)) + return NULL; + + void *pages; + + if (allocate_pages(this, &pages, flags) < B_OK) + return NULL; + + return InitSlab(slab_in_pages(pages, slab_size), pages, + slab_size - sizeof(slab)); +} + + +void +SmallObjectCache::ReturnSlab(slab *slab) +{ + UninitSlab(slab); + free_pages(this, slab->pages); +} + + +slab * +SmallObjectCache::ObjectSlab(void *object) const +{ + return slab_in_pages(lower_boundary(object, object_size), slab_size); +} + + +static slab * +allocate_slab(uint32 flags) +{ + return (slab *)object_cache_alloc(sSlabCache, flags); +} + + +static void +free_slab(slab *slab) +{ + object_cache_free(sSlabCache, slab); +} + + +static HashedObjectCache::Link * +allocate_link(uint32 flags) +{ + return (HashedObjectCache::Link *)object_cache_alloc(sLinkCache, flags); +} + + +static void +free_link(HashedObjectCache::Link *link) +{ + object_cache_free(sLinkCache, link); +} + + +slab * +HashedObjectCache::CreateSlab(uint32 flags) +{ + if (!check_cache_quota(this)) + return NULL; + + slab *slab = allocate_slab(flags); + if (slab == NULL) + return NULL; + + void *pages; + if (allocate_pages(this, &pages, flags) < B_OK) { + free_slab(slab); + return NULL; + } + + if (InitSlab(slab, pages, slab_size) == NULL) { + free_pages(this, pages); + free_slab(slab); + return NULL; + } + + return slab; +} + + +void +HashedObjectCache::ReturnSlab(slab *slab) +{ + UninitSlab(slab); + free_pages(this, slab->pages); +} + + +slab * +HashedObjectCache::ObjectSlab(void *object) const +{ + Link *link = hash_table.Lookup(object); + if (link == NULL) + panic("object cache: requested object missing from hash table"); + return link->parent; +} + + +status_t +HashedObjectCache::PrepareObject(slab *source, void *object) +{ + Link *link = allocate_link(CACHE_DONT_SLEEP); + if (link == NULL) + return B_NO_MEMORY; + + link->buffer = object; + link->parent = source; + + hash_table.Insert(link); + return B_OK; +} + + +void +HashedObjectCache::UnprepareObject(slab *source, void *object) +{ + Link *link = hash_table.Lookup(object); + if (link == NULL) + panic("object cache: requested object missing from hash table"); + if (link->parent != source) + panic("object cache: slab mismatch"); + + hash_table.Remove(link); + free_link(link); +} + + static inline bool is_magazine_empty(depot_magazine *magazine) { @@ -568,3 +953,47 @@ base_depot_make_empty(base_depot *depot) empty_magazine(depot, _pop(depot->empty)); } + +static int +dump_slabs(int argc, char *argv[]) +{ + kprintf("%10s %32s %8s %8s %6s\n", "address", "name", "objsize", "usage", + "empty"); + + ObjectCacheList::Iterator it = sObjectCaches.GetIterator(); + + while (it.HasNext()) { + object_cache *cache = it.Next(); + + kprintf("%p %32s %8lu %8lu %6lu\n", cache, cache->name, + cache->object_size, cache->usage, cache->empty_count); + } + + return 0; +} + + +status_t +slab_init() +{ + status_t status = benaphore_init(&sObjectCacheListLock, "object cache list"); + if (status < B_OK) + panic("slab_init: failed to create object cache list lock"); + + new (&sObjectCaches) ObjectCacheList(); + + sSlabCache = create_object_cache("slab cache", sizeof(slab), 4, NULL, NULL, + NULL); + if (sSlabCache == NULL) + panic("slab_init: failed to create slab cache"); + + sLinkCache = create_object_cache("link cache", + sizeof(HashedObjectCache::Link), 4, NULL, NULL, NULL); + if (sLinkCache == NULL) + panic("slab_init: failed to create link cache"); + + add_debugger_command("slabs", dump_slabs, "list all object caches"); + + return B_OK; +} + diff --git a/src/system/kernel/slab/interface.cpp b/src/system/kernel/slab/interface.cpp deleted file mode 100644 index 2a2df8da91..0000000000 --- a/src/system/kernel/slab/interface.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2007, Hugo Santos. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Hugo Santos, hugosantos@gmail.com - */ - -#include -#include - -class AbstractCache { -public: - virtual ~AbstractCache() {} - - virtual void *Allocate(uint32_t flags) = 0; - virtual void Return(void *object) = 0; -}; - - -typedef MergedLinkCacheStrategy SmallObjectStrategy; -typedef HashCacheStrategy LargeObjectStrategy; - -typedef Cache SmallObjectCache; -typedef Cache LargeObjectCache; - - -class SmallObjectAbstractCache : public AbstractCache, - public LocalCache { -public: - typedef LocalCache Base; - - SmallObjectAbstractCache(const char *name, size_t objectSize, - size_t alignment, base_cache_constructor constructor, - base_cache_destructor destructor, void *cookie) - : Base(name, objectSize, alignment, constructor, destructor, cookie) {} - - void *Allocate(uint32_t flags) { return Base::Alloc(flags); } - void Return(void *object) { Base::Free(object); } -}; - - -class LargeObjectAbstractCache : public AbstractCache, - public LocalCache { -public: - typedef LocalCache Base; - - LargeObjectAbstractCache(const char *name, size_t objectSize, - size_t alignment, base_cache_constructor constructor, - base_cache_destructor destructor, void *cookie) - : Base(name, objectSize, alignment, constructor, destructor, cookie) {} - - void *Allocate(uint32_t flags) { return Base::Alloc(flags); } - void Return(void *object) { Base::Free(object); } -}; - - -object_cache_t -object_cache_create(const char *name, size_t object_size, size_t alignment, - status_t (*_constructor)(void *, void *), void (*_destructor)(void *, - void *), void *cookie) -{ - if (object_size == 0) - return NULL; - else if (object_size <= 256) - return new (std::nothrow) SmallObjectAbstractCache(name, object_size, - alignment, _constructor, _destructor, cookie); - - return new (std::nothrow) LargeObjectAbstractCache(name, object_size, - alignment, _constructor, _destructor, cookie); -} - - -void * -object_cache_alloc(object_cache_t cache) -{ - return object_cache_alloc_etc(cache, 0); -} - - -void * -object_cache_alloc_etc(object_cache_t cache, uint32_t flags) -{ - return ((AbstractCache *)cache)->Allocate(flags); -} - - -void -object_cache_free(object_cache_t cache, void *object) -{ - ((AbstractCache *)cache)->Return(object); -} - - -void -object_cache_destroy(object_cache_t cache) -{ - delete (AbstractCache *)cache; -} -