diff --git a/headers/private/kernel/slab/Base.h b/headers/private/kernel/slab/Base.h index 16e3674586..49ad79643d 100644 --- a/headers/private/kernel/slab/Base.h +++ b/headers/private/kernel/slab/Base.h @@ -58,6 +58,7 @@ typedef struct cache_object_link { typedef struct cache_slab { void *pages; size_t count, size; + size_t offset; cache_object_link *free; struct list_link link; } cache_slab; @@ -73,11 +74,11 @@ void base_cache_destroy(base_cache *cache, void base_cache_low_memory(base_cache *cache, int32 level, void (*return_slab)(base_cache *, cache_slab *)); -cache_object_link *base_cache_allocate_object(base_cache *cache); -cache_object_link *base_cache_allocate_object_with_new_slab(base_cache *cache, +void *base_cache_allocate_object(base_cache *cache); +void *base_cache_allocate_object_with_new_slab(base_cache *cache, cache_slab *slab); int base_cache_return_object(base_cache *cache, cache_slab *slab, - cache_object_link *link); + void *object); typedef status_t (*base_cache_owner_prepare)(void *parent, cache_slab *slab, void *object); @@ -86,15 +87,13 @@ typedef void (*base_cache_owner_unprepare)(void *parent, cache_slab *slab, cache_slab *base_cache_construct_slab(base_cache *cache, cache_slab *slab, void *pages, size_t byte_count, void *parent, - cache_object_link *(*get_link)(void *parent, void *object), base_cache_owner_prepare prepare, base_cache_owner_unprepare unprepare); -void base_cache_destruct_slab(base_cache *cache, cache_slab *slab); +void base_cache_destruct_slab(base_cache *cache, cache_slab *slab, + void *parent, base_cache_owner_unprepare unprepare); #ifdef __cplusplus } -typedef std::pair CacheObjectInfo; - // Slab implementation, glues together the frontend, backend as // well as the Slab strategy used. template @@ -109,8 +108,8 @@ public: : fStrategy(this) { if (benaphore_init(&fLock, name) >= B_OK) { - base_cache_init(this, name, Strategy::RequiredSpace(objectSize), - alignment, constructor, destructor, cookie); + base_cache_init(this, name, objectSize, alignment, constructor, + destructor, cookie); register_low_memory_handler(_LowMemory, this, 0); } } @@ -131,29 +130,29 @@ public: { BenaphoreLocker _(fLock); - cache_object_link *link = base_cache_allocate_object(this); + void *object = base_cache_allocate_object(this); // if the cache is returning NULL it is because it ran out of slabs - if (link == NULL) { + if (object == NULL) { cache_slab *newSlab = fStrategy.NewSlab(flags); if (newSlab == NULL) return NULL; - link = base_cache_allocate_object_with_new_slab(this, newSlab); - if (link == NULL) + object = base_cache_allocate_object_with_new_slab(this, newSlab); + if (object == NULL) panic("cache: failed to allocate with an empty slab"); } - return fStrategy.Object(link); + return object; } void ReturnObject(void *object) { BenaphoreLocker _(fLock); - CacheObjectInfo location = fStrategy.ObjectInformation(object); + cache_slab *slab = fStrategy.ObjectSlab(object); - if (base_cache_return_object(this, location.first, location.second)) - fStrategy.ReturnSlab(location.first); + if (base_cache_return_object(this, slab, object)) + fStrategy.ReturnSlab(slab); } private: diff --git a/headers/private/kernel/slab/HashStrategy.h b/headers/private/kernel/slab/HashStrategy.h index eb472280e5..b28ae25ce9 100644 --- a/headers/private/kernel/slab/HashStrategy.h +++ b/headers/private/kernel/slab/HashStrategy.h @@ -17,26 +17,27 @@ struct BaseHashCacheStrategy { - struct Link : cache_object_link, HashTableLink { + struct Link : HashTableLink { + const void *buffer; cache_slab *slab; - void *buffer; }; struct HashTableDefinition { typedef BaseHashCacheStrategy ParentType; - typedef void * KeyType; + typedef const void * KeyType; typedef Link ValueType; HashTableDefinition(BaseHashCacheStrategy *_parent) : parent(_parent) {} - size_t HashKey(void *key) const + size_t HashKey(const void *key) const { - return (((uint8_t *)key) - ((uint8_t *)0)) >> parent->fLowerBoundary; + return (((const uint8_t *)key) + - ((const uint8_t *)0)) >> parent->fLowerBoundary; } size_t Hash(Link *value) const { return HashKey(value->buffer); } - bool Compare(void *key, Link *value) const + bool Compare(const void *key, Link *value) const { return value->buffer == key; } @@ -66,15 +67,9 @@ struct BaseHashCacheStrategy { BaseHashCacheStrategy(base_cache *parent) : fHashTable(this), fLowerBoundary(__Fls0(parent->object_size)) {} - void *Object(cache_object_link *link) const + cache_slab *ObjectSlab(void *object) const { - return ((Link *)link)->buffer; - } - - CacheObjectInfo ObjectInformation(void *object) const - { - Link *link = _Linkage(object); - return CacheObjectInfo(link->slab, link); + return _Linkage(object)->slab; } protected: @@ -86,11 +81,6 @@ protected: return link; } - static cache_object_link *_Linkage(void *_this, void *object) - { - return ((BaseHashCacheStrategy *)_this)->_Linkage(object); - } - HashTable fHashTable; const size_t fLowerBoundary; }; @@ -106,11 +96,6 @@ struct HashCacheStrategy : BaseCacheStrategy, BaseHashCacheStrategy { : BaseCacheStrategy(parent), BaseHashCacheStrategy(parent), fSlabCache("slab cache", 0), fLinkCache("link cache", 0) {} - static size_t RequiredSpace(size_t objectSize) - { - return objectSize; - } - BaseSlab *NewSlab(uint32_t flags) { size_t byteCount = _SlabSize(); @@ -129,8 +114,7 @@ struct HashCacheStrategy : BaseCacheStrategy, BaseHashCacheStrategy { // it's very important that we cast this to BaseHashCacheStrategy // so we get the proper instance offset through void * cache_slab *result = BaseCacheStrategy::_ConstructSlab(slab, - pages, _SlabSize(), (BaseHashCacheStrategy *)this, _Linkage, - _PrepareObject, _UnprepareObject); + pages, _SlabSize(), this, _PrepareObject, _UnprepareObject); if (result == NULL) { Backend::FreePages(Parent(), slab->id); fSlabCache.Free(slab); @@ -141,8 +125,7 @@ struct HashCacheStrategy : BaseCacheStrategy, BaseHashCacheStrategy { void ReturnSlab(BaseSlab *slab) { - _ClearSlab(slab->pages, _SlabSize()); - BaseCacheStrategy::_DestructSlab(slab); + BaseCacheStrategy::_DestructSlab(slab, this, _UnprepareObject); fSlabCache.Free((Slab *)slab); } @@ -156,8 +139,7 @@ private: static status_t _PrepareObject(void *_self, cache_slab *slab, void *object) { - BaseHashCacheStrategy *base = (BaseHashCacheStrategy *)_self; - Strategy *self = (Strategy *)base; + Strategy *self = (Strategy *)_self; Link *link = self->fLinkCache.Alloc(CACHE_DONT_SLEEP); if (link == NULL) @@ -173,8 +155,7 @@ private: static void _UnprepareObject(void *_self, cache_slab *slab, void *object) { - BaseHashCacheStrategy *base = (BaseHashCacheStrategy *)_self; - ((Strategy *)base)->_UnprepareObject(object); + ((Strategy *)_self)->_UnprepareObject(object); } void _UnprepareObject(void *object) @@ -184,15 +165,6 @@ private: fLinkCache.Free(link); } - void _ClearSlab(void *pages, size_t size) - { - uint8_t *data = (uint8_t *)pages; - uint8_t *end = data + size; - - for (uint8_t *it = data; it < end; it += Parent()->object_size) - _UnprepareObject(it); - } - TypedCache fSlabCache; TypedCache fLinkCache; }; diff --git a/headers/private/kernel/slab/MergedStrategy.h b/headers/private/kernel/slab/MergedStrategy.h index d6f8b53d80..4dcf587e0d 100644 --- a/headers/private/kernel/slab/MergedStrategy.h +++ b/headers/private/kernel/slab/MergedStrategy.h @@ -21,21 +21,10 @@ class MergedLinkCacheStrategy : public BaseCacheStrategy { public: typedef typename BaseCacheStrategy::BaseSlab BaseSlab; typedef typename BaseCacheStrategy::Slab Slab; - typedef cache_object_link Link; MergedLinkCacheStrategy(base_cache *parent) : BaseCacheStrategy(parent) {} - static size_t RequiredSpace(size_t objectSize) - { - return objectSize + sizeof(Link); - } - - void *Object(Link *link) const - { - return ((uint8_t *)link) - (Parent()->object_size - sizeof(Link)); - } - static inline const void * LowerBoundary(void *object, size_t byteCount) { @@ -43,10 +32,9 @@ public: return null + ((((uint8_t *)object) - null) & ~(byteCount - 1)); } - CacheObjectInfo ObjectInformation(void *object) const + BaseSlab *ObjectSlab(void *object) const { - Slab *slab = _SlabInPages(LowerBoundary(object, _SlabSize())); - return CacheObjectInfo(slab, _Linkage(object)); + return _SlabInPages(LowerBoundary(object, _SlabSize())); } BaseSlab *NewSlab(uint32_t flags) @@ -65,12 +53,12 @@ public: _SlabInPages(pages)->id = id; return BaseCacheStrategy::_ConstructSlab(_SlabInPages(pages), - pages, byteCount - sizeof(Slab), this, _Linkage, NULL, NULL); + pages, byteCount - sizeof(Slab), this, NULL, NULL); } void ReturnSlab(BaseSlab *slab) { - BaseCacheStrategy::_DestructSlab(slab); + BaseCacheStrategy::_DestructSlab(slab, NULL, NULL); } private: @@ -84,108 +72,10 @@ private: base_cache *Parent() const { return BaseCacheStrategy::Parent(); } - Link *_Linkage(void *object) const - { - return (Link *)(((uint8_t *)object) - + (Parent()->object_size - sizeof(Link))); - } - Slab *_SlabInPages(const void *pages) const { return (Slab *)(((uint8_t *)pages) + _SlabSize() - sizeof(Slab)); } - - static Link *_Linkage(void *_this, void *object) - { - return ((MergedLinkCacheStrategy *)_this)->_Linkage(object); - } -}; - -// This slab strategy includes the ObjectLink at the end of each object and the -// slab at the end of the allocated pages. It maintains a pointer to the owning -// slab in the ObjectLink. This is optimized for medium sized objects whose -// length is not a power of 2. -template -class MergedLinkAndSlabCacheStrategy : public BaseCacheStrategy { -public: - typedef MergedLinkAndSlabCacheStrategy Strategy; - typedef typename BaseCacheStrategy::BaseSlab BaseSlab; - typedef typename BaseCacheStrategy::Slab Slab; - - struct Link : cache_object_link { - cache_slab *slab; - }; - - MergedLinkAndSlabCacheStrategy(base_cache *parent) - : BaseCacheStrategy(parent) {} - - static size_t RequiredSpace(size_t objectSize) - { - return objectSize + sizeof(Link); - } - - void *Object(cache_object_link *_link) const - { - Link *link = static_cast(_link); - - return ((uint8_t *)link) - (Parent()->object_size - sizeof(Link)); - } - - CacheObjectInfo ObjectInformation(void *object) const - { - Link *link = _Linkage(object); - return CacheObjectInfo(link->slab, link); - } - - BaseSlab *NewSlab(uint32_t flags) - { - typename Backend::AllocationID id; - void *pages; - - size_t size = _SlabSize(); - if (Backend::AllocatePages(Parent(), &id, &pages, size, flags) < B_OK) - return NULL; - - _SlabInPages(pages)->id = id; - - return BaseCacheStrategy::_ConstructSlab(_SlabInPages(pages), - pages, size - sizeof(Slab), this, _Linkage, _PrepareObject, NULL); - } - - void ReturnSlab(BaseSlab *slab) - { - BaseCacheStrategy::_DestructSlab(slab); - } - -private: - size_t _SlabSize() const - { - return BaseCacheStrategy::SlabSize(sizeof(Slab)); - } - - base_cache *Parent() const { return BaseCacheStrategy::Parent(); } - - Link *_Linkage(void *_object) const - { - uint8_t *object = (uint8_t *)_object; - return (Link *)(object + (Parent()->object_size - sizeof(Link))); - } - - Slab *_SlabInPages(const void *pages) const - { - return (Slab *)(((uint8_t *)pages) + _SlabSize() - sizeof(Slab)); - } - - static cache_object_link *_Linkage(void *_this, void *object) - { - return static_cast(_this)->_Linkage(object); - } - - static status_t _PrepareObject(void *_this, cache_slab *slab, void *object) - { - static_cast(_this)->_Linkage(object)->slab = slab; - return B_OK; - } }; #endif diff --git a/headers/private/kernel/slab/Strategy.h b/headers/private/kernel/slab/Strategy.h index b4bf29404d..1929aa1693 100644 --- a/headers/private/kernel/slab/Strategy.h +++ b/headers/private/kernel/slab/Strategy.h @@ -15,7 +15,6 @@ template class BaseCacheStrategy { protected: - typedef cache_object_link ObjectLink; typedef cache_slab BaseSlab; BaseCacheStrategy(base_cache *parent) @@ -35,16 +34,17 @@ protected: }; BaseSlab *_ConstructSlab(Slab *slab, void *pages, size_t byteCount, - void *parent, ObjectLink *(*getLink)(void *parent, void *object), - base_cache_owner_prepare prepare, base_cache_owner_unprepare unprepare) + void *parent, base_cache_owner_prepare prepare, + base_cache_owner_unprepare unprepare) { return base_cache_construct_slab(fParent, slab, pages, byteCount, - parent, getLink, prepare, unprepare); + parent, prepare, unprepare); } - void _DestructSlab(BaseSlab *slab) + void _DestructSlab(BaseSlab *slab, void *parent, + base_cache_owner_unprepare unprepare) { - base_cache_destruct_slab(fParent, slab); + base_cache_destruct_slab(fParent, slab, parent, unprepare); Backend::FreePages(fParent, ((Slab *)slab)->id); } diff --git a/src/add-ons/kernel/network/stack/net_buffer.cpp b/src/add-ons/kernel/network/stack/net_buffer.cpp index aed10c4f9f..a3659b1945 100644 --- a/src/add-ons/kernel/network/stack/net_buffer.cpp +++ b/src/add-ons/kernel/network/stack/net_buffer.cpp @@ -65,7 +65,7 @@ struct net_buffer_private : net_buffer { }; -typedef MergedLinkAndSlabCacheStrategy AreaMergedCacheStrategy; +typedef MergedLinkCacheStrategy AreaMergedCacheStrategy; typedef HashCacheStrategy AreaHashCacheStrategy; typedef Cache NetBufferCache; diff --git a/src/system/kernel/slab/Slab.cpp b/src/system/kernel/slab/Slab.cpp index b01b216c95..c25284c387 100644 --- a/src/system/kernel/slab/Slab.cpp +++ b/src/system/kernel/slab/Slab.cpp @@ -21,7 +21,7 @@ // same code. We'll have to resolve all of the dependencies // then, for now, it is still not required. -//#define TRACE_SLAB +#define TRACE_SLAB #ifdef TRACE_SLAB #define TRACE_CACHE(cache, format, args...) \ @@ -50,15 +50,12 @@ public: typedef MergedLinkCacheStrategy SmallObjectStrategy; -typedef MergedLinkAndSlabCacheStrategy MediumObjectStrategy; -typedef HashCacheStrategy BigObjectStrategy; +typedef HashCacheStrategy LargeObjectStrategy; typedef Cache SmallObjectCache; -typedef Cache MediumObjectCache; -typedef Cache BigObjectCache; +typedef Cache LargeObjectCache; -// SmallObjectCache: one word overhead per object, needs pages to be aligned class SmallObjectAbstractCache : public AbstractCache, public LocalCache { public: @@ -74,30 +71,12 @@ public: }; -// MediumObjectCache: two words overhead per object -class MediumObjectAbstractCache : public AbstractCache, - public LocalCache { +class LargeObjectAbstractCache : public AbstractCache, + public LocalCache { public: - typedef LocalCache Base; + typedef LocalCache Base; - MediumObjectAbstractCache(const char *name, size_t objectSize, - size_t alignment, base_cache_constructor constructor, - base_cache_destructor destructor, void *cookie) - : Base(name, objectSize, alignment, constructor, destructor, cookie) {} - - void *Allocate(uint32_t flags) { return Base::Alloc(flags); } - void Return(void *object) { Base::Free(object); } -}; - - -// BigObjectCache: uses an hash table to map objects to links but packs -// the objects tightly in the pages. Good for power of 2 lengths. -class BigObjectAbstractCache : public AbstractCache, - public LocalCache { -public: - typedef LocalCache Base; - - BigObjectAbstractCache(const char *name, size_t objectSize, + LargeObjectAbstractCache(const char *name, size_t objectSize, size_t alignment, base_cache_constructor constructor, base_cache_destructor destructor, void *cookie) : Base(name, objectSize, alignment, constructor, destructor, cookie) {} @@ -127,6 +106,21 @@ SListPush(Type* &head, Type *object) } +static inline void * +_LinkToObject(cache_object_link *link, size_t objectSize) +{ + return ((uint8_t *)link) - (objectSize - sizeof(cache_object_link)); +} + + +static inline cache_object_link * +_ObjectToLink(void *object, size_t objectSize) +{ + return (cache_object_link *)(((uint8_t *)object) + + (objectSize - sizeof(cache_object_link))); +} + + status_t slab_area_backend_allocate(base_cache *cache, area_id *id, void **pages, size_t byteCount, uint32_t flags) @@ -162,6 +156,9 @@ base_cache_init(base_cache *cache, const char *name, size_t objectSize, { strlcpy(cache->name, name, sizeof(cache->name)); + if (objectSize < sizeof(void *) && alignment < sizeof(void *)) + objectSize = sizeof(void *); + if (alignment > 0 && (objectSize & (alignment - 1))) cache->object_size = objectSize + alignment - (objectSize & (alignment - 1)); @@ -245,7 +242,7 @@ base_cache_low_memory(base_cache *cache, int32 level, } -cache_object_link * +void * base_cache_allocate_object(base_cache *cache) { cache_slab *slab; @@ -274,11 +271,11 @@ base_cache_allocate_object(base_cache *cache) list_add_item(&cache->full, slab); } - return link; + return _LinkToObject(link, cache->object_size); } -cache_object_link * +void * base_cache_allocate_object_with_new_slab(base_cache *cache, cache_slab *newSlab) { @@ -289,9 +286,10 @@ base_cache_allocate_object_with_new_slab(base_cache *cache, int base_cache_return_object(base_cache *cache, cache_slab *slab, - cache_object_link *link) + void *object) { // We return true if the slab is completely unused. + cache_object_link *link = _ObjectToLink(object, cache->object_size); TRACE_CACHE(cache, "returning %p to %p, %lu used (%lu empty slabs).", link, slab, slab->size - slab->count, cache->empty_count); @@ -318,7 +316,6 @@ base_cache_return_object(base_cache *cache, cache_slab *slab, cache_slab * base_cache_construct_slab(base_cache *cache, cache_slab *slab, void *pages, size_t byteCount, void *parent, - cache_object_link *(*getLink)(void *parent, void *object), base_cache_owner_prepare prepare, base_cache_owner_unprepare unprepare) { TRACE_CACHE(cache, "construct (%p, %p, %lu)", slab, pages, byteCount); @@ -328,17 +325,17 @@ base_cache_construct_slab(base_cache *cache, cache_slab *slab, void *pages, slab->free = NULL; size_t spareBytes = byteCount - (slab->size * cache->object_size); - size_t cycle = cache->cache_color_cycle; + slab->offset = cache->cache_color_cycle; - if (cycle > spareBytes) - cache->cache_color_cycle = cycle = 0; + if (slab->offset > spareBytes) + cache->cache_color_cycle = slab->offset = 0; else cache->cache_color_cycle += kCacheColorPeriod; - TRACE_CACHE(cache, " %lu objects, %lu spare bytes, cycle %lu", - slab->size, spareBytes, cycle); + TRACE_CACHE(cache, " %lu objects, %lu spare bytes, offset %lu", + slab->size, spareBytes, slab->offset); - uint8_t *data = ((uint8_t *)pages) + cycle; + uint8_t *data = ((uint8_t *)pages) + slab->offset; for (size_t i = 0; i < slab->size; i++) { if (prepare || cache->constructor) { @@ -356,7 +353,7 @@ base_cache_construct_slab(base_cache *cache, cache_slab *slab, void *pages, if (!failedOnFirst && unprepare) unprepare(parent, slab, data); - data = ((uint8_t *)pages) + cycle; + data = ((uint8_t *)pages) + slab->offset; for (size_t j = 0; j < i; j++) { if (cache->destructor) cache->destructor(cache->cookie, data); @@ -369,7 +366,7 @@ base_cache_construct_slab(base_cache *cache, cache_slab *slab, void *pages, } } - SListPush(slab->free, getLink(parent, data)); + SListPush(slab->free, _ObjectToLink(data, cache->object_size)); data += cache->object_size; } @@ -378,20 +375,21 @@ base_cache_construct_slab(base_cache *cache, cache_slab *slab, void *pages, void -base_cache_destruct_slab(base_cache *cache, cache_slab *slab) +base_cache_destruct_slab(base_cache *cache, cache_slab *slab, void *parent, + base_cache_owner_unprepare unprepare) { TRACE_CACHE(cache, "destruct %p", slab); - if (cache->destructor == NULL) - return; - if (slab->count != slab->size) panic("cache: destroying a slab which isn't empty."); - uint8_t *data = (uint8_t *)slab->pages; + uint8_t *data = ((uint8_t *)slab->pages) + slab->offset; for (size_t i = 0; i < slab->size; i++) { - cache->destructor(cache->cookie, data); + if (cache->destructor) + cache->destructor(cache->cookie, data); + if (unprepare) + unprepare(parent, slab, data); data += cache->object_size; } } @@ -616,19 +614,6 @@ base_depot_make_empty(base_depot *depot) } -static inline int -__Fls(size_t value) -{ - if (value == 0) - return -1; - - int bit; - for (bit = 1; value != 1; bit++) - value >>= 1; - return bit; -} - - object_cache_t object_cache_create(const char *name, size_t object_size, size_t alignment, status_t (*_constructor)(void *, void *), void (*_destructor)(void *, @@ -636,18 +621,11 @@ object_cache_create(const char *name, size_t object_size, size_t alignment, { if (object_size == 0) return NULL; - else if (object_size < 64) + else if (object_size <= 256) return new (std::nothrow) SmallObjectAbstractCache(name, object_size, alignment, _constructor, _destructor, cookie); - size_t upperBoundary = 1 << __Fls(object_size); - - if (object_size <= 256 - || (upperBoundary - object_size) >= (4 * sizeof(void *))) - return new (std::nothrow) MediumObjectAbstractCache(name, object_size, - alignment, _constructor, _destructor, cookie); - - return new (std::nothrow) BigObjectAbstractCache(name, object_size, + return new (std::nothrow) LargeObjectAbstractCache(name, object_size, alignment, _constructor, _destructor, cookie); }