From b4e5e4982360e684c5a13d227b9a958dbe725554 Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Mon, 25 Jan 2010 13:46:58 +0000 Subject: [PATCH] MemoryManager: * Added support to do larger raw allocations (up to one large chunk (128 pages)) in the slab areas. For an even larger allocation an area is created (haven't seen that happen yet, though). * Added kernel tracing (SLAB_MEMORY_MANAGER_TRACING). * _FreeArea(): Copy and paste bug: The meta chunks of the to be freed area would be added to the free lists instead of being removed from them. This would corrupt the lists and also lead to all kinds of misuse of meta chunks. object caches: * Implemented CACHE_ALIGN_ON_SIZE. It is no longer set for all small object caches, but the block allocator sets it on all power of two size caches. * object_cache_reserve_internal(): Detect recursion and don't wait in such a case. The function could deadlock itself, since HashedObjectCache::CreateSlab() does allocate memory, thus potentially reentering. * object_cache_low_memory(): - I missed some returns when reworking that one in r35254, so the function might stop early and also leave the cache in maintenance mode, which would cause it to be ignored by object cache resizer and low memory handler from that point on. - Since ReturnSlab() potentially unlocks, the conditions weren't quite correct and too many slabs could be freed. - Simplified things a bit. * object_cache_alloc(): Since object_cache_reserve_internal() does potentially unlock the cache, the situation might have changed and their might not be an empty slab available, but a partial one. The function would crash. * Renamed the object cache tracing variable to SLAB_OBJECT_CACHE_TRACING. * Renamed debugger command "cache_info" to "slab_cache" to avoid confusion with the VMCache commands. * ObjectCache::usage was not maintained anymore since I introduced the MemoryManager. object_cache_get_usage() would thus always return 0 and the block cache would not be considered cached memory. This was only of informational relevance, though. slab allocator misc.: * Disable the object depots of block allocator caches for object sizes > 2 KB. Allocations of those sizes aren't so common that the object depots yield any benefit. * The slab allocator is now fully self-sufficient. It allocates its bootstrap memory from the MemoryManager, and the hash tables for HashedObjectCaches use the block allocator instead of the heap, now. * Added option to use the slab allocator for malloc() and friends (USE_SLAB_ALLOCATOR_FOR_MALLOC). Currently disabled. Works in principle and has virtually no lock contention. Handling for low memory situations is yet missing, though. * Improved the output of some debugger commands. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@35283 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- build/config_headers/kernel_debug_config.h | 3 + build/config_headers/tracing_config.h | 3 +- headers/private/kernel/vm/vm.h | 3 +- src/system/kernel/heap.cpp | 12 + src/system/kernel/slab/HashedObjectCache.cpp | 71 +- src/system/kernel/slab/HashedObjectCache.h | 18 +- src/system/kernel/slab/MemoryManager.cpp | 848 +++++++++++++++++-- src/system/kernel/slab/MemoryManager.h | 34 +- src/system/kernel/slab/ObjectCache.cpp | 19 +- src/system/kernel/slab/ObjectCache.h | 1 + src/system/kernel/slab/Slab.cpp | 113 +-- src/system/kernel/slab/SmallObjectCache.cpp | 5 +- src/system/kernel/slab/allocator.cpp | 193 ++++- src/system/kernel/slab/slab_private.h | 29 +- src/system/kernel/vm/vm.cpp | 12 +- 15 files changed, 1147 insertions(+), 217 deletions(-) diff --git a/build/config_headers/kernel_debug_config.h b/build/config_headers/kernel_debug_config.h index bfed08c20a..08aec0c48d 100644 --- a/build/config_headers/kernel_debug_config.h +++ b/build/config_headers/kernel_debug_config.h @@ -94,6 +94,9 @@ // Enables swap support. #define ENABLE_SWAP_SUPPORT 1 +// Use the slab allocator as generic memory allocator (malloc()/free()). +#define USE_SLAB_ALLOCATOR_FOR_MALLOC 0 + // When set limits the amount of available RAM (in MB). //#define LIMIT_AVAILABLE_MEMORY 256 diff --git a/build/config_headers/tracing_config.h b/build/config_headers/tracing_config.h index 94594a79c2..e85427fa86 100644 --- a/build/config_headers/tracing_config.h +++ b/build/config_headers/tracing_config.h @@ -30,7 +30,6 @@ #define KTRACE_PRINTF_STACK_TRACE 0 /* stack trace depth */ #define NET_BUFFER_TRACING 0 #define NET_BUFFER_TRACING_STACK_TRACE 0 /* stack trace depth */ -#define OBJECT_CACHE_TRACING 0 #define PAGE_ALLOCATION_TRACING 0 #define PAGE_DAEMON_TRACING 0 #define PAGE_WRITER_TRACING 0 @@ -41,6 +40,8 @@ #define SCHEDULER_TRACING 0 #define SCHEDULING_ANALYSIS_TRACING 0 #define SIGNAL_TRACING 0 +#define SLAB_MEMORY_MANAGER_TRACING 0 +#define SLAB_OBJECT_CACHE_TRACING 0 #define SWAP_TRACING 0 #define SYSCALL_TRACING 0 #define SYSCALL_TRACING_IGNORE_KTRACE_OUTPUT 1 diff --git a/headers/private/kernel/vm/vm.h b/headers/private/kernel/vm/vm.h index ca1137ebc5..106bedfa55 100644 --- a/headers/private/kernel/vm/vm.h +++ b/headers/private/kernel/vm/vm.h @@ -49,8 +49,7 @@ void vm_free_unused_boot_loader_range(addr_t start, addr_t end); addr_t vm_allocate_early(struct kernel_args *args, size_t virtualSize, size_t physicalSize, uint32 attributes, bool blockAlign); -void slab_init(struct kernel_args *args, addr_t initialBase, - size_t initialSize); +void slab_init(struct kernel_args *args); void slab_init_post_area(); void slab_init_post_sem(); void slab_init_post_thread(); diff --git a/src/system/kernel/heap.cpp b/src/system/kernel/heap.cpp index 47356b9de8..42092714ea 100644 --- a/src/system/kernel/heap.cpp +++ b/src/system/kernel/heap.cpp @@ -1687,6 +1687,9 @@ heap_set_get_caller(heap_allocator* heap, addr_t (*getCaller)()) #endif +#if !USE_SLAB_ALLOCATOR_FOR_MALLOC + + static status_t heap_realloc(heap_allocator *heap, void *address, void **newAddress, size_t newSize) @@ -1794,6 +1797,9 @@ heap_realloc(heap_allocator *heap, void *address, void **newAddress, } +#endif // !USE_SLAB_ALLOCATOR_FOR_MALLOC + + inline uint32 heap_index_for(size_t size, int32 cpu) { @@ -2045,6 +2051,9 @@ heap_init_post_thread() // #pragma mark - Public API +#if !USE_SLAB_ALLOCATOR_FOR_MALLOC + + void * memalign(size_t alignment, size_t size) { @@ -2312,6 +2321,9 @@ realloc(void *address, size_t newSize) } +#endif // !USE_SLAB_ALLOCATOR_FOR_MALLOC + + void * calloc(size_t numElements, size_t size) { diff --git a/src/system/kernel/slab/HashedObjectCache.cpp b/src/system/kernel/slab/HashedObjectCache.cpp index 3d64b973ef..eb50df4f3d 100644 --- a/src/system/kernel/slab/HashedObjectCache.cpp +++ b/src/system/kernel/slab/HashedObjectCache.cpp @@ -61,6 +61,16 @@ HashedObjectCache::Create(const char* name, size_t object_size, HashedObjectCache* cache = new(buffer) HashedObjectCache(); + // init the hash table + size_t hashSize = cache->hash_table.ResizeNeeded(); + buffer = slab_internal_alloc(hashSize, flags); + if (buffer == NULL) { + cache->Delete(); + return NULL; + } + + cache->hash_table.Resize(buffer, hashSize, true); + if (cache->Init(name, object_size, alignment, maximum, flags, cookie, constructor, destructor, reclaimer) != B_OK) { cache->Delete(); @@ -96,21 +106,20 @@ HashedObjectCache::CreateSlab(uint32 flags) Unlock(); slab* slab = allocate_slab(flags); + if (slab != NULL) { + void* pages; + if (MemoryManager::Allocate(this, flags, pages) == B_OK) { + Lock(); + if (InitSlab(slab, pages, slab_size, flags)) + return slab; + Unlock(); + MemoryManager::Free(pages, flags); + } - Lock(); - - if (slab == NULL) - return NULL; - - void* pages; - if (MemoryManager::Allocate(this, flags, pages) == B_OK) { - if (InitSlab(slab, pages, slab_size, flags)) - return slab; - - MemoryManager::Free(pages, flags); + free_slab(slab, flags); } - free_slab(slab, flags); + Lock(); return NULL; } @@ -119,8 +128,10 @@ void HashedObjectCache::ReturnSlab(slab* slab, uint32 flags) { UninitSlab(slab); + Unlock(); MemoryManager::Free(slab->pages, flags); free_slab(slab, flags); + Lock(); } @@ -147,11 +158,9 @@ HashedObjectCache::PrepareObject(slab* source, void* object, uint32 flags) link->buffer = object; link->parent = source; - hash_table.Insert(link); - // TODO: This might resize the table! Currently it uses the heap, so - // we won't possibly reenter and deadlock on our own cache. We do ignore - // the flags, though! - // TODO: We don't pre-init the table, so Insert() can fail! + hash_table.InsertUnchecked(link); + _ResizeHashTableIfNeeded(flags); + return B_OK; } @@ -170,11 +179,37 @@ HashedObjectCache::UnprepareObject(slab* source, void* object, uint32 flags) return; } - hash_table.Remove(link); + hash_table.RemoveUnchecked(link); + _ResizeHashTableIfNeeded(flags); + _FreeLink(link, flags); } +void +HashedObjectCache::_ResizeHashTableIfNeeded(uint32 flags) +{ + size_t hashSize = hash_table.ResizeNeeded(); + if (hashSize != 0) { + Unlock(); + void* buffer = slab_internal_alloc(hashSize, flags); + Lock(); + + if (buffer != NULL) { + if (hash_table.ResizeNeeded() == hashSize) { + void* oldHash; + hash_table.Resize(buffer, hashSize, true, &oldHash); + if (oldHash != NULL) { + Unlock(); + slab_internal_free(oldHash, flags); + Lock(); + } + } + } + } +} + + /*static*/ inline HashedObjectCache::Link* HashedObjectCache::_AllocateLink(uint32 flags) { diff --git a/src/system/kernel/slab/HashedObjectCache.h b/src/system/kernel/slab/HashedObjectCache.h index 2bad679d7c..cba0630e00 100644 --- a/src/system/kernel/slab/HashedObjectCache.h +++ b/src/system/kernel/slab/HashedObjectCache.h @@ -11,6 +11,7 @@ #include #include "ObjectCache.h" +#include "slab_private.h" struct HashedObjectCache : ObjectCache { @@ -81,11 +82,26 @@ private: HashedObjectCache* parent; }; - typedef BOpenHashTable HashTable; + struct InternalAllocator { + void* Allocate(size_t size) const + { + return slab_internal_alloc(size, 0); + } + + void Free(void* memory) const + { + slab_internal_free(memory, 0); + } + }; + + typedef BOpenHashTable HashTable; friend class Definition; private: + void _ResizeHashTableIfNeeded(uint32 flags); + static Link* _AllocateLink(uint32 flags); static void _FreeLink(HashedObjectCache::Link* link, uint32 flags); diff --git a/src/system/kernel/slab/MemoryManager.cpp b/src/system/kernel/slab/MemoryManager.cpp index a7251ecb60..e9df60f602 100644 --- a/src/system/kernel/slab/MemoryManager.cpp +++ b/src/system/kernel/slab/MemoryManager.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -50,6 +51,369 @@ MemoryManager::AllocationEntry* MemoryManager::sAllocationEntryDontWait; bool MemoryManager::sMaintenanceNeeded; + +// #pragma mark - kernel tracing + + +#if SLAB_MEMORY_MANAGER_TRACING + + +//namespace SlabMemoryManagerCacheTracing { +struct MemoryManager::Tracing { + +class MemoryManagerTraceEntry : public AbstractTraceEntry { +public: + MemoryManagerTraceEntry() + { + } +}; + + +class Allocate : public MemoryManagerTraceEntry { +public: + Allocate(ObjectCache* cache, uint32 flags) + : + MemoryManagerTraceEntry(), + fCache(cache), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager alloc: cache: %p, flags: %#" B_PRIx32, + fCache, fFlags); + } + +private: + ObjectCache* fCache; + uint32 fFlags; +}; + + +class Free : public MemoryManagerTraceEntry { +public: + Free(void* address, uint32 flags) + : + MemoryManagerTraceEntry(), + fAddress(address), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager free: address: %p, flags: %#" B_PRIx32, + fAddress, fFlags); + } + +private: + void* fAddress; + uint32 fFlags; +}; + + +class AllocateRaw : public MemoryManagerTraceEntry { +public: + AllocateRaw(size_t size, uint32 flags) + : + MemoryManagerTraceEntry(), + fSize(size), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager alloc raw: size: %" B_PRIuSIZE + ", flags: %#" B_PRIx32, fSize, fFlags); + } + +private: + size_t fSize; + uint32 fFlags; +}; + + +class FreeRawOrReturnCache : public MemoryManagerTraceEntry { +public: + FreeRawOrReturnCache(void* address, uint32 flags) + : + MemoryManagerTraceEntry(), + fAddress(address), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager free raw/return: address: %p, flags: %#" + B_PRIx32, fAddress, fFlags); + } + +private: + void* fAddress; + uint32 fFlags; +}; + + +class AllocateArea : public MemoryManagerTraceEntry { +public: + AllocateArea(Area* area, uint32 flags) + : + MemoryManagerTraceEntry(), + fArea(area), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager alloc area: flags: %#" B_PRIx32 + " -> %p", fFlags, fArea); + } + +private: + Area* fArea; + uint32 fFlags; +}; + + +class AddArea : public MemoryManagerTraceEntry { +public: + AddArea(Area* area) + : + MemoryManagerTraceEntry(), + fArea(area) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager add area: %p", fArea); + } + +private: + Area* fArea; +}; + + +class FreeArea : public MemoryManagerTraceEntry { +public: + FreeArea(Area* area, bool areaRemoved, uint32 flags) + : + MemoryManagerTraceEntry(), + fArea(area), + fFlags(flags), + fRemoved(areaRemoved) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager free area: %p%s, flags: %#" B_PRIx32, + fArea, fRemoved ? " (removed)" : "", fFlags); + } + +private: + Area* fArea; + uint32 fFlags; + bool fRemoved; +}; + + +class AllocateMetaChunk : public MemoryManagerTraceEntry { +public: + AllocateMetaChunk(MetaChunk* metaChunk) + : + MemoryManagerTraceEntry(), + fMetaChunk(metaChunk->chunkBase) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager alloc meta chunk: %#" B_PRIxADDR, + fMetaChunk); + } + +private: + addr_t fMetaChunk; +}; + + +class FreeMetaChunk : public MemoryManagerTraceEntry { +public: + FreeMetaChunk(MetaChunk* metaChunk) + : + MemoryManagerTraceEntry(), + fMetaChunk(metaChunk->chunkBase) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager free meta chunk: %#" B_PRIxADDR, + fMetaChunk); + } + +private: + addr_t fMetaChunk; +}; + + +class AllocateChunk : public MemoryManagerTraceEntry { +public: + AllocateChunk(size_t chunkSize, MetaChunk* metaChunk, Chunk* chunk) + : + MemoryManagerTraceEntry(), + fChunkSize(chunkSize), + fMetaChunk(metaChunk->chunkBase), + fChunk(chunk - metaChunk->chunks) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager alloc chunk: size: %" B_PRIuSIZE + " -> meta chunk: %#" B_PRIxADDR ", chunk: %" B_PRIu32, fChunkSize, + fMetaChunk, fChunk); + } + +private: + size_t fChunkSize; + addr_t fMetaChunk; + uint32 fChunk; +}; + + +class AllocateChunks : public MemoryManagerTraceEntry { +public: + AllocateChunks(size_t chunkSize, uint32 chunkCount, MetaChunk* metaChunk, + Chunk* chunk) + : + MemoryManagerTraceEntry(), + fMetaChunk(metaChunk->chunkBase), + fChunkSize(chunkSize), + fChunkCount(chunkCount), + fChunk(chunk - metaChunk->chunks) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager alloc chunks: size: %" B_PRIuSIZE + ", count %" B_PRIu32 " -> meta chunk: %#" B_PRIxADDR ", chunk: %" + B_PRIu32, fChunkSize, fChunkCount, fMetaChunk, fChunk); + } + +private: + addr_t fMetaChunk; + size_t fChunkSize; + uint32 fChunkCount; + uint32 fChunk; +}; + + +class FreeChunk : public MemoryManagerTraceEntry { +public: + FreeChunk(MetaChunk* metaChunk, Chunk* chunk) + : + MemoryManagerTraceEntry(), + fMetaChunk(metaChunk->chunkBase), + fChunk(chunk - metaChunk->chunks) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager free chunk: meta chunk: %#" B_PRIxADDR + ", chunk: %" B_PRIu32, fMetaChunk, fChunk); + } + +private: + addr_t fMetaChunk; + uint32 fChunk; +}; + + +class Map : public MemoryManagerTraceEntry { +public: + Map(addr_t address, size_t size, uint32 flags) + : + MemoryManagerTraceEntry(), + fAddress(address), + fSize(size), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager map: %#" B_PRIxADDR ", size: %" + B_PRIuSIZE ", flags: %#" B_PRIx32, fAddress, fSize, fFlags); + } + +private: + addr_t fAddress; + size_t fSize; + uint32 fFlags; +}; + + +class Unmap : public MemoryManagerTraceEntry { +public: + Unmap(addr_t address, size_t size, uint32 flags) + : + MemoryManagerTraceEntry(), + fAddress(address), + fSize(size), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager unmap: %#" B_PRIxADDR ", size: %" + B_PRIuSIZE ", flags: %#" B_PRIx32, fAddress, fSize, fFlags); + } + +private: + addr_t fAddress; + size_t fSize; + uint32 fFlags; +}; + + +//} // namespace SlabMemoryManagerCacheTracing +}; // struct MemoryManager::Tracing + + +//# define T(x) new(std::nothrow) SlabMemoryManagerCacheTracing::x +# define T(x) new(std::nothrow) MemoryManager::Tracing::x + +#else +# define T(x) +#endif // SLAB_MEMORY_MANAGER_TRACING + + +// #pragma mark - MemoryManager + + /*static*/ void MemoryManager::Init(kernel_args* args) { @@ -137,6 +501,10 @@ MemoryManager::InitPostArea() "Lists all non-full slab meta chunks.\n" "If \"-c\" is given, the chunks of all meta chunks area printed as " "well.\n", 0); + add_debugger_command_etc("slab_raw_allocations", &_DumpRawAllocations, + "List all raw allocations in slab areas", + "\n" + "Lists all raw allocations in slab areas.\n", 0); } @@ -145,6 +513,8 @@ MemoryManager::Allocate(ObjectCache* cache, uint32 flags, void*& _pages) { // TODO: Support CACHE_UNLOCKED_PAGES! + T(Allocate(cache, flags)); + size_t chunkSize = cache->slab_size; TRACE("MemoryManager::Allocate(%p, %#" B_PRIx32 "): chunkSize: %" @@ -155,7 +525,7 @@ MemoryManager::Allocate(ObjectCache* cache, uint32 flags, void*& _pages) // allocate a chunk MetaChunk* metaChunk; Chunk* chunk; - status_t error = _AllocateChunk(chunkSize, flags, metaChunk, chunk); + status_t error = _AllocateChunks(chunkSize, 1, flags, metaChunk, chunk); if (error != B_OK) return error; @@ -172,7 +542,7 @@ MemoryManager::Allocate(ObjectCache* cache, uint32 flags, void*& _pages) return error; } - chunk->cache = cache; + chunk->reference = (addr_t)cache; _pages = (void*)chunkAddress; TRACE("MemoryManager::Allocate() done: %p (meta chunk: %d, chunk %d)\n", @@ -187,6 +557,8 @@ MemoryManager::Free(void* pages, uint32 flags) { TRACE("MemoryManager::Free(%p, %#" B_PRIx32 ")\n", pages, flags); + T(Free(pages, flags)); + // get the area and the meta chunk Area* area = (Area*)ROUNDDOWN((addr_t)pages, SLAB_AREA_SIZE); MetaChunk* metaChunk = &area->metaChunks[ @@ -210,6 +582,137 @@ MemoryManager::Free(void* pages, uint32 flags) } +/*static*/ status_t +MemoryManager::AllocateRaw(size_t size, uint32 flags, void*& _pages) +{ + T(AllocateRaw(size, flags)); + + size = ROUNDUP(size, SLAB_CHUNK_SIZE_SMALL); + + TRACE("MemoryManager::AllocateRaw(%" B_PRIuSIZE ", %#" B_PRIx32 ")\n", size, + flags); + + if (size > SLAB_CHUNK_SIZE_LARGE || (flags & CACHE_ALIGN_ON_SIZE) != 0) { + // Requested size greater than a large chunk or an aligned allocation. + // Allocate as an area. + if ((flags & CACHE_DONT_LOCK_KERNEL_SPACE) != 0) + return B_WOULD_BLOCK; + + area_id area = create_area_etc(VMAddressSpace::KernelID(), + "slab large raw allocation", &_pages, + (flags & CACHE_ALIGN_ON_SIZE) != 0 + ? B_ANY_KERNEL_BLOCK_ADDRESS : B_ANY_KERNEL_ADDRESS, + size, B_FULL_LOCK, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0, + (flags & CACHE_DONT_WAIT_FOR_MEMORY) != 0 + ? CREATE_AREA_DONT_WAIT : 0); + return area >= 0 ? B_OK : area; + } + + // determine chunk size (small or medium) + size_t chunkSize = SLAB_CHUNK_SIZE_SMALL; + uint32 chunkCount = size / SLAB_CHUNK_SIZE_SMALL; + + if (size % SLAB_CHUNK_SIZE_MEDIUM == 0) { + chunkSize = SLAB_CHUNK_SIZE_MEDIUM; + chunkCount = size / SLAB_CHUNK_SIZE_MEDIUM; + } + + MutexLocker locker(sLock); + + // allocate the chunks + MetaChunk* metaChunk; + Chunk* chunk; + status_t error = _AllocateChunks(chunkSize, chunkCount, flags, metaChunk, + chunk); + if (error != B_OK) + return error; + + // map the chunks + Area* area = metaChunk->GetArea(); + addr_t chunkAddress = _ChunkAddress(metaChunk, chunk); + + locker.Unlock(); + error = _MapChunk(area->vmArea, chunkAddress, size, 0, flags); + locker.Lock(); + if (error != B_OK) { + // something failed -- free the chunks + for (uint32 i = 0; i < chunkCount; i++) + _FreeChunk(area, metaChunk, chunk + i, chunkAddress, true, flags); + return error; + } + + chunk->reference = (addr_t)chunkAddress + size - 1; + _pages = (void*)chunkAddress; + + TRACE("MemoryManager::AllocateRaw() done: %p (meta chunk: %d, chunk %d)\n", + _pages, int(metaChunk - area->metaChunks), + int(chunk - metaChunk->chunks)); + return B_OK; +} + + +/*static*/ ObjectCache* +MemoryManager::FreeRawOrReturnCache(void* pages, uint32 flags) +{ + TRACE("MemoryManager::FreeRawOrReturnCache(%p, %#" B_PRIx32 ")\n", pages, + flags); + + T(FreeRawOrReturnCache(pages, flags)); + + // get the area + addr_t areaBase = ROUNDDOWN((addr_t)pages, SLAB_AREA_SIZE); + + ReadLocker readLocker(sAreaTableLock); + Area* area = sAreaTable.Lookup(areaBase); + readLocker.Unlock(); + + if (area == NULL) { + // Probably a large allocation. Look up the VM area. + VMAddressSpace* addressSpace = VMAddressSpace::Kernel(); + addressSpace->ReadLock(); + VMArea* area = addressSpace->LookupArea((addr_t)pages); + addressSpace->ReadUnlock(); + + if (area != NULL && (addr_t)pages == area->Base()) + delete_area(area->id); + else + panic("freeing unknown block %p from area %p", pages, area); + + return NULL; + } + + MetaChunk* metaChunk = &area->metaChunks[ + ((addr_t)pages % SLAB_AREA_SIZE) / SLAB_CHUNK_SIZE_LARGE]; + + // get the chunk + ASSERT((addr_t)pages >= metaChunk->chunkBase); + uint16 chunkIndex = _ChunkIndexForAddress(metaChunk, (addr_t)pages); + Chunk* chunk = &metaChunk->chunks[chunkIndex]; + + addr_t reference = chunk->reference; + if ((reference & 1) == 0) + return (ObjectCache*)reference; + + // Seems we have a raw chunk allocation. + ASSERT((addr_t)pages == _ChunkAddress(metaChunk, chunk)); + ASSERT(reference > (addr_t)pages); + ASSERT(reference <= areaBase + SLAB_AREA_SIZE - 1); + size_t size = reference - (addr_t)pages + 1; + ASSERT((size % SLAB_CHUNK_SIZE_SMALL) == 0); + + // unmap the chunks + _UnmapChunk(area->vmArea, (addr_t)pages, size, flags); + + // and free them + MutexLocker locker(sLock); + uint32 chunkCount = size / metaChunk->chunkSize; + for (uint32 i = 0; i < chunkCount; i++) + _FreeChunk(area, metaChunk, chunk + i, (addr_t)pages, true, flags); + + return NULL; +} + + /*static*/ size_t MemoryManager::AcceptableChunkSize(size_t size) { @@ -221,6 +724,48 @@ MemoryManager::AcceptableChunkSize(size_t size) } +/*static*/ ObjectCache* +MemoryManager::GetAllocationInfo(void* address, size_t& _size) +{ + // get the area + addr_t areaBase = ROUNDDOWN((addr_t)address, SLAB_AREA_SIZE); + + ReadLocker readLocker(sAreaTableLock); + Area* area = sAreaTable.Lookup(areaBase); + readLocker.Unlock(); + + if (area == NULL) { + VMAddressSpace* addressSpace = VMAddressSpace::Kernel(); + addressSpace->ReadLock(); + VMArea* area = addressSpace->LookupArea((addr_t)address); + if (area != NULL && (addr_t)address == area->Base()) + _size = area->Size(); + else + _size = 0; + addressSpace->ReadUnlock(); + + return NULL; + } + + MetaChunk* metaChunk = &area->metaChunks[ + ((addr_t)address % SLAB_AREA_SIZE) / SLAB_CHUNK_SIZE_LARGE]; + + // get the chunk + ASSERT((addr_t)address >= metaChunk->chunkBase); + uint16 chunkIndex = _ChunkIndexForAddress(metaChunk, (addr_t)address); + + addr_t reference = metaChunk->chunks[chunkIndex].reference; + if ((reference & 1) == 0) { + ObjectCache* cache = (ObjectCache*)reference; + _size = cache->object_size; + return cache; + } + + _size = reference - (addr_t)address + 1; + return NULL; +} + + /*static*/ ObjectCache* MemoryManager::CacheForAddress(void* address) { @@ -241,7 +786,8 @@ MemoryManager::CacheForAddress(void* address) ASSERT((addr_t)address >= metaChunk->chunkBase); uint16 chunkIndex = _ChunkIndexForAddress(metaChunk, (addr_t)address); - return metaChunk->chunks[chunkIndex].cache; + addr_t reference = metaChunk->chunks[chunkIndex].reference; + return (reference & 1) == 0 ? (ObjectCache*)reference : NULL; } @@ -283,8 +829,8 @@ MemoryManager::PerformMaintenance() /*static*/ status_t -MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags, - MetaChunk*& _metaChunk, Chunk*& _chunk) +MemoryManager::_AllocateChunks(size_t chunkSize, uint32 chunkCount, + uint32 flags, MetaChunk*& _metaChunk, Chunk*& _chunk) { MetaChunkList* metaChunkList = NULL; if (chunkSize == SLAB_CHUNK_SIZE_SMALL) { @@ -292,12 +838,12 @@ MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags, } else if (chunkSize == SLAB_CHUNK_SIZE_MEDIUM) { metaChunkList = &sPartialMetaChunksMedium; } else if (chunkSize != SLAB_CHUNK_SIZE_LARGE) { - panic("MemoryManager::_AllocateChunk(): Unsupported chunk size: %" + panic("MemoryManager::_AllocateChunks(): Unsupported chunk size: %" B_PRIuSIZE, chunkSize); return B_BAD_VALUE; } - if (_GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk)) + if (_GetChunks(metaChunkList, chunkSize, chunkCount, _metaChunk, _chunk)) return B_OK; if (sFreeAreas != NULL) { @@ -305,7 +851,7 @@ MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags, sFreeAreaCount--; _RequestMaintenance(); - _GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk); + _GetChunks(metaChunkList, chunkSize, chunkCount, _metaChunk, _chunk); return B_OK; } @@ -334,8 +880,10 @@ MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags, entry.Wait(); mutex_lock(&sLock); - if (_GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk)) + if (_GetChunks(metaChunkList, chunkSize, chunkCount, _metaChunk, + _chunk)) { return B_OK; + } } // prepare the allocation entry others can wait on @@ -359,17 +907,94 @@ MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags, // Try again to get a meta chunk. Something might have been freed in the // meantime. We can free the area in this case. - if (_GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk)) { + if (_GetChunks(metaChunkList, chunkSize, chunkCount, _metaChunk, _chunk)) { _FreeArea(area, true, flags); return B_OK; } _AddArea(area); - _GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk); + _GetChunks(metaChunkList, chunkSize, chunkCount, _metaChunk, _chunk); return B_OK; } +/*static*/ bool +MemoryManager::_GetChunks(MetaChunkList* metaChunkList, size_t chunkSize, + uint32 chunkCount, MetaChunk*& _metaChunk, Chunk*& _chunk) +{ + // the common and less complicated special case + if (chunkCount == 1) + return _GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk); + + ASSERT(metaChunkList != NULL); + + // Iterate through the partial meta chunk list and try to find a free + // range that is large enough. + MetaChunk* metaChunk = NULL; + for (MetaChunkList::Iterator it = metaChunkList->GetIterator(); + (metaChunk = it.Next()) != NULL;) { + if (metaChunk->firstFreeChunk + chunkCount - 1 + <= metaChunk->lastFreeChunk) { + break; + } + } + + if (metaChunk == NULL) { + // try to get a free meta chunk + if ((SLAB_CHUNK_SIZE_LARGE - kAreaAdminSize) / chunkSize >= chunkCount) + metaChunk = sFreeShortMetaChunks.RemoveHead(); + if (metaChunk == NULL) + metaChunk = sFreeCompleteMetaChunks.RemoveHead(); + + if (metaChunk == NULL) + return false; + + metaChunkList->Add(metaChunk); + metaChunk->GetArea()->usedMetaChunkCount++; + _PrepareMetaChunk(metaChunk, chunkSize); + + T(AllocateMetaChunk(metaChunk)); + } + + // pull the chunks out of the free list + Chunk* firstChunk = metaChunk->chunks + metaChunk->firstFreeChunk; + Chunk* lastChunk = firstChunk + (chunkCount - 1); + Chunk** chunkPointer = &metaChunk->freeChunks; + uint32 remainingChunks = chunkCount; + while (remainingChunks > 0) { + ASSERT_PRINT(chunkPointer, "remaining: %" B_PRIu32 "/%" B_PRIu32 + ", area: %p, meta chunk: %" B_PRIdSSIZE "\n", remainingChunks, + chunkCount, metaChunk->GetArea(), + metaChunk - metaChunk->GetArea()->metaChunks); + Chunk* chunk = *chunkPointer; + if (chunk >= firstChunk && chunk <= lastChunk) { + *chunkPointer = chunk->next; + chunk->reference = 1; + remainingChunks--; + } else + chunkPointer = &chunk->next; + } + + // allocate the chunks + metaChunk->usedChunkCount += chunkCount; + if (metaChunk->usedChunkCount == metaChunk->chunkCount) { + // meta chunk is full now -- remove it from its list + if (metaChunkList != NULL) + metaChunkList->Remove(metaChunk); + } + + // update the free range + metaChunk->firstFreeChunk += chunkCount; + + _chunk = firstChunk; + _metaChunk = metaChunk; + + T(AllocateChunks(chunkSize, chunkCount, metaChunk, firstChunk)); + + return true; +} + + /*static*/ bool MemoryManager::_GetChunk(MetaChunkList* metaChunkList, size_t chunkSize, MetaChunk*& _metaChunk, Chunk*& _chunk) @@ -393,6 +1018,8 @@ MemoryManager::_GetChunk(MetaChunkList* metaChunkList, size_t chunkSize, metaChunk->GetArea()->usedMetaChunkCount++; _PrepareMetaChunk(metaChunk, chunkSize); + + T(AllocateMetaChunk(metaChunk)); } // allocate the chunk @@ -404,6 +1031,20 @@ MemoryManager::_GetChunk(MetaChunkList* metaChunkList, size_t chunkSize, _chunk = _pop(metaChunk->freeChunks); _metaChunk = metaChunk; + + // update the free range + uint32 chunkIndex = _chunk - metaChunk->chunks; + if (chunkIndex >= metaChunk->firstFreeChunk + && chunkIndex <= metaChunk->lastFreeChunk) { + if (chunkIndex - metaChunk->firstFreeChunk + <= metaChunk->lastFreeChunk - chunkIndex) { + metaChunk->firstFreeChunk = chunkIndex + 1; + } else + metaChunk->lastFreeChunk = chunkIndex - 1; + } + + T(AllocateChunk(chunkSize, metaChunk, _chunk)); + return true; } @@ -419,11 +1060,17 @@ MemoryManager::_FreeChunk(Area* area, MetaChunk* metaChunk, Chunk* chunk, mutex_lock(&sLock); } + T(FreeChunk(metaChunk, chunk)); + _push(metaChunk->freeChunks, chunk); + uint32 chunkIndex = chunk - metaChunk->chunks; + // free the meta chunk, if it is unused now ASSERT(metaChunk->usedChunkCount > 0); if (--metaChunk->usedChunkCount == 0) { + T(FreeMetaChunk(metaChunk)); + // remove from partial meta chunk list if (metaChunk->chunkSize == SLAB_CHUNK_SIZE_SMALL) sPartialMetaChunksSmall.Remove(metaChunk); @@ -450,6 +1097,28 @@ MemoryManager::_FreeChunk(Area* area, MetaChunk* metaChunk, Chunk* chunk, sPartialMetaChunksSmall.Add(metaChunk, false); else if (metaChunk->chunkSize == SLAB_CHUNK_SIZE_MEDIUM) sPartialMetaChunksMedium.Add(metaChunk, false); + + metaChunk->firstFreeChunk = chunkIndex; + metaChunk->lastFreeChunk = chunkIndex; + } else { + // extend the free range, if the chunk adjoins + if (chunkIndex + 1 == metaChunk->firstFreeChunk) { + uint32 firstFree = chunkIndex; + for (; firstFree > 0; firstFree--) { + Chunk* previousChunk = &metaChunk->chunks[firstFree - 1]; + if (!_IsChunkFree(metaChunk, previousChunk)) + break; + } + metaChunk->firstFreeChunk = firstFree; + } else if (chunkIndex == (uint32)metaChunk->lastFreeChunk + 1) { + uint32 lastFree = chunkIndex; + for (; lastFree + 1 < metaChunk->chunkCount; lastFree++) { + Chunk* nextChunk = &metaChunk->chunks[lastFree + 1]; + if (!_IsChunkFree(metaChunk, nextChunk)) + break; + } + metaChunk->lastFreeChunk = lastFree; + } } } @@ -471,14 +1140,19 @@ MemoryManager::_PrepareMetaChunk(MetaChunk* metaChunk, size_t chunkSize) metaChunk->usedChunkCount = 0; metaChunk->freeChunks = NULL; - for (uint32 i = 0; i < metaChunk->chunkCount; i++) + for (int32 i = metaChunk->chunkCount - 1; i >= 0; i--) _push(metaChunk->freeChunks, metaChunk->chunks + i); + + metaChunk->firstFreeChunk = 0; + metaChunk->lastFreeChunk = metaChunk->chunkCount - 1; } /*static*/ void MemoryManager::_AddArea(Area* area) { + T(AddArea(area)); + // add the area to the hash table WriteLocker writeLocker(sAreaTableLock); sAreaTable.InsertUnchecked(area); @@ -565,6 +1239,9 @@ MemoryManager::_AllocateArea(uint32 flags, Area*& _area) mutex_lock(&sLock); _area = area; + + T(AllocateArea(area, flags)); + return B_OK; } @@ -574,16 +1251,18 @@ MemoryManager::_FreeArea(Area* area, bool areaRemoved, uint32 flags) { TRACE("MemoryManager::_FreeArea(%p, %#" B_PRIx32 ")\n", area, flags); + T(FreeArea(area, areaRemoved, flags)); + ASSERT(area->usedMetaChunkCount == 0); if (!areaRemoved) { // remove the area's meta chunks from the free lists ASSERT(area->metaChunks[0].usedChunkCount == 0); - sFreeShortMetaChunks.Add(&area->metaChunks[0]); + sFreeShortMetaChunks.Remove(&area->metaChunks[0]); for (int32 i = 1; i < SLAB_META_CHUNKS_PER_AREA; i++) { ASSERT(area->metaChunks[i].usedChunkCount == 0); - sFreeCompleteMetaChunks.Add(&area->metaChunks[i]); + sFreeCompleteMetaChunks.Remove(&area->metaChunks[i]); } // remove the area from the hash table @@ -624,6 +1303,8 @@ MemoryManager::_MapChunk(VMArea* vmArea, addr_t address, size_t size, TRACE("MemoryManager::_MapChunk(%p, %#" B_PRIxADDR ", %#" B_PRIxSIZE ")\n", vmArea, address, size); + T(Map(address, size, flags)); + if (vmArea == NULL) { // everything is mapped anyway return B_OK; @@ -686,6 +1367,8 @@ MemoryManager::_MapChunk(VMArea* vmArea, addr_t address, size_t size, MemoryManager::_UnmapChunk(VMArea* vmArea, addr_t address, size_t size, uint32 flags) { + T(Unmap(address, size, flags)); + if (vmArea == NULL) return B_ERROR; @@ -728,32 +1411,6 @@ MemoryManager::_UnmapChunk(VMArea* vmArea, addr_t address, size_t size, } -/*static*/ void -MemoryManager::_UnmapChunkEarly(addr_t address, size_t size) -{ - VMAddressSpace* addressSpace = VMAddressSpace::Kernel(); - VMTranslationMap* translationMap = addressSpace->TranslationMap(); - - translationMap->Lock(); - - for (size_t offset = 0; offset < B_PAGE_SIZE; offset += B_PAGE_SIZE) { - addr_t physicalAddress; - uint32 flags; - if (translationMap->Query(address + offset, &physicalAddress, &flags) - == B_OK - && (flags & PAGE_PRESENT) != 0) { - vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE); - DEBUG_PAGE_ACCESS_START(page); - vm_page_set_state(page, PAGE_STATE_FREE); - } - } - - translationMap->Unmap(address, address + size - 1); - - translationMap->Unlock(); -} - - /*static*/ void MemoryManager::_UnmapFreeChunksEarly(Area* area) { @@ -820,6 +1477,48 @@ MemoryManager::_RequestMaintenance() } +/*static*/ int +MemoryManager::_DumpRawAllocations(int argc, char** argv) +{ + kprintf("area meta chunk chunk base size (KB)\n"); + + size_t totalSize = 0; + + for (AreaTable::Iterator it = sAreaTable.GetIterator(); + Area* area = it.Next();) { + for (int32 i = 0; i < SLAB_META_CHUNKS_PER_AREA; i++) { + MetaChunk* metaChunk = area->metaChunks + i; + if (metaChunk->chunkSize == 0) + continue; + for (uint32 k = 0; k < metaChunk->chunkCount; k++) { + Chunk* chunk = metaChunk->chunks + k; + + // skip free chunks + if (_IsChunkFree(metaChunk, chunk)) + continue; + + addr_t reference = chunk->reference; + if ((reference & 1) == 0 || reference == 1) + continue; + + addr_t chunkAddress = _ChunkAddress(metaChunk, chunk); + size_t size = reference - chunkAddress + 1; + totalSize += size; + + kprintf("%p %10" B_PRId32 " %5" B_PRIu32 " %p %9" + B_PRIuSIZE "\n", area, i, k, (void*)chunkAddress, + size / 1024); + } + } + } + + kprintf("total: %9" B_PRIuSIZE "\n", + totalSize / 1024); + + return 0; +} + + /*static*/ void MemoryManager::_PrintMetaChunkTableHeader(bool printChunks) { @@ -855,8 +1554,9 @@ MemoryManager::_DumpMetaChunk(MetaChunk* metaChunk, bool printChunks, kprintf("%5d %p --- %6s meta chunk", metaChunkIndex, (void*)metaChunk->chunkBase, type); if (metaChunk->chunkSize != 0) { - kprintf(": %4u/%4u used ----------------------------\n", - metaChunk->usedChunkCount, metaChunk->chunkCount); + kprintf(": %4u/%4u used, %-4u-%4u free ------------\n", + metaChunk->usedChunkCount, metaChunk->chunkCount, + metaChunk->firstFreeChunk, metaChunk->lastFreeChunk); } else kprintf(" --------------------------------------------\n"); @@ -867,18 +1567,20 @@ MemoryManager::_DumpMetaChunk(MetaChunk* metaChunk, bool printChunks, Chunk* chunk = metaChunk->chunks + i; // skip free chunks - if (chunk->next == NULL) + if (_IsChunkFree(metaChunk, chunk)) continue; - if (chunk->next >= metaChunk->chunks - && chunk->next < metaChunk->chunks + metaChunk->chunkCount) { - continue; - } - ObjectCache* cache = chunk->cache; - kprintf("%5" B_PRIu32 " %p %p %11" B_PRIuSIZE " %s\n", i, - (void*)_ChunkAddress(metaChunk, chunk), cache, - cache != NULL ? cache->object_size : 0, - cache != NULL ? cache->name : ""); + addr_t reference = chunk->reference; + if ((reference & 1) == 0) { + ObjectCache* cache = (ObjectCache*)reference; + kprintf("%5" B_PRIu32 " %p %p %11" B_PRIuSIZE " %s\n", i, + (void*)_ChunkAddress(metaChunk, chunk), cache, + cache != NULL ? cache->object_size : 0, + cache != NULL ? cache->name : ""); + } else if (reference != 1) { + kprintf("%5" B_PRIu32 " %p raw allocation up to %p\n", i, + (void*)_ChunkAddress(metaChunk, chunk), (void*)reference); + } } } @@ -986,14 +1688,22 @@ MemoryManager::_DumpAreas(int argc, char** argv) { kprintf(" base area meta small medium large\n"); + size_t totalTotalSmall = 0; + size_t totalUsedSmall = 0; + size_t totalTotalMedium = 0; + size_t totalUsedMedium = 0; + size_t totalUsedLarge = 0; + uint32 areaCount = 0; + for (AreaTable::Iterator it = sAreaTable.GetIterator(); Area* area = it.Next();) { + areaCount++; + // sum up the free/used counts for the chunk sizes int totalSmall = 0; int usedSmall = 0; int totalMedium = 0; int usedMedium = 0; - int totalLarge = 0; int usedLarge = 0; for (int32 i = 0; i < SLAB_META_CHUNKS_PER_AREA; i++) { @@ -1011,21 +1721,43 @@ MemoryManager::_DumpAreas(int argc, char** argv) usedMedium += metaChunk->usedChunkCount; break; case SLAB_CHUNK_SIZE_LARGE: - totalLarge += metaChunk->chunkCount; usedLarge += metaChunk->usedChunkCount; break; } } - kprintf("%p %p %2u/%2u %4d/%4d %3d/%3d %2d/%2d\n", + kprintf("%p %p %2u/%2u %4d/%4d %3d/%3d %5d\n", area, area->vmArea, area->usedMetaChunkCount, SLAB_META_CHUNKS_PER_AREA, usedSmall, totalSmall, usedMedium, - totalMedium, usedLarge, totalLarge); + totalMedium, usedLarge); + + totalTotalSmall += totalSmall; + totalUsedSmall += usedSmall; + totalTotalMedium += totalMedium; + totalUsedMedium += usedMedium; + totalUsedLarge += usedLarge; } - kprintf("%d free areas:\n", sFreeAreaCount); - for (Area* area = sFreeAreas; area != NULL; area = area->next) + kprintf("%d free area%s:\n", sFreeAreaCount, + sFreeAreaCount == 1 ? "" : "s"); + for (Area* area = sFreeAreas; area != NULL; area = area->next) { + areaCount++; kprintf("%p %p\n", area, area->vmArea); + } + + kprintf("total usage:\n"); + kprintf(" small: %" B_PRIuSIZE "/%" B_PRIuSIZE "\n", totalUsedSmall, + totalTotalSmall); + kprintf(" medium: %" B_PRIuSIZE "/%" B_PRIuSIZE "\n", totalUsedMedium, + totalTotalMedium); + kprintf(" large: %" B_PRIuSIZE "\n", totalUsedLarge); + kprintf(" memory: %" B_PRIuSIZE "/%" B_PRIuSIZE " KB\n", + (totalUsedSmall * SLAB_CHUNK_SIZE_SMALL + + totalUsedMedium * SLAB_CHUNK_SIZE_MEDIUM + + totalUsedLarge * SLAB_CHUNK_SIZE_LARGE) / 1024, + areaCount * SLAB_AREA_SIZE / 1024); + kprintf(" overhead: %" B_PRIuSIZE " KB\n", + areaCount * kAreaAdminSize / 1024); return 0; } diff --git a/src/system/kernel/slab/MemoryManager.h b/src/system/kernel/slab/MemoryManager.h index d389b6e2e3..c84e74738b 100644 --- a/src/system/kernel/slab/MemoryManager.h +++ b/src/system/kernel/slab/MemoryManager.h @@ -41,19 +41,28 @@ public: void*& _pages); static void Free(void* pages, uint32 flags); + static status_t AllocateRaw(size_t size, uint32 flags, + void*& _pages); + static ObjectCache* FreeRawOrReturnCache(void* pages, + uint32 flags); + static size_t AcceptableChunkSize(size_t size); + static ObjectCache* GetAllocationInfo(void* address, + size_t& _size); static ObjectCache* CacheForAddress(void* address); static bool MaintenanceNeeded(); static void PerformMaintenance(); private: + struct Tracing; + struct Area; struct Chunk { union { Chunk* next; - ObjectCache* cache; + addr_t reference; }; }; @@ -63,6 +72,8 @@ private: size_t totalSize; uint16 chunkCount; uint16 usedChunkCount; + uint16 firstFreeChunk; // *some* free range + uint16 lastFreeChunk; // inclusive Chunk chunks[SLAB_SMALL_CHUNKS_PER_META_CHUNK]; Chunk* freeChunks; @@ -115,7 +126,11 @@ private: }; private: - static status_t _AllocateChunk(size_t chunkSize, uint32 flags, + static status_t _AllocateChunks(size_t chunkSize, + uint32 chunkCount, uint32 flags, + MetaChunk*& _metaChunk, Chunk*& _chunk); + static bool _GetChunks(MetaChunkList* metaChunkList, + size_t chunkSize, uint32 chunkCount, MetaChunk*& _metaChunk, Chunk*& _chunk); static bool _GetChunk(MetaChunkList* metaChunkList, size_t chunkSize, MetaChunk*& _metaChunk, @@ -135,10 +150,9 @@ private: static status_t _MapChunk(VMArea* vmArea, addr_t address, size_t size, size_t reserveAdditionalMemory, uint32 flags); - static status_t _UnmapChunk(VMArea* vmArea,addr_t address, + static status_t _UnmapChunk(VMArea* vmArea, addr_t address, size_t size, uint32 flags); - static void _UnmapChunkEarly(addr_t address, size_t size); static void _UnmapFreeChunksEarly(Area* area); static void _ConvertEarlyArea(Area* area); @@ -148,7 +162,10 @@ private: const MetaChunk* metaChunk, addr_t address); static addr_t _ChunkAddress(const MetaChunk* metaChunk, const Chunk* chunk); + static bool _IsChunkFree(const MetaChunk* metaChunk, + const Chunk* chunk); + static int _DumpRawAllocations(int argc, char** argv); static void _PrintMetaChunkTableHeader(bool printChunks); static void _DumpMetaChunk(MetaChunk* metaChunk, bool printChunks, bool printHeader); @@ -202,6 +219,15 @@ MemoryManager::_ChunkAddress(const MetaChunk* metaChunk, const Chunk* chunk) } +/*static*/ inline bool +MemoryManager::_IsChunkFree(const MetaChunk* metaChunk, const Chunk* chunk) +{ + return chunk->next == NULL + || (chunk->next >= metaChunk->chunks + && chunk->next < metaChunk->chunks + metaChunk->chunkCount); +} + + inline MemoryManager::Area* MemoryManager::MetaChunk::GetArea() const { diff --git a/src/system/kernel/slab/ObjectCache.cpp b/src/system/kernel/slab/ObjectCache.cpp index d4acf09357..668f30fdcd 100644 --- a/src/system/kernel/slab/ObjectCache.cpp +++ b/src/system/kernel/slab/ObjectCache.cpp @@ -112,15 +112,18 @@ ObjectCache::InitSlab(slab* slab, void* pages, size_t byteCount, uint32 flags) slab->pages = pages; slab->count = slab->size = byteCount / object_size; slab->free = NULL; - total_objects += slab->size; size_t spareBytes = byteCount - (slab->size * object_size); - slab->offset = cache_color_cycle; - if (slab->offset > spareBytes) - cache_color_cycle = slab->offset = 0; - else - cache_color_cycle += kCacheColorPeriod; + if ((this->flags & CACHE_ALIGN_ON_SIZE) != 0) { + slab->offset = cache_color_cycle; + + if (slab->offset > spareBytes) + cache_color_cycle = slab->offset = 0; + else + cache_color_cycle += kCacheColorPeriod; + } else + slab->offset = 0; TRACE_CACHE(this, " %lu objects, %lu spare bytes, offset %lu", slab->size, spareBytes, slab->offset); @@ -163,6 +166,9 @@ ObjectCache::InitSlab(slab* slab, void* pages, size_t byteCount, uint32 flags) data += object_size; } + usage += slab_size; + total_objects += slab->size; + return slab; } @@ -175,6 +181,7 @@ ObjectCache::UninitSlab(slab* slab) if (slab->count != slab->size) panic("cache: destroying a slab which isn't empty."); + usage -= slab_size; total_objects -= slab->size; DELETE_PARANOIA_CHECK_SET(slab); diff --git a/src/system/kernel/slab/ObjectCache.h b/src/system/kernel/slab/ObjectCache.h index 4cde8f8af3..9baa64c192 100644 --- a/src/system/kernel/slab/ObjectCache.h +++ b/src/system/kernel/slab/ObjectCache.h @@ -34,6 +34,7 @@ typedef DoublyLinkedList SlabList; struct ObjectCacheResizeEntry { ConditionVariable condition; + thread_id thread; }; struct ObjectCache : DoublyLinkedListLinkImpl { diff --git a/src/system/kernel/slab/Slab.cpp b/src/system/kernel/slab/Slab.cpp index b70148165e..4bc67cee83 100644 --- a/src/system/kernel/slab/Slab.cpp +++ b/src/system/kernel/slab/Slab.cpp @@ -50,10 +50,10 @@ static MaintenanceQueue sMaintenanceQueue; static ConditionVariable sMaintenanceCondition; -#if OBJECT_CACHE_TRACING +#if SLAB_OBJECT_CACHE_TRACING -namespace ObjectCacheTracing { +namespace SlabObjectCacheTracing { class ObjectCacheTraceEntry : public AbstractTraceEntry { public: @@ -186,13 +186,13 @@ class Reserve : public ObjectCacheTraceEntry { }; -} // namespace ObjectCacheTracing +} // namespace SlabObjectCacheTracing -# define T(x) new(std::nothrow) ObjectCacheTracing::x +# define T(x) new(std::nothrow) SlabObjectCacheTracing::x #else # define T(x) -#endif // OBJECT_CACHE_TRACING +#endif // SLAB_OBJECT_CACHE_TRACING // #pragma mark - @@ -211,8 +211,7 @@ dump_slabs(int argc, char* argv[]) kprintf("%p %22s %8lu %8lu %6lu %8lu %8lu %8lx\n", cache, cache->name, cache->object_size, cache->usage, cache->empty_count, - cache->used_count, cache->usage / cache->object_size, - cache->flags); + cache->used_count, cache->total_objects, cache->flags); } return 0; @@ -241,6 +240,8 @@ dump_cache_info(int argc, char* argv[]) kprintf("maximum: %lu\n", cache->maximum); kprintf("flags: 0x%lx\n", cache->flags); kprintf("cookie: %p\n", cache->cookie); + kprintf("resize entry don't wait: %p\n", cache->resize_entry_dont_wait); + kprintf("resize entry can wait: %p\n", cache->resize_entry_can_wait); return 0; } @@ -249,23 +250,6 @@ dump_cache_info(int argc, char* argv[]) // #pragma mark - -void* -slab_internal_alloc(size_t size, uint32 flags) -{ - if (flags & CACHE_DURING_BOOT) - return block_alloc_early(size); - - return block_alloc(size, flags); -} - - -void -slab_internal_free(void* buffer, uint32 flags) -{ - block_free(buffer, flags); -} - - void request_memory_manager_maintenance() { @@ -322,6 +306,7 @@ object_cache_reserve_internal(ObjectCache* cache, size_t objectCount, { // If someone else is already adding slabs, we wait for that to be finished // first. + thread_id thread = find_thread(NULL); while (true) { if (objectCount <= cache->total_objects - cache->used_count) return B_OK; @@ -329,9 +314,19 @@ object_cache_reserve_internal(ObjectCache* cache, size_t objectCount, ObjectCacheResizeEntry* resizeEntry = NULL; if (cache->resize_entry_dont_wait != NULL) { resizeEntry = cache->resize_entry_dont_wait; - } else if (cache->resize_entry_can_wait != NULL - && (flags & CACHE_DONT_WAIT_FOR_MEMORY) == 0) { + if (thread == resizeEntry->thread) + return B_WOULD_BLOCK; + // Note: We could still have reentered the function, i.e. + // resize_entry_can_wait would be ours. That doesn't matter much, + // though, since after the don't-wait thread has done its job + // everyone will be happy. + } else if (cache->resize_entry_can_wait != NULL) { resizeEntry = cache->resize_entry_can_wait; + if (thread == resizeEntry->thread) + return B_WOULD_BLOCK; + + if ((flags & CACHE_DONT_WAIT_FOR_MEMORY) != 0) + break; } else break; @@ -351,6 +346,7 @@ object_cache_reserve_internal(ObjectCache* cache, size_t objectCount, ObjectCacheResizeEntry myResizeEntry; resizeEntry = &myResizeEntry; resizeEntry->condition.Init(cache, "wait for slabs"); + resizeEntry->thread = thread; // add new slabs until there are as many free ones as requested while (objectCount > cache->total_objects - cache->used_count) { @@ -431,29 +427,13 @@ object_cache_low_memory(void* dummy, uint32 resources, int32 level) break; } - // If the object cache has minimum object reserve, make sure that we - // don't free too many slabs. - if (cache->min_object_reserve > 0 && cache->empty_count > 0) { + while (cache->empty_count > minimumAllowed) { + // make sure we respect the cache's minimum object reserve size_t objectsPerSlab = cache->empty.Head()->size; size_t freeObjects = cache->total_objects - cache->used_count; + if (freeObjects < cache->min_object_reserve + objectsPerSlab) + break; - if (cache->min_object_reserve + objectsPerSlab >= freeObjects) - return; - - size_t slabsToFree = (freeObjects - cache->min_object_reserve) - / objectsPerSlab; - - if (cache->empty_count > minimumAllowed + slabsToFree) - minimumAllowed = cache->empty_count - slabsToFree; - } - - if (cache->empty_count <= minimumAllowed) - return; - - TRACE_CACHE(cache, "cache: memory pressure, will release down to %lu.", - minimumAllowed); - - while (cache->empty_count > minimumAllowed) { cache->ReturnSlab(cache->empty.RemoveHead(), 0); cache->empty_count--; } @@ -649,23 +629,26 @@ object_cache_alloc(object_cache* cache, uint32 flags) } MutexLocker _(cache->lock); - slab* source; + slab* source = NULL; - if (cache->partial.IsEmpty()) { - if (cache->empty.IsEmpty()) { - if (object_cache_reserve_internal(cache, 1, flags) < B_OK) { - T(Alloc(cache, flags, NULL)); - return NULL; - } - - cache->pressure++; - } + while (true) { + source = cache->partial.Head(); + if (source != NULL) + break; source = cache->empty.RemoveHead(); - cache->empty_count--; - cache->partial.Add(source); - } else { - source = cache->partial.Head(); + if (source != NULL) { + cache->empty_count--; + cache->partial.Add(source); + break; + } + + if (object_cache_reserve_internal(cache, 1, flags) != B_OK) { + T(Alloc(cache, flags, NULL)); + return NULL; + } + + cache->pressure++; } ParanoiaChecker _2(source); @@ -734,19 +717,17 @@ object_cache_get_usage(object_cache* cache, size_t* _allocatedMemory) void -slab_init(kernel_args* args, addr_t initialBase, size_t initialSize) +slab_init(kernel_args* args) { - dprintf("slab: init base %p + 0x%lx\n", (void*)initialBase, initialSize); - MemoryManager::Init(args); new (&sObjectCaches) ObjectCacheList(); - block_allocator_init_boot(initialBase, initialSize); + block_allocator_init_boot(); add_debugger_command("slabs", dump_slabs, "list all object caches"); - add_debugger_command("cache_info", dump_cache_info, - "dump information about a specific cache"); + add_debugger_command("slab_cache", dump_cache_info, + "dump information about a specific object cache"); } diff --git a/src/system/kernel/slab/SmallObjectCache.cpp b/src/system/kernel/slab/SmallObjectCache.cpp index 76fb73b52c..c69112eb5b 100644 --- a/src/system/kernel/slab/SmallObjectCache.cpp +++ b/src/system/kernel/slab/SmallObjectCache.cpp @@ -24,9 +24,8 @@ SmallObjectCache::Create(const char* name, size_t object_size, SmallObjectCache* cache = new(buffer) SmallObjectCache(); - if (cache->Init(name, object_size, alignment, maximum, - flags | CACHE_ALIGN_ON_SIZE, cookie, constructor, destructor, - reclaimer) != B_OK) { + if (cache->Init(name, object_size, alignment, maximum, flags, cookie, + constructor, destructor, reclaimer) != B_OK) { cache->Delete(); return NULL; } diff --git a/src/system/kernel/slab/allocator.cpp b/src/system/kernel/slab/allocator.cpp index 2dec030c53..1abd82700e 100644 --- a/src/system/kernel/slab/allocator.cpp +++ b/src/system/kernel/slab/allocator.cpp @@ -13,8 +13,12 @@ #include #include +#include + +#include +#include #include // for ROUNDUP -#include +#include #include #include @@ -38,9 +42,9 @@ static const size_t kNumBlockSizes = sizeof(kBlockSizes) / sizeof(size_t) - 1; static object_cache* sBlockCaches[kNumBlockSizes]; -static addr_t sBootStrapMemory; -static size_t sBootStrapMemorySize; -static size_t sUsedBootStrapMemory; +static addr_t sBootStrapMemory = 0; +static size_t sBootStrapMemorySize = 0; +static size_t sUsedBootStrapMemory = 0; static int @@ -68,24 +72,33 @@ size_to_index(size_t size) void* -block_alloc(size_t size, uint32 flags) +block_alloc(size_t size, size_t alignment, uint32 flags) { + if (alignment > 8) { + // Make size >= alignment and a power of two. This is sufficient, since + // all of our object caches with power of two sizes are aligned. We may + // waste quite a bit of memory, but memalign() is very rarely used + // in the kernel and always with power of two size == alignment anyway. + ASSERT((alignment & (alignment - 1)) == 0); + while (alignment < size) + alignment <<= 1; + size = alignment; + + // If we're not using an object cache, make sure that the memory + // manager knows it has to align the allocation. + if (size > kBlockSizes[kNumBlockSizes]) + flags |= CACHE_ALIGN_ON_SIZE; + } + // allocate from the respective object cache, if any int index = size_to_index(size); if (index >= 0) return object_cache_alloc(sBlockCaches[index], flags); - // the allocation is too large for our object caches -- create an area - if ((flags & CACHE_DONT_LOCK_KERNEL_SPACE) != 0) - return NULL; - + // the allocation is too large for our object caches -- ask the memory + // manager void* block; - area_id area = create_area_etc(VMAddressSpace::KernelID(), - "alloc'ed block", &block, B_ANY_KERNEL_ADDRESS, - ROUNDUP(size, B_PAGE_SIZE), B_FULL_LOCK, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0, - (flags & CACHE_DONT_WAIT_FOR_MEMORY) != 0 ? CREATE_AREA_DONT_WAIT : 0); - if (area < 0) + if (MemoryManager::AllocateRaw(size, flags, block) != B_OK) return NULL; return block; @@ -96,14 +109,31 @@ void* block_alloc_early(size_t size) { int index = size_to_index(size); - if (index < 0) - return NULL; - - if (sBlockCaches[index] != NULL) + if (index >= 0 && sBlockCaches[index] != NULL) return object_cache_alloc(sBlockCaches[index], CACHE_DURING_BOOT); - // No object cache yet. Use the bootstrap memory. This allocation must - // never be freed! + if (size > SLAB_CHUNK_SIZE_SMALL) { + // This is a sufficiently large allocation -- just ask the memory + // manager directly. + void* block; + if (MemoryManager::AllocateRaw(size, 0, block) != B_OK) + return NULL; + + return block; + } + + // A small allocation, but no object cache yet. Use the bootstrap memory. + // This allocation must never be freed! + if (sBootStrapMemorySize - sUsedBootStrapMemory < size) { + // We need more memory. + void* block; + if (MemoryManager::AllocateRaw(SLAB_CHUNK_SIZE_SMALL, 0, block) != B_OK) + return NULL; + sBootStrapMemory = (addr_t)block; + sBootStrapMemorySize = SLAB_CHUNK_SIZE_SMALL; + sUsedBootStrapMemory = 0; + } + size_t neededSize = ROUNDUP(size, sizeof(double)); if (sUsedBootStrapMemory + neededSize > sBootStrapMemorySize) return NULL; @@ -117,40 +147,41 @@ block_alloc_early(size_t size) void block_free(void* block, uint32 flags) { - if (ObjectCache* cache = MemoryManager::CacheForAddress(block)) { + if (block == NULL) + return; + + ObjectCache* cache = MemoryManager::FreeRawOrReturnCache(block, flags); + if (cache != NULL) { // a regular small allocation ASSERT(cache->object_size >= kBlockSizes[0]); ASSERT(cache->object_size <= kBlockSizes[kNumBlockSizes - 1]); ASSERT(cache == sBlockCaches[size_to_index(cache->object_size)]); object_cache_free(cache, block, flags); - } else { - // a large allocation -- look up the area - VMAddressSpace* addressSpace = VMAddressSpace::Kernel(); - addressSpace->ReadLock(); - VMArea* area = addressSpace->LookupArea((addr_t)block); - addressSpace->ReadUnlock(); - - if (area != NULL && (addr_t)block == area->Base()) - delete_area(area->id); - else - panic("freeing unknown block %p from area %p", block, area); } } void -block_allocator_init_boot(addr_t bootStrapBase, size_t bootStrapSize) +block_allocator_init_boot() { - sBootStrapMemory = bootStrapBase; - sBootStrapMemorySize = bootStrapSize; - sUsedBootStrapMemory = 0; - for (int index = 0; kBlockSizes[index] != 0; index++) { char name[32]; snprintf(name, sizeof(name), "block cache: %lu", kBlockSizes[index]); - sBlockCaches[index] = create_object_cache_etc(name, kBlockSizes[index], - 0, 0, CACHE_DURING_BOOT, NULL, NULL, NULL, NULL); + uint32 flags = CACHE_DURING_BOOT; + size_t size = kBlockSizes[index]; + + // align the power of two objects to their size + if ((size & (size - 1)) == 0) + flags |= CACHE_ALIGN_ON_SIZE; + + // For the larger allocation sizes disable the object depot, so we don't + // keep lot's of unused objects around. + if (size > 2048) + flags |= CACHE_NO_DEPOT; + + sBlockCaches[index] = create_object_cache_etc(name, size, 0, 0, flags, + NULL, NULL, NULL, NULL); if (sBlockCaches[index] == NULL) panic("allocator: failed to init block cache"); } @@ -162,7 +193,87 @@ block_allocator_init_rest() { #ifdef TEST_ALL_CACHES_DURING_BOOT for (int index = 0; kBlockSizes[index] != 0; index++) { - block_free(block_alloc(kBlockSizes[index] - sizeof(boundary_tag))); + block_free(block_alloc(kBlockSizes[index] - sizeof(boundary_tag)), 0, + 0); } #endif } + + +// #pragma mark - public API + + +#if USE_SLAB_ALLOCATOR_FOR_MALLOC + + +void* +memalign(size_t alignment, size_t size) +{ + return block_alloc(size, alignment, 0); +} + + +void* +memalign_nogrow(size_t alignment, size_t size) +{ + return block_alloc(size, alignment, + CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); +} + + +void* +malloc_nogrow(size_t size) +{ + return block_alloc(size, 0, + CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); +} + + +void* +malloc(size_t size) +{ + return block_alloc(size, 0, 0); +} + + +void +free(void* address) +{ + block_free(address, 0); +} + + +void* +realloc(void* address, size_t newSize) +{ + if (newSize == 0) { + block_free(address, 0); + return NULL; + } + + if (address == NULL) + return block_alloc(newSize, 0, 0); + + size_t oldSize; + ObjectCache* cache = MemoryManager::GetAllocationInfo(address, oldSize); + if (cache == NULL && oldSize == 0) { + panic("block_realloc(): allocation %p not known", address); + return NULL; + } + + if (oldSize == newSize) + return address; + + void* newBlock = block_alloc(newSize, 0, 0); + if (newBlock == NULL) + return NULL; + + memcpy(newBlock, address, std::min(oldSize, newSize)); + + block_free(address, 0); + + return newBlock; +} + + +#endif // USE_SLAB_ALLOCATOR_FOR_MALLOC diff --git a/src/system/kernel/slab/slab_private.h b/src/system/kernel/slab/slab_private.h index a4cb7b94fd..0282c9e8e6 100644 --- a/src/system/kernel/slab/slab_private.h +++ b/src/system/kernel/slab/slab_private.h @@ -11,6 +11,8 @@ #include +#include + //#define TRACE_SLAB #ifdef TRACE_SLAB @@ -26,16 +28,12 @@ struct ObjectCache; -void* slab_internal_alloc(size_t size, uint32 flags); -void slab_internal_free(void *_buffer, uint32 flags); - void request_memory_manager_maintenance(); -void* block_alloc(size_t size, uint32 flags); +void* block_alloc(size_t size, size_t alignment, uint32 flags); void* block_alloc_early(size_t size); -void block_free(void *block, uint32 flags); -void block_allocator_init_boot(addr_t bootStrapBase, - size_t bootStrapSize); +void block_free(void* block, uint32 flags); +void block_allocator_init_boot(); void block_allocator_init_rest(); @@ -58,4 +56,21 @@ _push(Type*& head, Type* object) } +static inline void* +slab_internal_alloc(size_t size, uint32 flags) +{ + if (flags & CACHE_DURING_BOOT) + return block_alloc_early(size); + + return block_alloc(size, 0, flags); +} + + +static inline void +slab_internal_free(void* buffer, uint32 flags) +{ + block_free(buffer, flags); +} + + #endif // SLAB_PRIVATE_H diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index c25004165b..e74d9ca6f1 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -3223,17 +3223,14 @@ vm_init(kernel_args* args) if (heapSize < 1024 * 1024) panic("vm_init: go buy some RAM please."); + slab_init(args); + // map in the new heap and initialize it addr_t heapBase = vm_allocate_early(args, heapSize, heapSize, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, false); TRACE(("heap at 0x%lx\n", heapBase)); heap_init(heapBase, heapSize); - size_t slabInitialSize = B_PAGE_SIZE; - addr_t slabInitialBase = vm_allocate_early(args, slabInitialSize, - slabInitialSize, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, false); - slab_init(args, slabInitialBase, slabInitialSize); - // initialize the free page list and physical page mapper vm_page_init(args); @@ -3262,11 +3259,6 @@ vm_init(kernel_args* args) create_area("kernel heap", &address, B_EXACT_ADDRESS, heapSize, B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - address = (void*)ROUNDDOWN(slabInitialBase, B_PAGE_SIZE); - create_area("initial slab space", &address, B_EXACT_ADDRESS, - slabInitialSize, B_ALREADY_WIRED, B_KERNEL_READ_AREA - | B_KERNEL_WRITE_AREA); - allocate_kernel_args(args); create_preloaded_image_areas(&args->kernel_image);