diff --git a/build/config_headers/kernel_debug_config.h b/build/config_headers/kernel_debug_config.h index bfed08c20a..08aec0c48d 100644 --- a/build/config_headers/kernel_debug_config.h +++ b/build/config_headers/kernel_debug_config.h @@ -94,6 +94,9 @@ // Enables swap support. #define ENABLE_SWAP_SUPPORT 1 +// Use the slab allocator as generic memory allocator (malloc()/free()). +#define USE_SLAB_ALLOCATOR_FOR_MALLOC 0 + // When set limits the amount of available RAM (in MB). //#define LIMIT_AVAILABLE_MEMORY 256 diff --git a/build/config_headers/tracing_config.h b/build/config_headers/tracing_config.h index 94594a79c2..e85427fa86 100644 --- a/build/config_headers/tracing_config.h +++ b/build/config_headers/tracing_config.h @@ -30,7 +30,6 @@ #define KTRACE_PRINTF_STACK_TRACE 0 /* stack trace depth */ #define NET_BUFFER_TRACING 0 #define NET_BUFFER_TRACING_STACK_TRACE 0 /* stack trace depth */ -#define OBJECT_CACHE_TRACING 0 #define PAGE_ALLOCATION_TRACING 0 #define PAGE_DAEMON_TRACING 0 #define PAGE_WRITER_TRACING 0 @@ -41,6 +40,8 @@ #define SCHEDULER_TRACING 0 #define SCHEDULING_ANALYSIS_TRACING 0 #define SIGNAL_TRACING 0 +#define SLAB_MEMORY_MANAGER_TRACING 0 +#define SLAB_OBJECT_CACHE_TRACING 0 #define SWAP_TRACING 0 #define SYSCALL_TRACING 0 #define SYSCALL_TRACING_IGNORE_KTRACE_OUTPUT 1 diff --git a/headers/private/kernel/vm/vm.h b/headers/private/kernel/vm/vm.h index ca1137ebc5..106bedfa55 100644 --- a/headers/private/kernel/vm/vm.h +++ b/headers/private/kernel/vm/vm.h @@ -49,8 +49,7 @@ void vm_free_unused_boot_loader_range(addr_t start, addr_t end); addr_t vm_allocate_early(struct kernel_args *args, size_t virtualSize, size_t physicalSize, uint32 attributes, bool blockAlign); -void slab_init(struct kernel_args *args, addr_t initialBase, - size_t initialSize); +void slab_init(struct kernel_args *args); void slab_init_post_area(); void slab_init_post_sem(); void slab_init_post_thread(); diff --git a/src/system/kernel/heap.cpp b/src/system/kernel/heap.cpp index 47356b9de8..42092714ea 100644 --- a/src/system/kernel/heap.cpp +++ b/src/system/kernel/heap.cpp @@ -1687,6 +1687,9 @@ heap_set_get_caller(heap_allocator* heap, addr_t (*getCaller)()) #endif +#if !USE_SLAB_ALLOCATOR_FOR_MALLOC + + static status_t heap_realloc(heap_allocator *heap, void *address, void **newAddress, size_t newSize) @@ -1794,6 +1797,9 @@ heap_realloc(heap_allocator *heap, void *address, void **newAddress, } +#endif // !USE_SLAB_ALLOCATOR_FOR_MALLOC + + inline uint32 heap_index_for(size_t size, int32 cpu) { @@ -2045,6 +2051,9 @@ heap_init_post_thread() // #pragma mark - Public API +#if !USE_SLAB_ALLOCATOR_FOR_MALLOC + + void * memalign(size_t alignment, size_t size) { @@ -2312,6 +2321,9 @@ realloc(void *address, size_t newSize) } +#endif // !USE_SLAB_ALLOCATOR_FOR_MALLOC + + void * calloc(size_t numElements, size_t size) { diff --git a/src/system/kernel/slab/HashedObjectCache.cpp b/src/system/kernel/slab/HashedObjectCache.cpp index 3d64b973ef..eb50df4f3d 100644 --- a/src/system/kernel/slab/HashedObjectCache.cpp +++ b/src/system/kernel/slab/HashedObjectCache.cpp @@ -61,6 +61,16 @@ HashedObjectCache::Create(const char* name, size_t object_size, HashedObjectCache* cache = new(buffer) HashedObjectCache(); + // init the hash table + size_t hashSize = cache->hash_table.ResizeNeeded(); + buffer = slab_internal_alloc(hashSize, flags); + if (buffer == NULL) { + cache->Delete(); + return NULL; + } + + cache->hash_table.Resize(buffer, hashSize, true); + if (cache->Init(name, object_size, alignment, maximum, flags, cookie, constructor, destructor, reclaimer) != B_OK) { cache->Delete(); @@ -96,21 +106,20 @@ HashedObjectCache::CreateSlab(uint32 flags) Unlock(); slab* slab = allocate_slab(flags); + if (slab != NULL) { + void* pages; + if (MemoryManager::Allocate(this, flags, pages) == B_OK) { + Lock(); + if (InitSlab(slab, pages, slab_size, flags)) + return slab; + Unlock(); + MemoryManager::Free(pages, flags); + } - Lock(); - - if (slab == NULL) - return NULL; - - void* pages; - if (MemoryManager::Allocate(this, flags, pages) == B_OK) { - if (InitSlab(slab, pages, slab_size, flags)) - return slab; - - MemoryManager::Free(pages, flags); + free_slab(slab, flags); } - free_slab(slab, flags); + Lock(); return NULL; } @@ -119,8 +128,10 @@ void HashedObjectCache::ReturnSlab(slab* slab, uint32 flags) { UninitSlab(slab); + Unlock(); MemoryManager::Free(slab->pages, flags); free_slab(slab, flags); + Lock(); } @@ -147,11 +158,9 @@ HashedObjectCache::PrepareObject(slab* source, void* object, uint32 flags) link->buffer = object; link->parent = source; - hash_table.Insert(link); - // TODO: This might resize the table! Currently it uses the heap, so - // we won't possibly reenter and deadlock on our own cache. We do ignore - // the flags, though! - // TODO: We don't pre-init the table, so Insert() can fail! + hash_table.InsertUnchecked(link); + _ResizeHashTableIfNeeded(flags); + return B_OK; } @@ -170,11 +179,37 @@ HashedObjectCache::UnprepareObject(slab* source, void* object, uint32 flags) return; } - hash_table.Remove(link); + hash_table.RemoveUnchecked(link); + _ResizeHashTableIfNeeded(flags); + _FreeLink(link, flags); } +void +HashedObjectCache::_ResizeHashTableIfNeeded(uint32 flags) +{ + size_t hashSize = hash_table.ResizeNeeded(); + if (hashSize != 0) { + Unlock(); + void* buffer = slab_internal_alloc(hashSize, flags); + Lock(); + + if (buffer != NULL) { + if (hash_table.ResizeNeeded() == hashSize) { + void* oldHash; + hash_table.Resize(buffer, hashSize, true, &oldHash); + if (oldHash != NULL) { + Unlock(); + slab_internal_free(oldHash, flags); + Lock(); + } + } + } + } +} + + /*static*/ inline HashedObjectCache::Link* HashedObjectCache::_AllocateLink(uint32 flags) { diff --git a/src/system/kernel/slab/HashedObjectCache.h b/src/system/kernel/slab/HashedObjectCache.h index 2bad679d7c..cba0630e00 100644 --- a/src/system/kernel/slab/HashedObjectCache.h +++ b/src/system/kernel/slab/HashedObjectCache.h @@ -11,6 +11,7 @@ #include #include "ObjectCache.h" +#include "slab_private.h" struct HashedObjectCache : ObjectCache { @@ -81,11 +82,26 @@ private: HashedObjectCache* parent; }; - typedef BOpenHashTable HashTable; + struct InternalAllocator { + void* Allocate(size_t size) const + { + return slab_internal_alloc(size, 0); + } + + void Free(void* memory) const + { + slab_internal_free(memory, 0); + } + }; + + typedef BOpenHashTable HashTable; friend class Definition; private: + void _ResizeHashTableIfNeeded(uint32 flags); + static Link* _AllocateLink(uint32 flags); static void _FreeLink(HashedObjectCache::Link* link, uint32 flags); diff --git a/src/system/kernel/slab/MemoryManager.cpp b/src/system/kernel/slab/MemoryManager.cpp index a7251ecb60..e9df60f602 100644 --- a/src/system/kernel/slab/MemoryManager.cpp +++ b/src/system/kernel/slab/MemoryManager.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -50,6 +51,369 @@ MemoryManager::AllocationEntry* MemoryManager::sAllocationEntryDontWait; bool MemoryManager::sMaintenanceNeeded; + +// #pragma mark - kernel tracing + + +#if SLAB_MEMORY_MANAGER_TRACING + + +//namespace SlabMemoryManagerCacheTracing { +struct MemoryManager::Tracing { + +class MemoryManagerTraceEntry : public AbstractTraceEntry { +public: + MemoryManagerTraceEntry() + { + } +}; + + +class Allocate : public MemoryManagerTraceEntry { +public: + Allocate(ObjectCache* cache, uint32 flags) + : + MemoryManagerTraceEntry(), + fCache(cache), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager alloc: cache: %p, flags: %#" B_PRIx32, + fCache, fFlags); + } + +private: + ObjectCache* fCache; + uint32 fFlags; +}; + + +class Free : public MemoryManagerTraceEntry { +public: + Free(void* address, uint32 flags) + : + MemoryManagerTraceEntry(), + fAddress(address), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager free: address: %p, flags: %#" B_PRIx32, + fAddress, fFlags); + } + +private: + void* fAddress; + uint32 fFlags; +}; + + +class AllocateRaw : public MemoryManagerTraceEntry { +public: + AllocateRaw(size_t size, uint32 flags) + : + MemoryManagerTraceEntry(), + fSize(size), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager alloc raw: size: %" B_PRIuSIZE + ", flags: %#" B_PRIx32, fSize, fFlags); + } + +private: + size_t fSize; + uint32 fFlags; +}; + + +class FreeRawOrReturnCache : public MemoryManagerTraceEntry { +public: + FreeRawOrReturnCache(void* address, uint32 flags) + : + MemoryManagerTraceEntry(), + fAddress(address), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager free raw/return: address: %p, flags: %#" + B_PRIx32, fAddress, fFlags); + } + +private: + void* fAddress; + uint32 fFlags; +}; + + +class AllocateArea : public MemoryManagerTraceEntry { +public: + AllocateArea(Area* area, uint32 flags) + : + MemoryManagerTraceEntry(), + fArea(area), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager alloc area: flags: %#" B_PRIx32 + " -> %p", fFlags, fArea); + } + +private: + Area* fArea; + uint32 fFlags; +}; + + +class AddArea : public MemoryManagerTraceEntry { +public: + AddArea(Area* area) + : + MemoryManagerTraceEntry(), + fArea(area) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager add area: %p", fArea); + } + +private: + Area* fArea; +}; + + +class FreeArea : public MemoryManagerTraceEntry { +public: + FreeArea(Area* area, bool areaRemoved, uint32 flags) + : + MemoryManagerTraceEntry(), + fArea(area), + fFlags(flags), + fRemoved(areaRemoved) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager free area: %p%s, flags: %#" B_PRIx32, + fArea, fRemoved ? " (removed)" : "", fFlags); + } + +private: + Area* fArea; + uint32 fFlags; + bool fRemoved; +}; + + +class AllocateMetaChunk : public MemoryManagerTraceEntry { +public: + AllocateMetaChunk(MetaChunk* metaChunk) + : + MemoryManagerTraceEntry(), + fMetaChunk(metaChunk->chunkBase) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager alloc meta chunk: %#" B_PRIxADDR, + fMetaChunk); + } + +private: + addr_t fMetaChunk; +}; + + +class FreeMetaChunk : public MemoryManagerTraceEntry { +public: + FreeMetaChunk(MetaChunk* metaChunk) + : + MemoryManagerTraceEntry(), + fMetaChunk(metaChunk->chunkBase) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager free meta chunk: %#" B_PRIxADDR, + fMetaChunk); + } + +private: + addr_t fMetaChunk; +}; + + +class AllocateChunk : public MemoryManagerTraceEntry { +public: + AllocateChunk(size_t chunkSize, MetaChunk* metaChunk, Chunk* chunk) + : + MemoryManagerTraceEntry(), + fChunkSize(chunkSize), + fMetaChunk(metaChunk->chunkBase), + fChunk(chunk - metaChunk->chunks) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager alloc chunk: size: %" B_PRIuSIZE + " -> meta chunk: %#" B_PRIxADDR ", chunk: %" B_PRIu32, fChunkSize, + fMetaChunk, fChunk); + } + +private: + size_t fChunkSize; + addr_t fMetaChunk; + uint32 fChunk; +}; + + +class AllocateChunks : public MemoryManagerTraceEntry { +public: + AllocateChunks(size_t chunkSize, uint32 chunkCount, MetaChunk* metaChunk, + Chunk* chunk) + : + MemoryManagerTraceEntry(), + fMetaChunk(metaChunk->chunkBase), + fChunkSize(chunkSize), + fChunkCount(chunkCount), + fChunk(chunk - metaChunk->chunks) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager alloc chunks: size: %" B_PRIuSIZE + ", count %" B_PRIu32 " -> meta chunk: %#" B_PRIxADDR ", chunk: %" + B_PRIu32, fChunkSize, fChunkCount, fMetaChunk, fChunk); + } + +private: + addr_t fMetaChunk; + size_t fChunkSize; + uint32 fChunkCount; + uint32 fChunk; +}; + + +class FreeChunk : public MemoryManagerTraceEntry { +public: + FreeChunk(MetaChunk* metaChunk, Chunk* chunk) + : + MemoryManagerTraceEntry(), + fMetaChunk(metaChunk->chunkBase), + fChunk(chunk - metaChunk->chunks) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager free chunk: meta chunk: %#" B_PRIxADDR + ", chunk: %" B_PRIu32, fMetaChunk, fChunk); + } + +private: + addr_t fMetaChunk; + uint32 fChunk; +}; + + +class Map : public MemoryManagerTraceEntry { +public: + Map(addr_t address, size_t size, uint32 flags) + : + MemoryManagerTraceEntry(), + fAddress(address), + fSize(size), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager map: %#" B_PRIxADDR ", size: %" + B_PRIuSIZE ", flags: %#" B_PRIx32, fAddress, fSize, fFlags); + } + +private: + addr_t fAddress; + size_t fSize; + uint32 fFlags; +}; + + +class Unmap : public MemoryManagerTraceEntry { +public: + Unmap(addr_t address, size_t size, uint32 flags) + : + MemoryManagerTraceEntry(), + fAddress(address), + fSize(size), + fFlags(flags) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("slab memory manager unmap: %#" B_PRIxADDR ", size: %" + B_PRIuSIZE ", flags: %#" B_PRIx32, fAddress, fSize, fFlags); + } + +private: + addr_t fAddress; + size_t fSize; + uint32 fFlags; +}; + + +//} // namespace SlabMemoryManagerCacheTracing +}; // struct MemoryManager::Tracing + + +//# define T(x) new(std::nothrow) SlabMemoryManagerCacheTracing::x +# define T(x) new(std::nothrow) MemoryManager::Tracing::x + +#else +# define T(x) +#endif // SLAB_MEMORY_MANAGER_TRACING + + +// #pragma mark - MemoryManager + + /*static*/ void MemoryManager::Init(kernel_args* args) { @@ -137,6 +501,10 @@ MemoryManager::InitPostArea() "Lists all non-full slab meta chunks.\n" "If \"-c\" is given, the chunks of all meta chunks area printed as " "well.\n", 0); + add_debugger_command_etc("slab_raw_allocations", &_DumpRawAllocations, + "List all raw allocations in slab areas", + "\n" + "Lists all raw allocations in slab areas.\n", 0); } @@ -145,6 +513,8 @@ MemoryManager::Allocate(ObjectCache* cache, uint32 flags, void*& _pages) { // TODO: Support CACHE_UNLOCKED_PAGES! + T(Allocate(cache, flags)); + size_t chunkSize = cache->slab_size; TRACE("MemoryManager::Allocate(%p, %#" B_PRIx32 "): chunkSize: %" @@ -155,7 +525,7 @@ MemoryManager::Allocate(ObjectCache* cache, uint32 flags, void*& _pages) // allocate a chunk MetaChunk* metaChunk; Chunk* chunk; - status_t error = _AllocateChunk(chunkSize, flags, metaChunk, chunk); + status_t error = _AllocateChunks(chunkSize, 1, flags, metaChunk, chunk); if (error != B_OK) return error; @@ -172,7 +542,7 @@ MemoryManager::Allocate(ObjectCache* cache, uint32 flags, void*& _pages) return error; } - chunk->cache = cache; + chunk->reference = (addr_t)cache; _pages = (void*)chunkAddress; TRACE("MemoryManager::Allocate() done: %p (meta chunk: %d, chunk %d)\n", @@ -187,6 +557,8 @@ MemoryManager::Free(void* pages, uint32 flags) { TRACE("MemoryManager::Free(%p, %#" B_PRIx32 ")\n", pages, flags); + T(Free(pages, flags)); + // get the area and the meta chunk Area* area = (Area*)ROUNDDOWN((addr_t)pages, SLAB_AREA_SIZE); MetaChunk* metaChunk = &area->metaChunks[ @@ -210,6 +582,137 @@ MemoryManager::Free(void* pages, uint32 flags) } +/*static*/ status_t +MemoryManager::AllocateRaw(size_t size, uint32 flags, void*& _pages) +{ + T(AllocateRaw(size, flags)); + + size = ROUNDUP(size, SLAB_CHUNK_SIZE_SMALL); + + TRACE("MemoryManager::AllocateRaw(%" B_PRIuSIZE ", %#" B_PRIx32 ")\n", size, + flags); + + if (size > SLAB_CHUNK_SIZE_LARGE || (flags & CACHE_ALIGN_ON_SIZE) != 0) { + // Requested size greater than a large chunk or an aligned allocation. + // Allocate as an area. + if ((flags & CACHE_DONT_LOCK_KERNEL_SPACE) != 0) + return B_WOULD_BLOCK; + + area_id area = create_area_etc(VMAddressSpace::KernelID(), + "slab large raw allocation", &_pages, + (flags & CACHE_ALIGN_ON_SIZE) != 0 + ? B_ANY_KERNEL_BLOCK_ADDRESS : B_ANY_KERNEL_ADDRESS, + size, B_FULL_LOCK, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0, + (flags & CACHE_DONT_WAIT_FOR_MEMORY) != 0 + ? CREATE_AREA_DONT_WAIT : 0); + return area >= 0 ? B_OK : area; + } + + // determine chunk size (small or medium) + size_t chunkSize = SLAB_CHUNK_SIZE_SMALL; + uint32 chunkCount = size / SLAB_CHUNK_SIZE_SMALL; + + if (size % SLAB_CHUNK_SIZE_MEDIUM == 0) { + chunkSize = SLAB_CHUNK_SIZE_MEDIUM; + chunkCount = size / SLAB_CHUNK_SIZE_MEDIUM; + } + + MutexLocker locker(sLock); + + // allocate the chunks + MetaChunk* metaChunk; + Chunk* chunk; + status_t error = _AllocateChunks(chunkSize, chunkCount, flags, metaChunk, + chunk); + if (error != B_OK) + return error; + + // map the chunks + Area* area = metaChunk->GetArea(); + addr_t chunkAddress = _ChunkAddress(metaChunk, chunk); + + locker.Unlock(); + error = _MapChunk(area->vmArea, chunkAddress, size, 0, flags); + locker.Lock(); + if (error != B_OK) { + // something failed -- free the chunks + for (uint32 i = 0; i < chunkCount; i++) + _FreeChunk(area, metaChunk, chunk + i, chunkAddress, true, flags); + return error; + } + + chunk->reference = (addr_t)chunkAddress + size - 1; + _pages = (void*)chunkAddress; + + TRACE("MemoryManager::AllocateRaw() done: %p (meta chunk: %d, chunk %d)\n", + _pages, int(metaChunk - area->metaChunks), + int(chunk - metaChunk->chunks)); + return B_OK; +} + + +/*static*/ ObjectCache* +MemoryManager::FreeRawOrReturnCache(void* pages, uint32 flags) +{ + TRACE("MemoryManager::FreeRawOrReturnCache(%p, %#" B_PRIx32 ")\n", pages, + flags); + + T(FreeRawOrReturnCache(pages, flags)); + + // get the area + addr_t areaBase = ROUNDDOWN((addr_t)pages, SLAB_AREA_SIZE); + + ReadLocker readLocker(sAreaTableLock); + Area* area = sAreaTable.Lookup(areaBase); + readLocker.Unlock(); + + if (area == NULL) { + // Probably a large allocation. Look up the VM area. + VMAddressSpace* addressSpace = VMAddressSpace::Kernel(); + addressSpace->ReadLock(); + VMArea* area = addressSpace->LookupArea((addr_t)pages); + addressSpace->ReadUnlock(); + + if (area != NULL && (addr_t)pages == area->Base()) + delete_area(area->id); + else + panic("freeing unknown block %p from area %p", pages, area); + + return NULL; + } + + MetaChunk* metaChunk = &area->metaChunks[ + ((addr_t)pages % SLAB_AREA_SIZE) / SLAB_CHUNK_SIZE_LARGE]; + + // get the chunk + ASSERT((addr_t)pages >= metaChunk->chunkBase); + uint16 chunkIndex = _ChunkIndexForAddress(metaChunk, (addr_t)pages); + Chunk* chunk = &metaChunk->chunks[chunkIndex]; + + addr_t reference = chunk->reference; + if ((reference & 1) == 0) + return (ObjectCache*)reference; + + // Seems we have a raw chunk allocation. + ASSERT((addr_t)pages == _ChunkAddress(metaChunk, chunk)); + ASSERT(reference > (addr_t)pages); + ASSERT(reference <= areaBase + SLAB_AREA_SIZE - 1); + size_t size = reference - (addr_t)pages + 1; + ASSERT((size % SLAB_CHUNK_SIZE_SMALL) == 0); + + // unmap the chunks + _UnmapChunk(area->vmArea, (addr_t)pages, size, flags); + + // and free them + MutexLocker locker(sLock); + uint32 chunkCount = size / metaChunk->chunkSize; + for (uint32 i = 0; i < chunkCount; i++) + _FreeChunk(area, metaChunk, chunk + i, (addr_t)pages, true, flags); + + return NULL; +} + + /*static*/ size_t MemoryManager::AcceptableChunkSize(size_t size) { @@ -221,6 +724,48 @@ MemoryManager::AcceptableChunkSize(size_t size) } +/*static*/ ObjectCache* +MemoryManager::GetAllocationInfo(void* address, size_t& _size) +{ + // get the area + addr_t areaBase = ROUNDDOWN((addr_t)address, SLAB_AREA_SIZE); + + ReadLocker readLocker(sAreaTableLock); + Area* area = sAreaTable.Lookup(areaBase); + readLocker.Unlock(); + + if (area == NULL) { + VMAddressSpace* addressSpace = VMAddressSpace::Kernel(); + addressSpace->ReadLock(); + VMArea* area = addressSpace->LookupArea((addr_t)address); + if (area != NULL && (addr_t)address == area->Base()) + _size = area->Size(); + else + _size = 0; + addressSpace->ReadUnlock(); + + return NULL; + } + + MetaChunk* metaChunk = &area->metaChunks[ + ((addr_t)address % SLAB_AREA_SIZE) / SLAB_CHUNK_SIZE_LARGE]; + + // get the chunk + ASSERT((addr_t)address >= metaChunk->chunkBase); + uint16 chunkIndex = _ChunkIndexForAddress(metaChunk, (addr_t)address); + + addr_t reference = metaChunk->chunks[chunkIndex].reference; + if ((reference & 1) == 0) { + ObjectCache* cache = (ObjectCache*)reference; + _size = cache->object_size; + return cache; + } + + _size = reference - (addr_t)address + 1; + return NULL; +} + + /*static*/ ObjectCache* MemoryManager::CacheForAddress(void* address) { @@ -241,7 +786,8 @@ MemoryManager::CacheForAddress(void* address) ASSERT((addr_t)address >= metaChunk->chunkBase); uint16 chunkIndex = _ChunkIndexForAddress(metaChunk, (addr_t)address); - return metaChunk->chunks[chunkIndex].cache; + addr_t reference = metaChunk->chunks[chunkIndex].reference; + return (reference & 1) == 0 ? (ObjectCache*)reference : NULL; } @@ -283,8 +829,8 @@ MemoryManager::PerformMaintenance() /*static*/ status_t -MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags, - MetaChunk*& _metaChunk, Chunk*& _chunk) +MemoryManager::_AllocateChunks(size_t chunkSize, uint32 chunkCount, + uint32 flags, MetaChunk*& _metaChunk, Chunk*& _chunk) { MetaChunkList* metaChunkList = NULL; if (chunkSize == SLAB_CHUNK_SIZE_SMALL) { @@ -292,12 +838,12 @@ MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags, } else if (chunkSize == SLAB_CHUNK_SIZE_MEDIUM) { metaChunkList = &sPartialMetaChunksMedium; } else if (chunkSize != SLAB_CHUNK_SIZE_LARGE) { - panic("MemoryManager::_AllocateChunk(): Unsupported chunk size: %" + panic("MemoryManager::_AllocateChunks(): Unsupported chunk size: %" B_PRIuSIZE, chunkSize); return B_BAD_VALUE; } - if (_GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk)) + if (_GetChunks(metaChunkList, chunkSize, chunkCount, _metaChunk, _chunk)) return B_OK; if (sFreeAreas != NULL) { @@ -305,7 +851,7 @@ MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags, sFreeAreaCount--; _RequestMaintenance(); - _GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk); + _GetChunks(metaChunkList, chunkSize, chunkCount, _metaChunk, _chunk); return B_OK; } @@ -334,8 +880,10 @@ MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags, entry.Wait(); mutex_lock(&sLock); - if (_GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk)) + if (_GetChunks(metaChunkList, chunkSize, chunkCount, _metaChunk, + _chunk)) { return B_OK; + } } // prepare the allocation entry others can wait on @@ -359,17 +907,94 @@ MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags, // Try again to get a meta chunk. Something might have been freed in the // meantime. We can free the area in this case. - if (_GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk)) { + if (_GetChunks(metaChunkList, chunkSize, chunkCount, _metaChunk, _chunk)) { _FreeArea(area, true, flags); return B_OK; } _AddArea(area); - _GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk); + _GetChunks(metaChunkList, chunkSize, chunkCount, _metaChunk, _chunk); return B_OK; } +/*static*/ bool +MemoryManager::_GetChunks(MetaChunkList* metaChunkList, size_t chunkSize, + uint32 chunkCount, MetaChunk*& _metaChunk, Chunk*& _chunk) +{ + // the common and less complicated special case + if (chunkCount == 1) + return _GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk); + + ASSERT(metaChunkList != NULL); + + // Iterate through the partial meta chunk list and try to find a free + // range that is large enough. + MetaChunk* metaChunk = NULL; + for (MetaChunkList::Iterator it = metaChunkList->GetIterator(); + (metaChunk = it.Next()) != NULL;) { + if (metaChunk->firstFreeChunk + chunkCount - 1 + <= metaChunk->lastFreeChunk) { + break; + } + } + + if (metaChunk == NULL) { + // try to get a free meta chunk + if ((SLAB_CHUNK_SIZE_LARGE - kAreaAdminSize) / chunkSize >= chunkCount) + metaChunk = sFreeShortMetaChunks.RemoveHead(); + if (metaChunk == NULL) + metaChunk = sFreeCompleteMetaChunks.RemoveHead(); + + if (metaChunk == NULL) + return false; + + metaChunkList->Add(metaChunk); + metaChunk->GetArea()->usedMetaChunkCount++; + _PrepareMetaChunk(metaChunk, chunkSize); + + T(AllocateMetaChunk(metaChunk)); + } + + // pull the chunks out of the free list + Chunk* firstChunk = metaChunk->chunks + metaChunk->firstFreeChunk; + Chunk* lastChunk = firstChunk + (chunkCount - 1); + Chunk** chunkPointer = &metaChunk->freeChunks; + uint32 remainingChunks = chunkCount; + while (remainingChunks > 0) { + ASSERT_PRINT(chunkPointer, "remaining: %" B_PRIu32 "/%" B_PRIu32 + ", area: %p, meta chunk: %" B_PRIdSSIZE "\n", remainingChunks, + chunkCount, metaChunk->GetArea(), + metaChunk - metaChunk->GetArea()->metaChunks); + Chunk* chunk = *chunkPointer; + if (chunk >= firstChunk && chunk <= lastChunk) { + *chunkPointer = chunk->next; + chunk->reference = 1; + remainingChunks--; + } else + chunkPointer = &chunk->next; + } + + // allocate the chunks + metaChunk->usedChunkCount += chunkCount; + if (metaChunk->usedChunkCount == metaChunk->chunkCount) { + // meta chunk is full now -- remove it from its list + if (metaChunkList != NULL) + metaChunkList->Remove(metaChunk); + } + + // update the free range + metaChunk->firstFreeChunk += chunkCount; + + _chunk = firstChunk; + _metaChunk = metaChunk; + + T(AllocateChunks(chunkSize, chunkCount, metaChunk, firstChunk)); + + return true; +} + + /*static*/ bool MemoryManager::_GetChunk(MetaChunkList* metaChunkList, size_t chunkSize, MetaChunk*& _metaChunk, Chunk*& _chunk) @@ -393,6 +1018,8 @@ MemoryManager::_GetChunk(MetaChunkList* metaChunkList, size_t chunkSize, metaChunk->GetArea()->usedMetaChunkCount++; _PrepareMetaChunk(metaChunk, chunkSize); + + T(AllocateMetaChunk(metaChunk)); } // allocate the chunk @@ -404,6 +1031,20 @@ MemoryManager::_GetChunk(MetaChunkList* metaChunkList, size_t chunkSize, _chunk = _pop(metaChunk->freeChunks); _metaChunk = metaChunk; + + // update the free range + uint32 chunkIndex = _chunk - metaChunk->chunks; + if (chunkIndex >= metaChunk->firstFreeChunk + && chunkIndex <= metaChunk->lastFreeChunk) { + if (chunkIndex - metaChunk->firstFreeChunk + <= metaChunk->lastFreeChunk - chunkIndex) { + metaChunk->firstFreeChunk = chunkIndex + 1; + } else + metaChunk->lastFreeChunk = chunkIndex - 1; + } + + T(AllocateChunk(chunkSize, metaChunk, _chunk)); + return true; } @@ -419,11 +1060,17 @@ MemoryManager::_FreeChunk(Area* area, MetaChunk* metaChunk, Chunk* chunk, mutex_lock(&sLock); } + T(FreeChunk(metaChunk, chunk)); + _push(metaChunk->freeChunks, chunk); + uint32 chunkIndex = chunk - metaChunk->chunks; + // free the meta chunk, if it is unused now ASSERT(metaChunk->usedChunkCount > 0); if (--metaChunk->usedChunkCount == 0) { + T(FreeMetaChunk(metaChunk)); + // remove from partial meta chunk list if (metaChunk->chunkSize == SLAB_CHUNK_SIZE_SMALL) sPartialMetaChunksSmall.Remove(metaChunk); @@ -450,6 +1097,28 @@ MemoryManager::_FreeChunk(Area* area, MetaChunk* metaChunk, Chunk* chunk, sPartialMetaChunksSmall.Add(metaChunk, false); else if (metaChunk->chunkSize == SLAB_CHUNK_SIZE_MEDIUM) sPartialMetaChunksMedium.Add(metaChunk, false); + + metaChunk->firstFreeChunk = chunkIndex; + metaChunk->lastFreeChunk = chunkIndex; + } else { + // extend the free range, if the chunk adjoins + if (chunkIndex + 1 == metaChunk->firstFreeChunk) { + uint32 firstFree = chunkIndex; + for (; firstFree > 0; firstFree--) { + Chunk* previousChunk = &metaChunk->chunks[firstFree - 1]; + if (!_IsChunkFree(metaChunk, previousChunk)) + break; + } + metaChunk->firstFreeChunk = firstFree; + } else if (chunkIndex == (uint32)metaChunk->lastFreeChunk + 1) { + uint32 lastFree = chunkIndex; + for (; lastFree + 1 < metaChunk->chunkCount; lastFree++) { + Chunk* nextChunk = &metaChunk->chunks[lastFree + 1]; + if (!_IsChunkFree(metaChunk, nextChunk)) + break; + } + metaChunk->lastFreeChunk = lastFree; + } } } @@ -471,14 +1140,19 @@ MemoryManager::_PrepareMetaChunk(MetaChunk* metaChunk, size_t chunkSize) metaChunk->usedChunkCount = 0; metaChunk->freeChunks = NULL; - for (uint32 i = 0; i < metaChunk->chunkCount; i++) + for (int32 i = metaChunk->chunkCount - 1; i >= 0; i--) _push(metaChunk->freeChunks, metaChunk->chunks + i); + + metaChunk->firstFreeChunk = 0; + metaChunk->lastFreeChunk = metaChunk->chunkCount - 1; } /*static*/ void MemoryManager::_AddArea(Area* area) { + T(AddArea(area)); + // add the area to the hash table WriteLocker writeLocker(sAreaTableLock); sAreaTable.InsertUnchecked(area); @@ -565,6 +1239,9 @@ MemoryManager::_AllocateArea(uint32 flags, Area*& _area) mutex_lock(&sLock); _area = area; + + T(AllocateArea(area, flags)); + return B_OK; } @@ -574,16 +1251,18 @@ MemoryManager::_FreeArea(Area* area, bool areaRemoved, uint32 flags) { TRACE("MemoryManager::_FreeArea(%p, %#" B_PRIx32 ")\n", area, flags); + T(FreeArea(area, areaRemoved, flags)); + ASSERT(area->usedMetaChunkCount == 0); if (!areaRemoved) { // remove the area's meta chunks from the free lists ASSERT(area->metaChunks[0].usedChunkCount == 0); - sFreeShortMetaChunks.Add(&area->metaChunks[0]); + sFreeShortMetaChunks.Remove(&area->metaChunks[0]); for (int32 i = 1; i < SLAB_META_CHUNKS_PER_AREA; i++) { ASSERT(area->metaChunks[i].usedChunkCount == 0); - sFreeCompleteMetaChunks.Add(&area->metaChunks[i]); + sFreeCompleteMetaChunks.Remove(&area->metaChunks[i]); } // remove the area from the hash table @@ -624,6 +1303,8 @@ MemoryManager::_MapChunk(VMArea* vmArea, addr_t address, size_t size, TRACE("MemoryManager::_MapChunk(%p, %#" B_PRIxADDR ", %#" B_PRIxSIZE ")\n", vmArea, address, size); + T(Map(address, size, flags)); + if (vmArea == NULL) { // everything is mapped anyway return B_OK; @@ -686,6 +1367,8 @@ MemoryManager::_MapChunk(VMArea* vmArea, addr_t address, size_t size, MemoryManager::_UnmapChunk(VMArea* vmArea, addr_t address, size_t size, uint32 flags) { + T(Unmap(address, size, flags)); + if (vmArea == NULL) return B_ERROR; @@ -728,32 +1411,6 @@ MemoryManager::_UnmapChunk(VMArea* vmArea, addr_t address, size_t size, } -/*static*/ void -MemoryManager::_UnmapChunkEarly(addr_t address, size_t size) -{ - VMAddressSpace* addressSpace = VMAddressSpace::Kernel(); - VMTranslationMap* translationMap = addressSpace->TranslationMap(); - - translationMap->Lock(); - - for (size_t offset = 0; offset < B_PAGE_SIZE; offset += B_PAGE_SIZE) { - addr_t physicalAddress; - uint32 flags; - if (translationMap->Query(address + offset, &physicalAddress, &flags) - == B_OK - && (flags & PAGE_PRESENT) != 0) { - vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE); - DEBUG_PAGE_ACCESS_START(page); - vm_page_set_state(page, PAGE_STATE_FREE); - } - } - - translationMap->Unmap(address, address + size - 1); - - translationMap->Unlock(); -} - - /*static*/ void MemoryManager::_UnmapFreeChunksEarly(Area* area) { @@ -820,6 +1477,48 @@ MemoryManager::_RequestMaintenance() } +/*static*/ int +MemoryManager::_DumpRawAllocations(int argc, char** argv) +{ + kprintf("area meta chunk chunk base size (KB)\n"); + + size_t totalSize = 0; + + for (AreaTable::Iterator it = sAreaTable.GetIterator(); + Area* area = it.Next();) { + for (int32 i = 0; i < SLAB_META_CHUNKS_PER_AREA; i++) { + MetaChunk* metaChunk = area->metaChunks + i; + if (metaChunk->chunkSize == 0) + continue; + for (uint32 k = 0; k < metaChunk->chunkCount; k++) { + Chunk* chunk = metaChunk->chunks + k; + + // skip free chunks + if (_IsChunkFree(metaChunk, chunk)) + continue; + + addr_t reference = chunk->reference; + if ((reference & 1) == 0 || reference == 1) + continue; + + addr_t chunkAddress = _ChunkAddress(metaChunk, chunk); + size_t size = reference - chunkAddress + 1; + totalSize += size; + + kprintf("%p %10" B_PRId32 " %5" B_PRIu32 " %p %9" + B_PRIuSIZE "\n", area, i, k, (void*)chunkAddress, + size / 1024); + } + } + } + + kprintf("total: %9" B_PRIuSIZE "\n", + totalSize / 1024); + + return 0; +} + + /*static*/ void MemoryManager::_PrintMetaChunkTableHeader(bool printChunks) { @@ -855,8 +1554,9 @@ MemoryManager::_DumpMetaChunk(MetaChunk* metaChunk, bool printChunks, kprintf("%5d %p --- %6s meta chunk", metaChunkIndex, (void*)metaChunk->chunkBase, type); if (metaChunk->chunkSize != 0) { - kprintf(": %4u/%4u used ----------------------------\n", - metaChunk->usedChunkCount, metaChunk->chunkCount); + kprintf(": %4u/%4u used, %-4u-%4u free ------------\n", + metaChunk->usedChunkCount, metaChunk->chunkCount, + metaChunk->firstFreeChunk, metaChunk->lastFreeChunk); } else kprintf(" --------------------------------------------\n"); @@ -867,18 +1567,20 @@ MemoryManager::_DumpMetaChunk(MetaChunk* metaChunk, bool printChunks, Chunk* chunk = metaChunk->chunks + i; // skip free chunks - if (chunk->next == NULL) + if (_IsChunkFree(metaChunk, chunk)) continue; - if (chunk->next >= metaChunk->chunks - && chunk->next < metaChunk->chunks + metaChunk->chunkCount) { - continue; - } - ObjectCache* cache = chunk->cache; - kprintf("%5" B_PRIu32 " %p %p %11" B_PRIuSIZE " %s\n", i, - (void*)_ChunkAddress(metaChunk, chunk), cache, - cache != NULL ? cache->object_size : 0, - cache != NULL ? cache->name : ""); + addr_t reference = chunk->reference; + if ((reference & 1) == 0) { + ObjectCache* cache = (ObjectCache*)reference; + kprintf("%5" B_PRIu32 " %p %p %11" B_PRIuSIZE " %s\n", i, + (void*)_ChunkAddress(metaChunk, chunk), cache, + cache != NULL ? cache->object_size : 0, + cache != NULL ? cache->name : ""); + } else if (reference != 1) { + kprintf("%5" B_PRIu32 " %p raw allocation up to %p\n", i, + (void*)_ChunkAddress(metaChunk, chunk), (void*)reference); + } } } @@ -986,14 +1688,22 @@ MemoryManager::_DumpAreas(int argc, char** argv) { kprintf(" base area meta small medium large\n"); + size_t totalTotalSmall = 0; + size_t totalUsedSmall = 0; + size_t totalTotalMedium = 0; + size_t totalUsedMedium = 0; + size_t totalUsedLarge = 0; + uint32 areaCount = 0; + for (AreaTable::Iterator it = sAreaTable.GetIterator(); Area* area = it.Next();) { + areaCount++; + // sum up the free/used counts for the chunk sizes int totalSmall = 0; int usedSmall = 0; int totalMedium = 0; int usedMedium = 0; - int totalLarge = 0; int usedLarge = 0; for (int32 i = 0; i < SLAB_META_CHUNKS_PER_AREA; i++) { @@ -1011,21 +1721,43 @@ MemoryManager::_DumpAreas(int argc, char** argv) usedMedium += metaChunk->usedChunkCount; break; case SLAB_CHUNK_SIZE_LARGE: - totalLarge += metaChunk->chunkCount; usedLarge += metaChunk->usedChunkCount; break; } } - kprintf("%p %p %2u/%2u %4d/%4d %3d/%3d %2d/%2d\n", + kprintf("%p %p %2u/%2u %4d/%4d %3d/%3d %5d\n", area, area->vmArea, area->usedMetaChunkCount, SLAB_META_CHUNKS_PER_AREA, usedSmall, totalSmall, usedMedium, - totalMedium, usedLarge, totalLarge); + totalMedium, usedLarge); + + totalTotalSmall += totalSmall; + totalUsedSmall += usedSmall; + totalTotalMedium += totalMedium; + totalUsedMedium += usedMedium; + totalUsedLarge += usedLarge; } - kprintf("%d free areas:\n", sFreeAreaCount); - for (Area* area = sFreeAreas; area != NULL; area = area->next) + kprintf("%d free area%s:\n", sFreeAreaCount, + sFreeAreaCount == 1 ? "" : "s"); + for (Area* area = sFreeAreas; area != NULL; area = area->next) { + areaCount++; kprintf("%p %p\n", area, area->vmArea); + } + + kprintf("total usage:\n"); + kprintf(" small: %" B_PRIuSIZE "/%" B_PRIuSIZE "\n", totalUsedSmall, + totalTotalSmall); + kprintf(" medium: %" B_PRIuSIZE "/%" B_PRIuSIZE "\n", totalUsedMedium, + totalTotalMedium); + kprintf(" large: %" B_PRIuSIZE "\n", totalUsedLarge); + kprintf(" memory: %" B_PRIuSIZE "/%" B_PRIuSIZE " KB\n", + (totalUsedSmall * SLAB_CHUNK_SIZE_SMALL + + totalUsedMedium * SLAB_CHUNK_SIZE_MEDIUM + + totalUsedLarge * SLAB_CHUNK_SIZE_LARGE) / 1024, + areaCount * SLAB_AREA_SIZE / 1024); + kprintf(" overhead: %" B_PRIuSIZE " KB\n", + areaCount * kAreaAdminSize / 1024); return 0; } diff --git a/src/system/kernel/slab/MemoryManager.h b/src/system/kernel/slab/MemoryManager.h index d389b6e2e3..c84e74738b 100644 --- a/src/system/kernel/slab/MemoryManager.h +++ b/src/system/kernel/slab/MemoryManager.h @@ -41,19 +41,28 @@ public: void*& _pages); static void Free(void* pages, uint32 flags); + static status_t AllocateRaw(size_t size, uint32 flags, + void*& _pages); + static ObjectCache* FreeRawOrReturnCache(void* pages, + uint32 flags); + static size_t AcceptableChunkSize(size_t size); + static ObjectCache* GetAllocationInfo(void* address, + size_t& _size); static ObjectCache* CacheForAddress(void* address); static bool MaintenanceNeeded(); static void PerformMaintenance(); private: + struct Tracing; + struct Area; struct Chunk { union { Chunk* next; - ObjectCache* cache; + addr_t reference; }; }; @@ -63,6 +72,8 @@ private: size_t totalSize; uint16 chunkCount; uint16 usedChunkCount; + uint16 firstFreeChunk; // *some* free range + uint16 lastFreeChunk; // inclusive Chunk chunks[SLAB_SMALL_CHUNKS_PER_META_CHUNK]; Chunk* freeChunks; @@ -115,7 +126,11 @@ private: }; private: - static status_t _AllocateChunk(size_t chunkSize, uint32 flags, + static status_t _AllocateChunks(size_t chunkSize, + uint32 chunkCount, uint32 flags, + MetaChunk*& _metaChunk, Chunk*& _chunk); + static bool _GetChunks(MetaChunkList* metaChunkList, + size_t chunkSize, uint32 chunkCount, MetaChunk*& _metaChunk, Chunk*& _chunk); static bool _GetChunk(MetaChunkList* metaChunkList, size_t chunkSize, MetaChunk*& _metaChunk, @@ -135,10 +150,9 @@ private: static status_t _MapChunk(VMArea* vmArea, addr_t address, size_t size, size_t reserveAdditionalMemory, uint32 flags); - static status_t _UnmapChunk(VMArea* vmArea,addr_t address, + static status_t _UnmapChunk(VMArea* vmArea, addr_t address, size_t size, uint32 flags); - static void _UnmapChunkEarly(addr_t address, size_t size); static void _UnmapFreeChunksEarly(Area* area); static void _ConvertEarlyArea(Area* area); @@ -148,7 +162,10 @@ private: const MetaChunk* metaChunk, addr_t address); static addr_t _ChunkAddress(const MetaChunk* metaChunk, const Chunk* chunk); + static bool _IsChunkFree(const MetaChunk* metaChunk, + const Chunk* chunk); + static int _DumpRawAllocations(int argc, char** argv); static void _PrintMetaChunkTableHeader(bool printChunks); static void _DumpMetaChunk(MetaChunk* metaChunk, bool printChunks, bool printHeader); @@ -202,6 +219,15 @@ MemoryManager::_ChunkAddress(const MetaChunk* metaChunk, const Chunk* chunk) } +/*static*/ inline bool +MemoryManager::_IsChunkFree(const MetaChunk* metaChunk, const Chunk* chunk) +{ + return chunk->next == NULL + || (chunk->next >= metaChunk->chunks + && chunk->next < metaChunk->chunks + metaChunk->chunkCount); +} + + inline MemoryManager::Area* MemoryManager::MetaChunk::GetArea() const { diff --git a/src/system/kernel/slab/ObjectCache.cpp b/src/system/kernel/slab/ObjectCache.cpp index d4acf09357..668f30fdcd 100644 --- a/src/system/kernel/slab/ObjectCache.cpp +++ b/src/system/kernel/slab/ObjectCache.cpp @@ -112,15 +112,18 @@ ObjectCache::InitSlab(slab* slab, void* pages, size_t byteCount, uint32 flags) slab->pages = pages; slab->count = slab->size = byteCount / object_size; slab->free = NULL; - total_objects += slab->size; size_t spareBytes = byteCount - (slab->size * object_size); - slab->offset = cache_color_cycle; - if (slab->offset > spareBytes) - cache_color_cycle = slab->offset = 0; - else - cache_color_cycle += kCacheColorPeriod; + if ((this->flags & CACHE_ALIGN_ON_SIZE) != 0) { + slab->offset = cache_color_cycle; + + if (slab->offset > spareBytes) + cache_color_cycle = slab->offset = 0; + else + cache_color_cycle += kCacheColorPeriod; + } else + slab->offset = 0; TRACE_CACHE(this, " %lu objects, %lu spare bytes, offset %lu", slab->size, spareBytes, slab->offset); @@ -163,6 +166,9 @@ ObjectCache::InitSlab(slab* slab, void* pages, size_t byteCount, uint32 flags) data += object_size; } + usage += slab_size; + total_objects += slab->size; + return slab; } @@ -175,6 +181,7 @@ ObjectCache::UninitSlab(slab* slab) if (slab->count != slab->size) panic("cache: destroying a slab which isn't empty."); + usage -= slab_size; total_objects -= slab->size; DELETE_PARANOIA_CHECK_SET(slab); diff --git a/src/system/kernel/slab/ObjectCache.h b/src/system/kernel/slab/ObjectCache.h index 4cde8f8af3..9baa64c192 100644 --- a/src/system/kernel/slab/ObjectCache.h +++ b/src/system/kernel/slab/ObjectCache.h @@ -34,6 +34,7 @@ typedef DoublyLinkedList SlabList; struct ObjectCacheResizeEntry { ConditionVariable condition; + thread_id thread; }; struct ObjectCache : DoublyLinkedListLinkImpl { diff --git a/src/system/kernel/slab/Slab.cpp b/src/system/kernel/slab/Slab.cpp index b70148165e..4bc67cee83 100644 --- a/src/system/kernel/slab/Slab.cpp +++ b/src/system/kernel/slab/Slab.cpp @@ -50,10 +50,10 @@ static MaintenanceQueue sMaintenanceQueue; static ConditionVariable sMaintenanceCondition; -#if OBJECT_CACHE_TRACING +#if SLAB_OBJECT_CACHE_TRACING -namespace ObjectCacheTracing { +namespace SlabObjectCacheTracing { class ObjectCacheTraceEntry : public AbstractTraceEntry { public: @@ -186,13 +186,13 @@ class Reserve : public ObjectCacheTraceEntry { }; -} // namespace ObjectCacheTracing +} // namespace SlabObjectCacheTracing -# define T(x) new(std::nothrow) ObjectCacheTracing::x +# define T(x) new(std::nothrow) SlabObjectCacheTracing::x #else # define T(x) -#endif // OBJECT_CACHE_TRACING +#endif // SLAB_OBJECT_CACHE_TRACING // #pragma mark - @@ -211,8 +211,7 @@ dump_slabs(int argc, char* argv[]) kprintf("%p %22s %8lu %8lu %6lu %8lu %8lu %8lx\n", cache, cache->name, cache->object_size, cache->usage, cache->empty_count, - cache->used_count, cache->usage / cache->object_size, - cache->flags); + cache->used_count, cache->total_objects, cache->flags); } return 0; @@ -241,6 +240,8 @@ dump_cache_info(int argc, char* argv[]) kprintf("maximum: %lu\n", cache->maximum); kprintf("flags: 0x%lx\n", cache->flags); kprintf("cookie: %p\n", cache->cookie); + kprintf("resize entry don't wait: %p\n", cache->resize_entry_dont_wait); + kprintf("resize entry can wait: %p\n", cache->resize_entry_can_wait); return 0; } @@ -249,23 +250,6 @@ dump_cache_info(int argc, char* argv[]) // #pragma mark - -void* -slab_internal_alloc(size_t size, uint32 flags) -{ - if (flags & CACHE_DURING_BOOT) - return block_alloc_early(size); - - return block_alloc(size, flags); -} - - -void -slab_internal_free(void* buffer, uint32 flags) -{ - block_free(buffer, flags); -} - - void request_memory_manager_maintenance() { @@ -322,6 +306,7 @@ object_cache_reserve_internal(ObjectCache* cache, size_t objectCount, { // If someone else is already adding slabs, we wait for that to be finished // first. + thread_id thread = find_thread(NULL); while (true) { if (objectCount <= cache->total_objects - cache->used_count) return B_OK; @@ -329,9 +314,19 @@ object_cache_reserve_internal(ObjectCache* cache, size_t objectCount, ObjectCacheResizeEntry* resizeEntry = NULL; if (cache->resize_entry_dont_wait != NULL) { resizeEntry = cache->resize_entry_dont_wait; - } else if (cache->resize_entry_can_wait != NULL - && (flags & CACHE_DONT_WAIT_FOR_MEMORY) == 0) { + if (thread == resizeEntry->thread) + return B_WOULD_BLOCK; + // Note: We could still have reentered the function, i.e. + // resize_entry_can_wait would be ours. That doesn't matter much, + // though, since after the don't-wait thread has done its job + // everyone will be happy. + } else if (cache->resize_entry_can_wait != NULL) { resizeEntry = cache->resize_entry_can_wait; + if (thread == resizeEntry->thread) + return B_WOULD_BLOCK; + + if ((flags & CACHE_DONT_WAIT_FOR_MEMORY) != 0) + break; } else break; @@ -351,6 +346,7 @@ object_cache_reserve_internal(ObjectCache* cache, size_t objectCount, ObjectCacheResizeEntry myResizeEntry; resizeEntry = &myResizeEntry; resizeEntry->condition.Init(cache, "wait for slabs"); + resizeEntry->thread = thread; // add new slabs until there are as many free ones as requested while (objectCount > cache->total_objects - cache->used_count) { @@ -431,29 +427,13 @@ object_cache_low_memory(void* dummy, uint32 resources, int32 level) break; } - // If the object cache has minimum object reserve, make sure that we - // don't free too many slabs. - if (cache->min_object_reserve > 0 && cache->empty_count > 0) { + while (cache->empty_count > minimumAllowed) { + // make sure we respect the cache's minimum object reserve size_t objectsPerSlab = cache->empty.Head()->size; size_t freeObjects = cache->total_objects - cache->used_count; + if (freeObjects < cache->min_object_reserve + objectsPerSlab) + break; - if (cache->min_object_reserve + objectsPerSlab >= freeObjects) - return; - - size_t slabsToFree = (freeObjects - cache->min_object_reserve) - / objectsPerSlab; - - if (cache->empty_count > minimumAllowed + slabsToFree) - minimumAllowed = cache->empty_count - slabsToFree; - } - - if (cache->empty_count <= minimumAllowed) - return; - - TRACE_CACHE(cache, "cache: memory pressure, will release down to %lu.", - minimumAllowed); - - while (cache->empty_count > minimumAllowed) { cache->ReturnSlab(cache->empty.RemoveHead(), 0); cache->empty_count--; } @@ -649,23 +629,26 @@ object_cache_alloc(object_cache* cache, uint32 flags) } MutexLocker _(cache->lock); - slab* source; + slab* source = NULL; - if (cache->partial.IsEmpty()) { - if (cache->empty.IsEmpty()) { - if (object_cache_reserve_internal(cache, 1, flags) < B_OK) { - T(Alloc(cache, flags, NULL)); - return NULL; - } - - cache->pressure++; - } + while (true) { + source = cache->partial.Head(); + if (source != NULL) + break; source = cache->empty.RemoveHead(); - cache->empty_count--; - cache->partial.Add(source); - } else { - source = cache->partial.Head(); + if (source != NULL) { + cache->empty_count--; + cache->partial.Add(source); + break; + } + + if (object_cache_reserve_internal(cache, 1, flags) != B_OK) { + T(Alloc(cache, flags, NULL)); + return NULL; + } + + cache->pressure++; } ParanoiaChecker _2(source); @@ -734,19 +717,17 @@ object_cache_get_usage(object_cache* cache, size_t* _allocatedMemory) void -slab_init(kernel_args* args, addr_t initialBase, size_t initialSize) +slab_init(kernel_args* args) { - dprintf("slab: init base %p + 0x%lx\n", (void*)initialBase, initialSize); - MemoryManager::Init(args); new (&sObjectCaches) ObjectCacheList(); - block_allocator_init_boot(initialBase, initialSize); + block_allocator_init_boot(); add_debugger_command("slabs", dump_slabs, "list all object caches"); - add_debugger_command("cache_info", dump_cache_info, - "dump information about a specific cache"); + add_debugger_command("slab_cache", dump_cache_info, + "dump information about a specific object cache"); } diff --git a/src/system/kernel/slab/SmallObjectCache.cpp b/src/system/kernel/slab/SmallObjectCache.cpp index 76fb73b52c..c69112eb5b 100644 --- a/src/system/kernel/slab/SmallObjectCache.cpp +++ b/src/system/kernel/slab/SmallObjectCache.cpp @@ -24,9 +24,8 @@ SmallObjectCache::Create(const char* name, size_t object_size, SmallObjectCache* cache = new(buffer) SmallObjectCache(); - if (cache->Init(name, object_size, alignment, maximum, - flags | CACHE_ALIGN_ON_SIZE, cookie, constructor, destructor, - reclaimer) != B_OK) { + if (cache->Init(name, object_size, alignment, maximum, flags, cookie, + constructor, destructor, reclaimer) != B_OK) { cache->Delete(); return NULL; } diff --git a/src/system/kernel/slab/allocator.cpp b/src/system/kernel/slab/allocator.cpp index 2dec030c53..1abd82700e 100644 --- a/src/system/kernel/slab/allocator.cpp +++ b/src/system/kernel/slab/allocator.cpp @@ -13,8 +13,12 @@ #include #include +#include + +#include +#include #include // for ROUNDUP -#include +#include #include #include @@ -38,9 +42,9 @@ static const size_t kNumBlockSizes = sizeof(kBlockSizes) / sizeof(size_t) - 1; static object_cache* sBlockCaches[kNumBlockSizes]; -static addr_t sBootStrapMemory; -static size_t sBootStrapMemorySize; -static size_t sUsedBootStrapMemory; +static addr_t sBootStrapMemory = 0; +static size_t sBootStrapMemorySize = 0; +static size_t sUsedBootStrapMemory = 0; static int @@ -68,24 +72,33 @@ size_to_index(size_t size) void* -block_alloc(size_t size, uint32 flags) +block_alloc(size_t size, size_t alignment, uint32 flags) { + if (alignment > 8) { + // Make size >= alignment and a power of two. This is sufficient, since + // all of our object caches with power of two sizes are aligned. We may + // waste quite a bit of memory, but memalign() is very rarely used + // in the kernel and always with power of two size == alignment anyway. + ASSERT((alignment & (alignment - 1)) == 0); + while (alignment < size) + alignment <<= 1; + size = alignment; + + // If we're not using an object cache, make sure that the memory + // manager knows it has to align the allocation. + if (size > kBlockSizes[kNumBlockSizes]) + flags |= CACHE_ALIGN_ON_SIZE; + } + // allocate from the respective object cache, if any int index = size_to_index(size); if (index >= 0) return object_cache_alloc(sBlockCaches[index], flags); - // the allocation is too large for our object caches -- create an area - if ((flags & CACHE_DONT_LOCK_KERNEL_SPACE) != 0) - return NULL; - + // the allocation is too large for our object caches -- ask the memory + // manager void* block; - area_id area = create_area_etc(VMAddressSpace::KernelID(), - "alloc'ed block", &block, B_ANY_KERNEL_ADDRESS, - ROUNDUP(size, B_PAGE_SIZE), B_FULL_LOCK, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0, - (flags & CACHE_DONT_WAIT_FOR_MEMORY) != 0 ? CREATE_AREA_DONT_WAIT : 0); - if (area < 0) + if (MemoryManager::AllocateRaw(size, flags, block) != B_OK) return NULL; return block; @@ -96,14 +109,31 @@ void* block_alloc_early(size_t size) { int index = size_to_index(size); - if (index < 0) - return NULL; - - if (sBlockCaches[index] != NULL) + if (index >= 0 && sBlockCaches[index] != NULL) return object_cache_alloc(sBlockCaches[index], CACHE_DURING_BOOT); - // No object cache yet. Use the bootstrap memory. This allocation must - // never be freed! + if (size > SLAB_CHUNK_SIZE_SMALL) { + // This is a sufficiently large allocation -- just ask the memory + // manager directly. + void* block; + if (MemoryManager::AllocateRaw(size, 0, block) != B_OK) + return NULL; + + return block; + } + + // A small allocation, but no object cache yet. Use the bootstrap memory. + // This allocation must never be freed! + if (sBootStrapMemorySize - sUsedBootStrapMemory < size) { + // We need more memory. + void* block; + if (MemoryManager::AllocateRaw(SLAB_CHUNK_SIZE_SMALL, 0, block) != B_OK) + return NULL; + sBootStrapMemory = (addr_t)block; + sBootStrapMemorySize = SLAB_CHUNK_SIZE_SMALL; + sUsedBootStrapMemory = 0; + } + size_t neededSize = ROUNDUP(size, sizeof(double)); if (sUsedBootStrapMemory + neededSize > sBootStrapMemorySize) return NULL; @@ -117,40 +147,41 @@ block_alloc_early(size_t size) void block_free(void* block, uint32 flags) { - if (ObjectCache* cache = MemoryManager::CacheForAddress(block)) { + if (block == NULL) + return; + + ObjectCache* cache = MemoryManager::FreeRawOrReturnCache(block, flags); + if (cache != NULL) { // a regular small allocation ASSERT(cache->object_size >= kBlockSizes[0]); ASSERT(cache->object_size <= kBlockSizes[kNumBlockSizes - 1]); ASSERT(cache == sBlockCaches[size_to_index(cache->object_size)]); object_cache_free(cache, block, flags); - } else { - // a large allocation -- look up the area - VMAddressSpace* addressSpace = VMAddressSpace::Kernel(); - addressSpace->ReadLock(); - VMArea* area = addressSpace->LookupArea((addr_t)block); - addressSpace->ReadUnlock(); - - if (area != NULL && (addr_t)block == area->Base()) - delete_area(area->id); - else - panic("freeing unknown block %p from area %p", block, area); } } void -block_allocator_init_boot(addr_t bootStrapBase, size_t bootStrapSize) +block_allocator_init_boot() { - sBootStrapMemory = bootStrapBase; - sBootStrapMemorySize = bootStrapSize; - sUsedBootStrapMemory = 0; - for (int index = 0; kBlockSizes[index] != 0; index++) { char name[32]; snprintf(name, sizeof(name), "block cache: %lu", kBlockSizes[index]); - sBlockCaches[index] = create_object_cache_etc(name, kBlockSizes[index], - 0, 0, CACHE_DURING_BOOT, NULL, NULL, NULL, NULL); + uint32 flags = CACHE_DURING_BOOT; + size_t size = kBlockSizes[index]; + + // align the power of two objects to their size + if ((size & (size - 1)) == 0) + flags |= CACHE_ALIGN_ON_SIZE; + + // For the larger allocation sizes disable the object depot, so we don't + // keep lot's of unused objects around. + if (size > 2048) + flags |= CACHE_NO_DEPOT; + + sBlockCaches[index] = create_object_cache_etc(name, size, 0, 0, flags, + NULL, NULL, NULL, NULL); if (sBlockCaches[index] == NULL) panic("allocator: failed to init block cache"); } @@ -162,7 +193,87 @@ block_allocator_init_rest() { #ifdef TEST_ALL_CACHES_DURING_BOOT for (int index = 0; kBlockSizes[index] != 0; index++) { - block_free(block_alloc(kBlockSizes[index] - sizeof(boundary_tag))); + block_free(block_alloc(kBlockSizes[index] - sizeof(boundary_tag)), 0, + 0); } #endif } + + +// #pragma mark - public API + + +#if USE_SLAB_ALLOCATOR_FOR_MALLOC + + +void* +memalign(size_t alignment, size_t size) +{ + return block_alloc(size, alignment, 0); +} + + +void* +memalign_nogrow(size_t alignment, size_t size) +{ + return block_alloc(size, alignment, + CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); +} + + +void* +malloc_nogrow(size_t size) +{ + return block_alloc(size, 0, + CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); +} + + +void* +malloc(size_t size) +{ + return block_alloc(size, 0, 0); +} + + +void +free(void* address) +{ + block_free(address, 0); +} + + +void* +realloc(void* address, size_t newSize) +{ + if (newSize == 0) { + block_free(address, 0); + return NULL; + } + + if (address == NULL) + return block_alloc(newSize, 0, 0); + + size_t oldSize; + ObjectCache* cache = MemoryManager::GetAllocationInfo(address, oldSize); + if (cache == NULL && oldSize == 0) { + panic("block_realloc(): allocation %p not known", address); + return NULL; + } + + if (oldSize == newSize) + return address; + + void* newBlock = block_alloc(newSize, 0, 0); + if (newBlock == NULL) + return NULL; + + memcpy(newBlock, address, std::min(oldSize, newSize)); + + block_free(address, 0); + + return newBlock; +} + + +#endif // USE_SLAB_ALLOCATOR_FOR_MALLOC diff --git a/src/system/kernel/slab/slab_private.h b/src/system/kernel/slab/slab_private.h index a4cb7b94fd..0282c9e8e6 100644 --- a/src/system/kernel/slab/slab_private.h +++ b/src/system/kernel/slab/slab_private.h @@ -11,6 +11,8 @@ #include +#include + //#define TRACE_SLAB #ifdef TRACE_SLAB @@ -26,16 +28,12 @@ struct ObjectCache; -void* slab_internal_alloc(size_t size, uint32 flags); -void slab_internal_free(void *_buffer, uint32 flags); - void request_memory_manager_maintenance(); -void* block_alloc(size_t size, uint32 flags); +void* block_alloc(size_t size, size_t alignment, uint32 flags); void* block_alloc_early(size_t size); -void block_free(void *block, uint32 flags); -void block_allocator_init_boot(addr_t bootStrapBase, - size_t bootStrapSize); +void block_free(void* block, uint32 flags); +void block_allocator_init_boot(); void block_allocator_init_rest(); @@ -58,4 +56,21 @@ _push(Type*& head, Type* object) } +static inline void* +slab_internal_alloc(size_t size, uint32 flags) +{ + if (flags & CACHE_DURING_BOOT) + return block_alloc_early(size); + + return block_alloc(size, 0, flags); +} + + +static inline void +slab_internal_free(void* buffer, uint32 flags) +{ + block_free(buffer, flags); +} + + #endif // SLAB_PRIVATE_H diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index c25004165b..e74d9ca6f1 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -3223,17 +3223,14 @@ vm_init(kernel_args* args) if (heapSize < 1024 * 1024) panic("vm_init: go buy some RAM please."); + slab_init(args); + // map in the new heap and initialize it addr_t heapBase = vm_allocate_early(args, heapSize, heapSize, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, false); TRACE(("heap at 0x%lx\n", heapBase)); heap_init(heapBase, heapSize); - size_t slabInitialSize = B_PAGE_SIZE; - addr_t slabInitialBase = vm_allocate_early(args, slabInitialSize, - slabInitialSize, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, false); - slab_init(args, slabInitialBase, slabInitialSize); - // initialize the free page list and physical page mapper vm_page_init(args); @@ -3262,11 +3259,6 @@ vm_init(kernel_args* args) create_area("kernel heap", &address, B_EXACT_ADDRESS, heapSize, B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - address = (void*)ROUNDDOWN(slabInitialBase, B_PAGE_SIZE); - create_area("initial slab space", &address, B_EXACT_ADDRESS, - slabInitialSize, B_ALREADY_WIRED, B_KERNEL_READ_AREA - | B_KERNEL_WRITE_AREA); - allocate_kernel_args(args); create_preloaded_image_areas(&args->kernel_image);