From 6bad4934393e784ea38685a76d28643deec67184 Mon Sep 17 00:00:00 2001 From: Hugo Santos Date: Sun, 29 Apr 2007 02:23:37 +0000 Subject: [PATCH] pushed the slab init a bit deeper. added a object cache based allocator, including a bootstrap mechanism to have it init during bootup. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@20896 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/private/kernel/slab/Base.h | 11 +- headers/private/kernel/slab/Depot.h | 2 +- headers/private/kernel/vm.h | 5 +- src/system/kernel/main.c | 2 - src/system/kernel/slab/Jamfile | 1 + src/system/kernel/slab/Slab.cpp | 234 ++++++++++++++++++++------ src/system/kernel/slab/allocator.cpp | 139 +++++++++++++++ src/system/kernel/slab/slab_private.h | 21 +++ src/system/kernel/vm/vm.cpp | 12 ++ 9 files changed, 367 insertions(+), 60 deletions(-) create mode 100644 src/system/kernel/slab/allocator.cpp create mode 100644 src/system/kernel/slab/slab_private.h diff --git a/headers/private/kernel/slab/Base.h b/headers/private/kernel/slab/Base.h index f8f27001fb..8843b36222 100644 --- a/headers/private/kernel/slab/Base.h +++ b/headers/private/kernel/slab/Base.h @@ -16,14 +16,15 @@ extern "C" { #endif -/* create_object_cache_etc flags */ enum { + /* create_object_cache_etc flags */ CACHE_NO_DEPOT = 1 << 0, -}; -/* object_cache_alloc flags */ -enum { - CACHE_DONT_SLEEP = 1 << 0, + /* object_cache_alloc flags */ + CACHE_DONT_SLEEP = 1 << 8, + + /* internal */ + CACHE_DURING_BOOT = 1 << 31 }; typedef struct object_cache object_cache; diff --git a/headers/private/kernel/slab/Depot.h b/headers/private/kernel/slab/Depot.h index 6bbe5f8bb9..e6fa721efe 100644 --- a/headers/private/kernel/slab/Depot.h +++ b/headers/private/kernel/slab/Depot.h @@ -26,7 +26,7 @@ typedef struct object_depot { } object_depot; -status_t object_depot_init(object_depot *depot, +status_t object_depot_init(object_depot *depot, uint32 flags, void (*return_object)(object_depot *, void *)); void object_depot_destroy(object_depot *depot); diff --git a/headers/private/kernel/vm.h b/headers/private/kernel/vm.h index b660dea3f2..086dac4fec 100644 --- a/headers/private/kernel/vm.h +++ b/headers/private/kernel/vm.h @@ -23,7 +23,6 @@ extern "C" { // startup only status_t vm_init(kernel_args *args); -status_t slab_init(); status_t vm_init_post_sem(struct kernel_args *args); status_t vm_init_post_thread(struct kernel_args *args); status_t vm_init_post_modules(struct kernel_args *args); @@ -32,6 +31,10 @@ void vm_free_unused_boot_loader_range(addr_t start, addr_t end); addr_t vm_allocate_early(kernel_args *args, size_t virtualSize, size_t physicalSize, uint32 attributes); + +void slab_init(addr_t initialBase, size_t initialSize); +void slab_init_post_sem(); + // to protect code regions with interrupts turned on void permit_page_faults(void); void forbid_page_faults(void); diff --git a/src/system/kernel/main.c b/src/system/kernel/main.c index 2c7905fb7f..4156294164 100644 --- a/src/system/kernel/main.c +++ b/src/system/kernel/main.c @@ -137,8 +137,6 @@ _start(kernel_args *bootKernelArgs, int currentCPU) generic_syscall_init(); TRACE("init cbuf\n"); cbuf_init(); - TRACE("init slab\n"); - slab_init(); TRACE("init teams\n"); team_init(&sKernelArgs); TRACE("init threads\n"); diff --git a/src/system/kernel/slab/Jamfile b/src/system/kernel/slab/Jamfile index b9aebd4cf1..3259668bb2 100644 --- a/src/system/kernel/slab/Jamfile +++ b/src/system/kernel/slab/Jamfile @@ -3,6 +3,7 @@ SubDir HAIKU_TOP src system kernel slab ; UsePrivateHeaders [ FDirName kernel slab ] ; KernelMergeObject kernel_slab.o : + allocator.cpp Slab.cpp : $(TARGET_KERNEL_PIC_CCFLAGS) -Wno-unused diff --git a/src/system/kernel/slab/Slab.cpp b/src/system/kernel/slab/Slab.cpp index 62f8a4064a..ca0499e082 100644 --- a/src/system/kernel/slab/Slab.cpp +++ b/src/system/kernel/slab/Slab.cpp @@ -8,6 +8,9 @@ #include +#include "slab_private.h" + +#include #include #include @@ -16,6 +19,7 @@ #include #include +#include #include #include // swap @@ -39,9 +43,6 @@ #endif -extern "C" status_t slab_init(); - - static const int kMagazineCapacity = 32; static const size_t kCacheColorPeriod = 8; @@ -155,11 +156,13 @@ struct depot_cpu_store { }; -static object_cache *sSlabCache, *sLinkCache; static ObjectCacheList sObjectCaches; static benaphore sObjectCacheListLock; +static uint8 *sInitialBegin, *sInitialLimit, *sInitialPointer; + +static status_t object_depot_init_locks(object_depot *depot); static depot_magazine *alloc_magazine(); static void free_magazine(depot_magazine *magazine); @@ -215,13 +218,57 @@ __fls0(size_t value) } +static void * +internal_alloc(size_t size, uint32 flags) +{ + if (flags & CACHE_DURING_BOOT) { + if ((sInitialPointer + size) > sInitialLimit) + panic("internal_alloc: ran out of initial space"); + + uint8 *buffer = sInitialPointer; + sInitialPointer += size; + return buffer; + } + + return block_alloc(size); +} + + +static void +internal_free(void *_buffer) +{ + uint8 *buffer = (uint8 *)_buffer; + + if (buffer >= sInitialBegin && buffer < sInitialLimit) + return; + + block_free(buffer); +} + + +static status_t +benaphore_boot_init(benaphore *lock, const char *name, uint32 flags) +{ + if (flags & CACHE_DURING_BOOT) { + lock->sem = -1; + lock->count = 0; + return B_OK; + } + + return benaphore_init(lock, name); +} + + static status_t allocate_pages(object_cache *cache, void **pages, uint32 flags) { TRACE_CACHE(cache, "allocate pages (%lu, 0x0%lx)", cache->slab_size, flags); + // if we are allocating, it is because we need the pages immediatly + // so we lock them. when moving the slab to the empty list we should + // unlock them, and lock them again when getting one from the empty list. area_id areaId = create_area(cache->name, pages, B_ANY_KERNEL_ADDRESS, - cache->slab_size, B_NO_LOCK, B_READ_AREA | B_WRITE_AREA); + cache->slab_size, B_FULL_LOCK, B_READ_AREA | B_WRITE_AREA); if (areaId < 0) return areaId; @@ -314,12 +361,12 @@ object_cache_init(object_cache *cache, const char *name, size_t objectSize, object_cache_constructor constructor, object_cache_destructor destructor, object_cache_reclaimer reclaimer) { - status_t status = benaphore_init(&cache->lock, name); + strlcpy(cache->name, name, sizeof(cache->name)); + + status_t status = benaphore_boot_init(&cache->lock, name, flags); if (status < B_OK) return status; - strlcpy(cache->name, name, sizeof(cache->name)); - if (objectSize < sizeof(object_link)) objectSize = sizeof(object_link); @@ -347,7 +394,7 @@ object_cache_init(object_cache *cache, const char *name, size_t objectSize, cache->flags |= CACHE_NO_DEPOT; if (!(cache->flags & CACHE_NO_DEPOT)) { - status_t status = object_depot_init(&cache->depot, + status_t status = object_depot_init(&cache->depot, flags, object_cache_return_object_wrapper); if (status < B_OK) { benaphore_destroy(&cache->lock); @@ -369,19 +416,43 @@ object_cache_init(object_cache *cache, const char *name, size_t objectSize, } +static status_t +object_cache_init_locks(object_cache *cache) +{ + status_t status = benaphore_init(&cache->lock, cache->name); + if (status < B_OK) + return status; + + if (cache->flags & CACHE_NO_DEPOT) + return B_OK; + + return object_depot_init_locks(&cache->depot); +} + + +static void +delete_cache(object_cache *cache) +{ + cache->~object_cache(); + internal_free(cache); +} + + static SmallObjectCache * create_small_object_cache(const char *name, size_t object_size, size_t alignment, size_t maximum, uint32 flags, void *cookie, object_cache_constructor constructor, object_cache_destructor destructor, object_cache_reclaimer reclaimer) { - SmallObjectCache *cache = new (std::nothrow) SmallObjectCache(); - if (cache == NULL) + void *buffer = internal_alloc(sizeof(SmallObjectCache), flags); + if (buffer == NULL) return NULL; + SmallObjectCache *cache = new (buffer) SmallObjectCache(); + if (object_cache_init(cache, name, object_size, alignment, maximum, flags, cookie, constructor, destructor, reclaimer) < B_OK) { - delete cache; + delete_cache(cache); return NULL; } @@ -397,13 +468,15 @@ create_hashed_object_cache(const char *name, size_t object_size, object_cache_constructor constructor, object_cache_destructor destructor, object_cache_reclaimer reclaimer) { - HashedObjectCache *cache = new (std::nothrow) HashedObjectCache(); - if (cache == NULL) + void *buffer = internal_alloc(sizeof(HashedObjectCache), flags); + if (buffer == NULL) return NULL; + HashedObjectCache *cache = new (buffer) HashedObjectCache(); + if (object_cache_init(cache, name, object_size, alignment, maximum, flags, cookie, constructor, destructor, reclaimer) < B_OK) { - delete cache; + delete_cache(cache); return NULL; } @@ -466,7 +539,7 @@ delete_object_cache(object_cache *cache) cache->ReturnSlab(cache->empty.RemoveHead()); benaphore_destroy(&cache->lock); - delete cache; + delete_cache(cache); } @@ -504,8 +577,8 @@ object_cache_alloc(object_cache *cache, uint32 flags) source->count--; cache->used_count++; - TRACE_CACHE(cache, "allocate %p from %p, %lu remaining.", link, source, - source->count); + TRACE_CACHE(cache, "allocate %p (%p) from %p, %lu remaining.", + link_to_object(link, cache->object_size), link, source, source->count); if (source->count == 0) { cache->partial.Remove(source); @@ -524,8 +597,9 @@ object_cache_return_to_slab(object_cache *cache, slab *source, void *object) object_link *link = object_to_link(object, cache->object_size); - TRACE_CACHE(cache, "returning %p to %p, %lu used (%lu empty slabs).", - link, source, source->size - source->count, cache->empty_count); + TRACE_CACHE(cache, "returning %p (%p) to %p, %lu used (%lu empty slabs).", + object, link, source, source->size - source->count, + cache->empty_count); _push(source->free, link); source->count++; @@ -693,28 +767,29 @@ SmallObjectCache::ObjectSlab(void *object) const static slab * allocate_slab(uint32 flags) { - return (slab *)object_cache_alloc(sSlabCache, flags); + return (slab *)internal_alloc(sizeof(slab), flags); } static void free_slab(slab *slab) { - object_cache_free(sSlabCache, slab); + internal_free(slab); } static HashedObjectCache::Link * allocate_link(uint32 flags) { - return (HashedObjectCache::Link *)object_cache_alloc(sLinkCache, flags); + return (HashedObjectCache::Link *) + internal_alloc(sizeof(HashedObjectCache::Link), flags); } static void free_link(HashedObjectCache::Link *link) { - object_cache_free(sLinkCache, link); + internal_free(link); } @@ -863,8 +938,8 @@ exchange_with_empty(object_depot *depot, depot_magazine* &magazine) static depot_magazine * alloc_magazine() { - depot_magazine *magazine = (depot_magazine *)malloc(sizeof(depot_magazine) - + kMagazineCapacity * sizeof(void *)); + depot_magazine *magazine = (depot_magazine *)internal_alloc( + sizeof(depot_magazine) + kMagazineCapacity * sizeof(void *), 0); if (magazine) { magazine->next = NULL; magazine->current_round = 0; @@ -878,7 +953,7 @@ alloc_magazine() static void free_magazine(depot_magazine *magazine) { - free(magazine); + internal_free(magazine); } @@ -892,25 +967,26 @@ empty_magazine(object_depot *depot, depot_magazine *magazine) status_t -object_depot_init(object_depot *depot, +object_depot_init(object_depot *depot, uint32 flags, void (*return_object)(object_depot *depot, void *object)) { depot->full = NULL; depot->empty = NULL; depot->full_count = depot->empty_count = 0; - status_t status = benaphore_init(&depot->lock, "depot"); + status_t status = benaphore_boot_init(&depot->lock, "depot", flags); if (status < B_OK) return status; - depot->stores = new (std::nothrow) depot_cpu_store[smp_get_num_cpus()]; + depot->stores = (depot_cpu_store *)internal_alloc(sizeof(depot_cpu_store) + * smp_get_num_cpus(), flags); if (depot->stores == NULL) { benaphore_destroy(&depot->lock); return B_NO_MEMORY; } for (int i = 0; i < smp_get_num_cpus(); i++) { - benaphore_init(&depot->stores[i].lock, "cpu store"); + benaphore_boot_init(&depot->stores[i].lock, "cpu store", flags); depot->stores[i].loaded = depot->stores[i].previous = NULL; } @@ -920,6 +996,23 @@ object_depot_init(object_depot *depot, } +status_t +object_depot_init_locks(object_depot *depot) +{ + status_t status = benaphore_init(&depot->lock, "depot"); + if (status < B_OK) + return status; + + for (int i = 0; i < smp_get_num_cpus(); i++) { + status = benaphore_init(&depot->stores[i].lock, "cpu store"); + if (status < B_OK) + return status; + } + + return B_OK; +} + + void object_depot_destroy(object_depot *depot) { @@ -929,7 +1022,7 @@ object_depot_destroy(object_depot *depot) benaphore_destroy(&depot->stores[i].lock); } - delete [] depot->stores; + internal_free(depot->stores); benaphore_destroy(&depot->lock); } @@ -1030,44 +1123,83 @@ object_depot_make_empty(object_depot *depot) static int dump_slabs(int argc, char *argv[]) { - kprintf("%10s %32s %8s %8s %6s %8s %6s\n", "address", "name", "objsize", "usage", - "empty", "usedobj", "flags"); + kprintf("%10s %22s %8s %8s %6s %8s %8s %8s\n", "address", "name", + "objsize", "usage", "empty", "usedobj", "total", "flags"); ObjectCacheList::Iterator it = sObjectCaches.GetIterator(); while (it.HasNext()) { object_cache *cache = it.Next(); - kprintf("%p %32s %8lu %8lu %6lu %8lu %6lx\n", cache, cache->name, + kprintf("%p %22s %8lu %8lu %6lu %8lu %8lu %8lx\n", cache, cache->name, cache->object_size, cache->usage, cache->empty_count, - cache->used_count, cache->flags); + cache->used_count, cache->usage / cache->object_size, + cache->flags); } return 0; } -status_t -slab_init() +static int +dump_cache_info(int argc, char *argv[]) +{ + if (argc < 2) { + kprintf("usage: cache_info [address]\n"); + return 0; + } + + object_cache *cache = (object_cache *)strtoul(argv[1], NULL, 16); + + kprintf("name: %s\n", cache->name); + kprintf("lock: { count: %i, sem: %ld }\n", cache->lock.count, + cache->lock.sem); + kprintf("object_size: %lu\n", cache->object_size); + kprintf("cache_color_cycle: %lu\n", cache->cache_color_cycle); + kprintf("used_count: %lu\n", cache->used_count); + kprintf("empty_count: %lu\n", cache->empty_count); + kprintf("pressure: %lu\n", cache->pressure); + kprintf("slab_size: %lu\n", cache->slab_size); + kprintf("usage: %lu\n", cache->usage); + kprintf("maximum: %lu\n", cache->maximum); + kprintf("flags: 0x%lx\n", cache->flags); + kprintf("cookie: %p\n", cache->cookie); + + return 0; +} + + +void +slab_init(addr_t initialBase, size_t initialSize) +{ + sInitialBegin = (uint8 *)initialBase; + sInitialLimit = sInitialBegin + initialSize; + sInitialPointer = sInitialBegin; + + new (&sObjectCaches) ObjectCacheList(); + + block_allocator_init_boot(); + + add_debugger_command("slabs", dump_slabs, "list all object caches"); + add_debugger_command("cache_info", dump_cache_info, + "dump information about a specific cache"); +} + + +void +slab_init_post_sem() { status_t status = benaphore_init(&sObjectCacheListLock, "object cache list"); if (status < B_OK) panic("slab_init: failed to create object cache list lock"); - new (&sObjectCaches) ObjectCacheList(); + ObjectCacheList::Iterator it = sObjectCaches.GetIterator(); - sSlabCache = create_object_cache("slab cache", sizeof(slab), 4, NULL, NULL, - NULL); - if (sSlabCache == NULL) - panic("slab_init: failed to create slab cache"); + while (it.HasNext()) { + if (object_cache_init_locks(it.Next()) < B_OK) + panic("slab_init: failed to create sems"); + } - sLinkCache = create_object_cache("link cache", - sizeof(HashedObjectCache::Link), 4, NULL, NULL, NULL); - if (sLinkCache == NULL) - panic("slab_init: failed to create link cache"); - - add_debugger_command("slabs", dump_slabs, "list all object caches"); - - return B_OK; + block_allocator_init_rest(); } diff --git a/src/system/kernel/slab/allocator.cpp b/src/system/kernel/slab/allocator.cpp new file mode 100644 index 0000000000..6647bc2faa --- /dev/null +++ b/src/system/kernel/slab/allocator.cpp @@ -0,0 +1,139 @@ +/* + * Copyright 2007, Hugo Santos. All Rights Reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Hugo Santos, hugosantos@gmail.com + */ + +#include + +#include "slab_private.h" + +#include + +#define DEBUG_ALLOCATOR + +static const size_t kBlockSizes[] = { + 16, 24, 32, 48, 64, 80, 96, 112, + 128, 160, 192, 224, 256, 320, 384, 448, + 512, 640, 768, 896, 1024, 1280, 1536, 1792, + 2048, 2560, 3072, 3584, 4096, 4608, 5120, 5632, + 6144, 6656, 7168, 7680, 8192, + 0 +}; + +static object_cache *sBlockCaches[sizeof(kBlockSizes) / sizeof(size_t)]; + +struct boundary_tag { + uint32 size; +#ifdef DEBUG_ALLOCATOR + uint32 magic; +#endif +}; + +static const uint32 kBoundaryMagic = 0x6da78d13; + + +static object_cache * +size_to_cache(size_t size) +{ + if (size <= 128) + return sBlockCaches[size / 16]; + else if (size <= 256) + return sBlockCaches[8 + (size - 128) / 32]; + else if (size <= 512) + return sBlockCaches[12 + (size - 256) / 64]; + else if (size <= 1024) + return sBlockCaches[16 + (size - 512) / 128]; + else if (size <= 2048) + return sBlockCaches[20 + (size - 1024) / 256]; + else if (size <= 8192) + return sBlockCaches[24 + (size - 2048) / 512]; + + return NULL; +} + + +void * +block_alloc(size_t size) +{ + object_cache *cache = size_to_cache(size + sizeof(boundary_tag)); + + void *block; + + if (cache) { + block = object_cache_alloc(cache, 0); + } else { + // TODO create areas + panic("allocator: unimplemented"); + return NULL; + } + + boundary_tag *tag = (boundary_tag *)block; + + tag->size = size; + +#ifdef DEBUG_ALLOCATOR + tag->magic = kBoundaryMagic; +#endif + + return ((uint8 *)block) + sizeof(boundary_tag); +} + + +void +block_free(void *block) +{ + boundary_tag *tag = (boundary_tag *)(((uint8 *)block) + - sizeof(boundary_tag)); + +#ifdef DEBUG_ALLOCATOR + if (tag->magic != kBoundaryMagic) + panic("allocator: boundary tag magic doesn't match this universe"); +#endif + + object_cache *cache = size_to_cache(tag->size); + if (cache == NULL) + panic("allocator: unimplemented"); + + object_cache_free(cache, tag); +} + + +static void +block_create_cache(size_t index, bool boot) +{ + char name[32]; + snprintf(name, sizeof(name), "block cache: %lu", kBlockSizes[index]); + + sBlockCaches[index] = create_object_cache_etc(name, kBlockSizes[index], + 0, 0, boot ? CACHE_DURING_BOOT : 0, NULL, NULL, NULL, NULL); + if (sBlockCaches[index] == NULL) + panic("allocator: failed to init block cache"); +} + + +void +block_allocator_init_boot() +{ + for (int index = 0; kBlockSizes[index] != 0; index++) { + if (kBlockSizes[index] > 256) + break; + + block_create_cache(index, true); + } +} + + +void +block_allocator_init_rest() +{ + for (int index = 0; kBlockSizes[index] != 0; index++) { + if (kBlockSizes[index] <= 256) + continue; + + block_create_cache(index, false); + } +} + diff --git a/src/system/kernel/slab/slab_private.h b/src/system/kernel/slab/slab_private.h new file mode 100644 index 0000000000..ed33f51731 --- /dev/null +++ b/src/system/kernel/slab/slab_private.h @@ -0,0 +1,21 @@ +/* + * Copyright 2007, Hugo Santos. All Rights Reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Hugo Santos, hugosantos@gmail.com + */ + + +#ifndef _SLAB_PRIVATE_H_ +#define _SLAB_PRIVATE_H_ + +extern "C" { + void *block_alloc(size_t size); + void block_free(void *block); + void block_allocator_init_boot(); + void block_allocator_init_rest(); +} + +#endif + diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index cbd50626f5..ff7118a892 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -2891,6 +2891,11 @@ vm_init(kernel_args *args) TRACE(("heap at 0x%lx\n", heapBase)); heap_init(heapBase, heapSize); + size_t slabInitialSize = 2 * B_PAGE_SIZE; + addr_t slabInitialBase = vm_allocate_early(args, slabInitialSize, + slabInitialSize, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + slab_init(slabInitialBase, slabInitialSize); + // initialize the free page list and physical page mapper vm_page_init(args); @@ -2919,6 +2924,11 @@ vm_init(kernel_args *args) create_area("kernel heap", &address, B_EXACT_ADDRESS, heapSize, B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + address = (void *)ROUNDOWN(slabInitialBase, B_PAGE_SIZE); + create_area("initial slab space", &address, B_EXACT_ADDRESS, + slabInitialSize, B_ALREADY_WIRED, B_KERNEL_READ_AREA + | B_KERNEL_WRITE_AREA); + allocate_kernel_args(args); args->kernel_image.name = "kernel"; @@ -2985,6 +2995,8 @@ vm_init_post_sem(kernel_args *args) sAreaHashLock = create_sem(WRITE_COUNT, "area hash"); + slab_init_post_sem(); + return heap_init_post_sem(args); }