From df7096a4aaa8076f7ebbc617f914700c9a5c0109 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Wed, 18 Jun 2008 19:55:51 +0000 Subject: [PATCH] * Added malloc_nogrow() function that does allocation without triggering or waiting for a heap grow. * Use that nogrow version in the VM code to avoid a deadlock with the address space lock when a grow operation would try to create an area while a malloc happened from such a function in the VM. * When waiting for a grow to happen, notify the waiting thread from the grower also if it failed to allocate a new heap. Otherwise a thread would just sit there and wait until another thread requested growing too and that one succeeded (or just forever in the worst case). * Make the dedicated grow heap growable too. If the current grow heaps run low on memory it will instruct the grower to allocate a new grow heap. This reduces the likelyhood of running out of memory with no way to grow to a minimum. As the growing is done asynchronously it is still possible to happen, but it is highly unlikely as the grow heap is solely used to allocate memory in the process of creating new heap areas and it will even try using normal public memory if the dedicated memory has run out. * Reduced the dedicated grow heap from 2 to 1MB. As it can now grow itself, it doesn't need to last so long. * Extract heap creation into it's own function that does area creation and heap attach and use this function for growing normal and grow heaps. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@26009 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/private/kernel/heap.h | 8 +- src/system/kernel/heap.cpp | 164 ++++++++++++------ src/system/kernel/vm/vm.cpp | 8 +- src/system/kernel/vm/vm_address_space.cpp | 3 +- src/system/kernel/vm/vm_cache.cpp | 3 +- .../kernel/vm/vm_store_anonymous_noswap.cpp | 3 +- src/system/kernel/vm/vm_store_device.c | 3 +- src/system/kernel/vm/vm_store_null.c | 3 +- 8 files changed, 135 insertions(+), 60 deletions(-) diff --git a/headers/private/kernel/heap.h b/headers/private/kernel/heap.h index 329fc650e1..cca812ffcc 100644 --- a/headers/private/kernel/heap.h +++ b/headers/private/kernel/heap.h @@ -14,14 +14,18 @@ #define INITIAL_HEAP_SIZE 16 * 1024 * 1024 // grow by another 8MB each time the heap runs out of memory #define HEAP_GROW_SIZE 8 * 1024 * 1024 -// allocate a dedicated 2MB area for dynamic growing -#define HEAP_DEDICATED_GROW_SIZE 2 * 1024 * 1024 +// allocate a dedicated 1MB area for dynamic growing +#define HEAP_DEDICATED_GROW_SIZE 1 * 1024 * 1024 #ifdef __cplusplus extern "C" { #endif +// malloc_nogrow disallows waiting for a grow to happen - only to be used by +// vm functions that may deadlock on a triggered area creation +void *malloc_nogrow(size_t size); + void *memalign(size_t alignment, size_t size); void deferred_free(void* block); diff --git a/src/system/kernel/heap.cpp b/src/system/kernel/heap.cpp index 52c66188b5..fe1b68d310 100644 --- a/src/system/kernel/heap.cpp +++ b/src/system/kernel/heap.cpp @@ -101,10 +101,11 @@ typedef DoublyLinkedList DeferredFreeList; static heap_allocator *sHeapList = NULL; static heap_allocator *sLastGrowRequest = NULL; -static heap_allocator *sGrowHeap = NULL; +static heap_allocator *sGrowHeapList = NULL; static thread_id sHeapGrowThread = -1; static sem_id sHeapGrowSem = -1; static sem_id sHeapGrownNotify = -1; +static bool sAddGrowHeap = false; static DeferredFreeList sDeferredFreeList; static spinlock sDeferredFreeListLock; @@ -257,8 +258,12 @@ dump_heap_list(int argc, char **argv) if (argc == 2) { if (strcmp(argv[1], "grow") == 0) { // only dump dedicated grow heap info - dprintf("dedicated grow heap:\n"); - dump_allocator(sGrowHeap); + dprintf("dedicated grow heap(s):\n"); + heap_allocator *heap = sGrowHeapList; + while (heap) { + dump_allocator(heap); + heap = heap->next; + } } else if (strcmp(argv[1], "stats") == 0) { uint32 heapCount = 0; heap_allocator *heap = sHeapList; @@ -1177,15 +1182,61 @@ deferred_deleter(void *arg, int iteration) // #pragma mark - +static heap_allocator * +heap_create_new_heap(const char *name, size_t size) +{ + void *heapAddress = NULL; + area_id heapArea = create_area(name, &heapAddress, + B_ANY_KERNEL_BLOCK_ADDRESS, size, B_FULL_LOCK, + B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + if (heapArea < B_OK) { + TRACE(("heap: couldn't allocate heap \"%s\"\n", name)); + return NULL; + } + + heap_allocator *newHeap = heap_attach((addr_t)heapAddress, size); + if (newHeap == NULL) { + panic("heap: could not attach heap to area!\n"); + delete_area(heapArea); + return NULL; + } + +#if PARANOID_VALIDATION + heap_validate_heap(newHeap); +#endif + return newHeap; +} + + static int32 heap_grow_thread(void *) { heap_allocator *heap = sHeapList; + heap_allocator *growHeap = sGrowHeapList; while (true) { // wait for a request to grow the heap list if (acquire_sem(sHeapGrowSem) < B_OK) continue; + if (sAddGrowHeap) { + while (growHeap->next) + growHeap = growHeap->next; + + // the last grow heap is going to run full soon, try to allocate + // a new one to make some room. + TRACE(("heap_grower: grow heaps will run out of memory soon\n")); + heap_allocator *newHeap = heap_create_new_heap( + "additional grow heap", HEAP_DEDICATED_GROW_SIZE); + if (newHeap != NULL) { +#if PARANOID_VALIDATION + heap_validate_heap(newHeap); +#endif + growHeap->next = newHeap; + sAddGrowHeap = false; + TRACE(("heap_grower: new grow heap %p linked in\n", newHeap)); + } + } + // find the last heap while (heap->next) heap = heap->next; @@ -1196,28 +1247,15 @@ heap_grow_thread(void *) } TRACE(("heap_grower: kernel heap will run out of memory soon, allocating new one\n")); - void *heapAddress = NULL; - area_id heapArea = create_area("additional heap", &heapAddress, - B_ANY_KERNEL_BLOCK_ADDRESS, HEAP_GROW_SIZE, B_FULL_LOCK, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - if (heapArea < B_OK) { - panic("heap_grower: couldn't allocate additional heap area\n"); - continue; - } - - heap_allocator *newHeap = heap_attach((addr_t)heapAddress, + heap_allocator *newHeap = heap_create_new_heap("additional heap", HEAP_GROW_SIZE); - if (newHeap == NULL) { - panic("heap_grower: could not attach additional heap!\n"); - delete_area(heapArea); - continue; - } - + if (newHeap != NULL) { #if PARANOID_VALIDATION - heap_validate_heap(newHeap); + heap_validate_heap(newHeap); #endif - heap->next = newHeap; - TRACE(("heap_grower: new heap linked in\n")); + heap->next = newHeap; + TRACE(("heap_grower: new heap linked in\n")); + } // notify anyone waiting for this request release_sem_etc(sHeapGrownNotify, -1, B_RELEASE_ALL); @@ -1282,19 +1320,10 @@ heap_init_post_sem() status_t heap_init_post_thread() { - void *dedicated = NULL; - area_id area = create_area("heap dedicated grow", &dedicated, - B_ANY_KERNEL_BLOCK_ADDRESS, HEAP_DEDICATED_GROW_SIZE, B_FULL_LOCK, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - if (area < 0) { - panic("heap_init_post_thread(): cannot allocate dedicated grow memory\n"); - return area; - } - - sGrowHeap = heap_attach((addr_t)dedicated, HEAP_DEDICATED_GROW_SIZE); - if (sGrowHeap == NULL) { + sGrowHeapList = heap_create_new_heap("dedicated grow heap", + HEAP_DEDICATED_GROW_SIZE); + if (sGrowHeapList == NULL) { panic("heap_init_post_thread(): failed to attach dedicated grow heap\n"); - delete_area(area); return B_ERROR; } @@ -1302,7 +1331,6 @@ heap_init_post_thread() B_URGENT_PRIORITY, NULL); if (sHeapGrowThread < 0) { panic("heap_init_post_thread(): cannot create heap grow thread\n"); - delete_area(area); return sHeapGrowThread; } @@ -1331,17 +1359,6 @@ memalign(size_t alignment, size_t size) return NULL; } - if (thread_get_current_thread_id() == sHeapGrowThread) { - // this is the grower thread, allocate from our dedicated memory - void *result = heap_memalign(sGrowHeap, alignment, size, NULL); - if (result == NULL) { - panic("heap: grow thread ran out of dedicated memory!\n"); - return NULL; - } - - return result; - } - heap_allocator *heap = sHeapList; while (heap) { bool shouldGrow = false; @@ -1352,6 +1369,10 @@ memalign(size_t alignment, size_t size) if (result == NULL) { // urgent request, do the request and wait switch_sem(sHeapGrowSem, sHeapGrownNotify); + if (heap->next == NULL) { + // the grower didn't manage to add a new heap + return NULL; + } } else { // not so urgent, just notify the grower release_sem_etc(sHeapGrowSem, 1, B_DO_NOT_RESCHEDULE); @@ -1375,6 +1396,46 @@ memalign(size_t alignment, size_t size) } +void * +malloc_nogrow(size_t size) +{ + // use dedicated memory in the grow thread by default + if (thread_get_current_thread_id() == sHeapGrowThread) { + bool shouldGrow = false; + heap_allocator *heap = sGrowHeapList; + while (heap) { + void *result = heap_memalign(heap, 0, size, &shouldGrow); + if (shouldGrow && heap->next == NULL && !sAddGrowHeap) { + // hopefully the heap grower will manage to create a new heap + // before running out of private memory... + dprintf("heap: requesting new grow heap\n"); + sAddGrowHeap = true; + release_sem_etc(sHeapGrowSem, 1, B_DO_NOT_RESCHEDULE); + } + + if (result != NULL) + return result; + + heap = heap->next; + } + } + + // try public memory, there might be something available + heap_allocator *heap = sHeapList; + while (heap) { + void *result = heap_memalign(heap, 0, size, NULL); + if (result != NULL) + return result; + + heap = heap->next; + } + + // no memory available + panic("heap: all heaps have run out of memory\n"); + return NULL; +} + + void * malloc(size_t size) { @@ -1402,9 +1463,14 @@ free(void *address) heap = heap->next; } - // maybe it was allocated from the dedicated grow heap - if (heap_free(sGrowHeap, address) == B_OK) - return; + // maybe it was allocated from a dedicated grow heap + heap = sGrowHeapList; + while (heap) { + if (heap_free(heap, address) == B_OK) + return; + + heap = heap->next; + } panic("free(): free failed for address %p\n", address); } diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index 3c2bb19daa..d1641e9f95 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -810,7 +810,7 @@ lookup_area(vm_address_space* addressSpace, area_id id) static vm_area * create_reserved_area_struct(vm_address_space *addressSpace, uint32 flags) { - vm_area *reserved = (vm_area *)malloc(sizeof(vm_area)); + vm_area *reserved = (vm_area *)malloc_nogrow(sizeof(vm_area)); if (reserved == NULL) return NULL; @@ -833,11 +833,11 @@ create_area_struct(vm_address_space *addressSpace, const char *name, if (length > B_OS_NAME_LENGTH) length = B_OS_NAME_LENGTH; - vm_area *area = (vm_area *)malloc(sizeof(vm_area)); + vm_area *area = (vm_area *)malloc_nogrow(sizeof(vm_area)); if (area == NULL) return NULL; - area->name = (char *)malloc(length); + area->name = (char *)malloc_nogrow(length); if (area->name == NULL) { free(area); return NULL; @@ -2856,7 +2856,7 @@ vm_map_page(vm_area *area, vm_page *page, addr_t address, uint32 protection) vm_page_mapping *mapping = NULL; if (area->wiring == B_NO_LOCK) { - mapping = (vm_page_mapping *)malloc(sizeof(vm_page_mapping)); + mapping = (vm_page_mapping *)malloc_nogrow(sizeof(vm_page_mapping)); if (mapping == NULL) return B_NO_MEMORY; diff --git a/src/system/kernel/vm/vm_address_space.cpp b/src/system/kernel/vm/vm_address_space.cpp index 91a7d708d1..c01bb3fba2 100644 --- a/src/system/kernel/vm/vm_address_space.cpp +++ b/src/system/kernel/vm/vm_address_space.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -263,7 +264,7 @@ vm_create_address_space(team_id id, addr_t base, addr_t size, vm_address_space *addressSpace; status_t status; - addressSpace = (vm_address_space *)malloc(sizeof(vm_address_space)); + addressSpace = (vm_address_space *)malloc_nogrow(sizeof(vm_address_space)); if (addressSpace == NULL) return B_NO_MEMORY; diff --git a/src/system/kernel/vm/vm_cache.cpp b/src/system/kernel/vm/vm_cache.cpp index 455a33c9b6..caf33ac107 100644 --- a/src/system/kernel/vm/vm_cache.cpp +++ b/src/system/kernel/vm/vm_cache.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -543,7 +544,7 @@ vm_cache_create(vm_store* store) return NULL; } - cache = (vm_cache*)malloc(sizeof(vm_cache)); + cache = (vm_cache*)malloc_nogrow(sizeof(vm_cache)); if (cache == NULL) return NULL; diff --git a/src/system/kernel/vm/vm_store_anonymous_noswap.cpp b/src/system/kernel/vm/vm_store_anonymous_noswap.cpp index dcee40bfab..31132c3ac1 100644 --- a/src/system/kernel/vm/vm_store_anonymous_noswap.cpp +++ b/src/system/kernel/vm/vm_store_anonymous_noswap.cpp @@ -9,6 +9,7 @@ #include "vm_store_anonymous_noswap.h" +#include #include #include #include @@ -165,7 +166,7 @@ vm_store * vm_store_create_anonymous_noswap(bool canOvercommit, int32 numPrecommittedPages, int32 numGuardPages) { - anonymous_store *store = (anonymous_store *)malloc( + anonymous_store *store = (anonymous_store *)malloc_nogrow( sizeof(anonymous_store)); if (store == NULL) return NULL; diff --git a/src/system/kernel/vm/vm_store_device.c b/src/system/kernel/vm/vm_store_device.c index 92b7433993..e771a89597 100644 --- a/src/system/kernel/vm/vm_store_device.c +++ b/src/system/kernel/vm/vm_store_device.c @@ -9,6 +9,7 @@ #include "vm_store_device.h" +#include #include #include @@ -87,7 +88,7 @@ static vm_store_ops device_ops = { struct vm_store * vm_store_create_device(addr_t baseAddress) { - struct device_store *store = malloc(sizeof(struct device_store)); + struct device_store *store = malloc_nogrow(sizeof(struct device_store)); if (store == NULL) return NULL; diff --git a/src/system/kernel/vm/vm_store_null.c b/src/system/kernel/vm/vm_store_null.c index 90ef4d040c..8acf56320e 100644 --- a/src/system/kernel/vm/vm_store_null.c +++ b/src/system/kernel/vm/vm_store_null.c @@ -9,6 +9,7 @@ #include "vm_store_null.h" +#include #include @@ -78,7 +79,7 @@ vm_store_create_null(void) { struct vm_store *store; - store = malloc(sizeof(struct vm_store)); + store = malloc_nogrow(sizeof(struct vm_store)); if (store == NULL) return NULL;