diff --git a/headers/private/kernel/heap.h b/headers/private/kernel/heap.h index 329fc650e1..cca812ffcc 100644 --- a/headers/private/kernel/heap.h +++ b/headers/private/kernel/heap.h @@ -14,14 +14,18 @@ #define INITIAL_HEAP_SIZE 16 * 1024 * 1024 // grow by another 8MB each time the heap runs out of memory #define HEAP_GROW_SIZE 8 * 1024 * 1024 -// allocate a dedicated 2MB area for dynamic growing -#define HEAP_DEDICATED_GROW_SIZE 2 * 1024 * 1024 +// allocate a dedicated 1MB area for dynamic growing +#define HEAP_DEDICATED_GROW_SIZE 1 * 1024 * 1024 #ifdef __cplusplus extern "C" { #endif +// malloc_nogrow disallows waiting for a grow to happen - only to be used by +// vm functions that may deadlock on a triggered area creation +void *malloc_nogrow(size_t size); + void *memalign(size_t alignment, size_t size); void deferred_free(void* block); diff --git a/src/system/kernel/heap.cpp b/src/system/kernel/heap.cpp index 52c66188b5..fe1b68d310 100644 --- a/src/system/kernel/heap.cpp +++ b/src/system/kernel/heap.cpp @@ -101,10 +101,11 @@ typedef DoublyLinkedList DeferredFreeList; static heap_allocator *sHeapList = NULL; static heap_allocator *sLastGrowRequest = NULL; -static heap_allocator *sGrowHeap = NULL; +static heap_allocator *sGrowHeapList = NULL; static thread_id sHeapGrowThread = -1; static sem_id sHeapGrowSem = -1; static sem_id sHeapGrownNotify = -1; +static bool sAddGrowHeap = false; static DeferredFreeList sDeferredFreeList; static spinlock sDeferredFreeListLock; @@ -257,8 +258,12 @@ dump_heap_list(int argc, char **argv) if (argc == 2) { if (strcmp(argv[1], "grow") == 0) { // only dump dedicated grow heap info - dprintf("dedicated grow heap:\n"); - dump_allocator(sGrowHeap); + dprintf("dedicated grow heap(s):\n"); + heap_allocator *heap = sGrowHeapList; + while (heap) { + dump_allocator(heap); + heap = heap->next; + } } else if (strcmp(argv[1], "stats") == 0) { uint32 heapCount = 0; heap_allocator *heap = sHeapList; @@ -1177,15 +1182,61 @@ deferred_deleter(void *arg, int iteration) // #pragma mark - +static heap_allocator * +heap_create_new_heap(const char *name, size_t size) +{ + void *heapAddress = NULL; + area_id heapArea = create_area(name, &heapAddress, + B_ANY_KERNEL_BLOCK_ADDRESS, size, B_FULL_LOCK, + B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + if (heapArea < B_OK) { + TRACE(("heap: couldn't allocate heap \"%s\"\n", name)); + return NULL; + } + + heap_allocator *newHeap = heap_attach((addr_t)heapAddress, size); + if (newHeap == NULL) { + panic("heap: could not attach heap to area!\n"); + delete_area(heapArea); + return NULL; + } + +#if PARANOID_VALIDATION + heap_validate_heap(newHeap); +#endif + return newHeap; +} + + static int32 heap_grow_thread(void *) { heap_allocator *heap = sHeapList; + heap_allocator *growHeap = sGrowHeapList; while (true) { // wait for a request to grow the heap list if (acquire_sem(sHeapGrowSem) < B_OK) continue; + if (sAddGrowHeap) { + while (growHeap->next) + growHeap = growHeap->next; + + // the last grow heap is going to run full soon, try to allocate + // a new one to make some room. + TRACE(("heap_grower: grow heaps will run out of memory soon\n")); + heap_allocator *newHeap = heap_create_new_heap( + "additional grow heap", HEAP_DEDICATED_GROW_SIZE); + if (newHeap != NULL) { +#if PARANOID_VALIDATION + heap_validate_heap(newHeap); +#endif + growHeap->next = newHeap; + sAddGrowHeap = false; + TRACE(("heap_grower: new grow heap %p linked in\n", newHeap)); + } + } + // find the last heap while (heap->next) heap = heap->next; @@ -1196,28 +1247,15 @@ heap_grow_thread(void *) } TRACE(("heap_grower: kernel heap will run out of memory soon, allocating new one\n")); - void *heapAddress = NULL; - area_id heapArea = create_area("additional heap", &heapAddress, - B_ANY_KERNEL_BLOCK_ADDRESS, HEAP_GROW_SIZE, B_FULL_LOCK, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - if (heapArea < B_OK) { - panic("heap_grower: couldn't allocate additional heap area\n"); - continue; - } - - heap_allocator *newHeap = heap_attach((addr_t)heapAddress, + heap_allocator *newHeap = heap_create_new_heap("additional heap", HEAP_GROW_SIZE); - if (newHeap == NULL) { - panic("heap_grower: could not attach additional heap!\n"); - delete_area(heapArea); - continue; - } - + if (newHeap != NULL) { #if PARANOID_VALIDATION - heap_validate_heap(newHeap); + heap_validate_heap(newHeap); #endif - heap->next = newHeap; - TRACE(("heap_grower: new heap linked in\n")); + heap->next = newHeap; + TRACE(("heap_grower: new heap linked in\n")); + } // notify anyone waiting for this request release_sem_etc(sHeapGrownNotify, -1, B_RELEASE_ALL); @@ -1282,19 +1320,10 @@ heap_init_post_sem() status_t heap_init_post_thread() { - void *dedicated = NULL; - area_id area = create_area("heap dedicated grow", &dedicated, - B_ANY_KERNEL_BLOCK_ADDRESS, HEAP_DEDICATED_GROW_SIZE, B_FULL_LOCK, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - if (area < 0) { - panic("heap_init_post_thread(): cannot allocate dedicated grow memory\n"); - return area; - } - - sGrowHeap = heap_attach((addr_t)dedicated, HEAP_DEDICATED_GROW_SIZE); - if (sGrowHeap == NULL) { + sGrowHeapList = heap_create_new_heap("dedicated grow heap", + HEAP_DEDICATED_GROW_SIZE); + if (sGrowHeapList == NULL) { panic("heap_init_post_thread(): failed to attach dedicated grow heap\n"); - delete_area(area); return B_ERROR; } @@ -1302,7 +1331,6 @@ heap_init_post_thread() B_URGENT_PRIORITY, NULL); if (sHeapGrowThread < 0) { panic("heap_init_post_thread(): cannot create heap grow thread\n"); - delete_area(area); return sHeapGrowThread; } @@ -1331,17 +1359,6 @@ memalign(size_t alignment, size_t size) return NULL; } - if (thread_get_current_thread_id() == sHeapGrowThread) { - // this is the grower thread, allocate from our dedicated memory - void *result = heap_memalign(sGrowHeap, alignment, size, NULL); - if (result == NULL) { - panic("heap: grow thread ran out of dedicated memory!\n"); - return NULL; - } - - return result; - } - heap_allocator *heap = sHeapList; while (heap) { bool shouldGrow = false; @@ -1352,6 +1369,10 @@ memalign(size_t alignment, size_t size) if (result == NULL) { // urgent request, do the request and wait switch_sem(sHeapGrowSem, sHeapGrownNotify); + if (heap->next == NULL) { + // the grower didn't manage to add a new heap + return NULL; + } } else { // not so urgent, just notify the grower release_sem_etc(sHeapGrowSem, 1, B_DO_NOT_RESCHEDULE); @@ -1375,6 +1396,46 @@ memalign(size_t alignment, size_t size) } +void * +malloc_nogrow(size_t size) +{ + // use dedicated memory in the grow thread by default + if (thread_get_current_thread_id() == sHeapGrowThread) { + bool shouldGrow = false; + heap_allocator *heap = sGrowHeapList; + while (heap) { + void *result = heap_memalign(heap, 0, size, &shouldGrow); + if (shouldGrow && heap->next == NULL && !sAddGrowHeap) { + // hopefully the heap grower will manage to create a new heap + // before running out of private memory... + dprintf("heap: requesting new grow heap\n"); + sAddGrowHeap = true; + release_sem_etc(sHeapGrowSem, 1, B_DO_NOT_RESCHEDULE); + } + + if (result != NULL) + return result; + + heap = heap->next; + } + } + + // try public memory, there might be something available + heap_allocator *heap = sHeapList; + while (heap) { + void *result = heap_memalign(heap, 0, size, NULL); + if (result != NULL) + return result; + + heap = heap->next; + } + + // no memory available + panic("heap: all heaps have run out of memory\n"); + return NULL; +} + + void * malloc(size_t size) { @@ -1402,9 +1463,14 @@ free(void *address) heap = heap->next; } - // maybe it was allocated from the dedicated grow heap - if (heap_free(sGrowHeap, address) == B_OK) - return; + // maybe it was allocated from a dedicated grow heap + heap = sGrowHeapList; + while (heap) { + if (heap_free(heap, address) == B_OK) + return; + + heap = heap->next; + } panic("free(): free failed for address %p\n", address); } diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index 3c2bb19daa..d1641e9f95 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -810,7 +810,7 @@ lookup_area(vm_address_space* addressSpace, area_id id) static vm_area * create_reserved_area_struct(vm_address_space *addressSpace, uint32 flags) { - vm_area *reserved = (vm_area *)malloc(sizeof(vm_area)); + vm_area *reserved = (vm_area *)malloc_nogrow(sizeof(vm_area)); if (reserved == NULL) return NULL; @@ -833,11 +833,11 @@ create_area_struct(vm_address_space *addressSpace, const char *name, if (length > B_OS_NAME_LENGTH) length = B_OS_NAME_LENGTH; - vm_area *area = (vm_area *)malloc(sizeof(vm_area)); + vm_area *area = (vm_area *)malloc_nogrow(sizeof(vm_area)); if (area == NULL) return NULL; - area->name = (char *)malloc(length); + area->name = (char *)malloc_nogrow(length); if (area->name == NULL) { free(area); return NULL; @@ -2856,7 +2856,7 @@ vm_map_page(vm_area *area, vm_page *page, addr_t address, uint32 protection) vm_page_mapping *mapping = NULL; if (area->wiring == B_NO_LOCK) { - mapping = (vm_page_mapping *)malloc(sizeof(vm_page_mapping)); + mapping = (vm_page_mapping *)malloc_nogrow(sizeof(vm_page_mapping)); if (mapping == NULL) return B_NO_MEMORY; diff --git a/src/system/kernel/vm/vm_address_space.cpp b/src/system/kernel/vm/vm_address_space.cpp index 91a7d708d1..c01bb3fba2 100644 --- a/src/system/kernel/vm/vm_address_space.cpp +++ b/src/system/kernel/vm/vm_address_space.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -263,7 +264,7 @@ vm_create_address_space(team_id id, addr_t base, addr_t size, vm_address_space *addressSpace; status_t status; - addressSpace = (vm_address_space *)malloc(sizeof(vm_address_space)); + addressSpace = (vm_address_space *)malloc_nogrow(sizeof(vm_address_space)); if (addressSpace == NULL) return B_NO_MEMORY; diff --git a/src/system/kernel/vm/vm_cache.cpp b/src/system/kernel/vm/vm_cache.cpp index 455a33c9b6..caf33ac107 100644 --- a/src/system/kernel/vm/vm_cache.cpp +++ b/src/system/kernel/vm/vm_cache.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -543,7 +544,7 @@ vm_cache_create(vm_store* store) return NULL; } - cache = (vm_cache*)malloc(sizeof(vm_cache)); + cache = (vm_cache*)malloc_nogrow(sizeof(vm_cache)); if (cache == NULL) return NULL; diff --git a/src/system/kernel/vm/vm_store_anonymous_noswap.cpp b/src/system/kernel/vm/vm_store_anonymous_noswap.cpp index dcee40bfab..31132c3ac1 100644 --- a/src/system/kernel/vm/vm_store_anonymous_noswap.cpp +++ b/src/system/kernel/vm/vm_store_anonymous_noswap.cpp @@ -9,6 +9,7 @@ #include "vm_store_anonymous_noswap.h" +#include #include #include #include @@ -165,7 +166,7 @@ vm_store * vm_store_create_anonymous_noswap(bool canOvercommit, int32 numPrecommittedPages, int32 numGuardPages) { - anonymous_store *store = (anonymous_store *)malloc( + anonymous_store *store = (anonymous_store *)malloc_nogrow( sizeof(anonymous_store)); if (store == NULL) return NULL; diff --git a/src/system/kernel/vm/vm_store_device.c b/src/system/kernel/vm/vm_store_device.c index 92b7433993..e771a89597 100644 --- a/src/system/kernel/vm/vm_store_device.c +++ b/src/system/kernel/vm/vm_store_device.c @@ -9,6 +9,7 @@ #include "vm_store_device.h" +#include #include #include @@ -87,7 +88,7 @@ static vm_store_ops device_ops = { struct vm_store * vm_store_create_device(addr_t baseAddress) { - struct device_store *store = malloc(sizeof(struct device_store)); + struct device_store *store = malloc_nogrow(sizeof(struct device_store)); if (store == NULL) return NULL; diff --git a/src/system/kernel/vm/vm_store_null.c b/src/system/kernel/vm/vm_store_null.c index 90ef4d040c..8acf56320e 100644 --- a/src/system/kernel/vm/vm_store_null.c +++ b/src/system/kernel/vm/vm_store_null.c @@ -9,6 +9,7 @@ #include "vm_store_null.h" +#include #include @@ -78,7 +79,7 @@ vm_store_create_null(void) { struct vm_store *store; - store = malloc(sizeof(struct vm_store)); + store = malloc_nogrow(sizeof(struct vm_store)); if (store == NULL) return NULL;