From b7d3ddaa0e180f67a65cf832a430aa665870e624 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Sun, 7 Feb 2010 01:59:53 +0000 Subject: [PATCH] Sync kernel heap with alignment handling fixes, simplifications and performance improvements from debug heap in r35422. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@35423 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- src/system/kernel/heap.cpp | 263 +++++++++++++++++++++---------------- 1 file changed, 150 insertions(+), 113 deletions(-) diff --git a/src/system/kernel/heap.cpp b/src/system/kernel/heap.cpp index 3c0982b595..b40136ca68 100644 --- a/src/system/kernel/heap.cpp +++ b/src/system/kernel/heap.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2008-2009, Michael Lotz, mmlr@mlotz.ch. + * Copyright 2008-2010, Michael Lotz, mmlr@mlotz.ch. * Copyright 2002-2009, Axel Dörfler, axeld@pinc-software.de. * Distributed under the terms of the MIT License. * @@ -1308,31 +1308,47 @@ heap_unlink_page(heap_page *page, heap_page **list) static heap_page * -heap_allocate_contiguous_pages(heap_allocator *heap, uint32 pageCount) +heap_allocate_contiguous_pages(heap_allocator *heap, uint32 pageCount, + size_t alignment) { MutexLocker pageLocker(heap->page_lock); heap_area *area = heap->areas; while (area) { - bool found = false; + if (area->free_page_count < pageCount) { + area = area->next; + continue; + } + + uint32 step = 1; + uint32 firstValid = 0; + const uint32 lastValid = area->page_count - pageCount + 1; + + if (alignment > heap->page_size) { + firstValid = (ROUNDUP(area->base, alignment) - area->base) + / heap->page_size; + step = alignment / heap->page_size; + } + int32 first = -1; - for (uint32 i = 0; i < area->page_count; i++) { - if (area->page_table[i].in_use) { - first = -1; + for (uint32 i = firstValid; i < lastValid; i += step) { + if (area->page_table[i].in_use) continue; - } - if (first < 0) - first = i; + first = i; - if (first >= 0) { - if ((i + 1 - first) == pageCount) { - found = true; + for (uint32 j = 1; j < pageCount; j++) { + if (area->page_table[i + j].in_use) { + first = -1; + i += j; break; } } + + if (first >= 0) + break; } - if (!found) { + if (first < 0) { area = area->next; continue; } @@ -1357,111 +1373,120 @@ heap_allocate_contiguous_pages(heap_allocator *heap, uint32 pageCount) } -static void * -heap_raw_alloc(heap_allocator *heap, size_t size, uint32 binIndex) -{ - TRACE(("raw_alloc: heap %p; size %lu; bin %lu\n", heap, size, binIndex)); - if (binIndex < heap->bin_count) { - heap_bin *bin = &heap->bins[binIndex]; - MutexLocker binLocker(bin->lock); - - heap_page *page = bin->page_list; - if (page == NULL) { - MutexLocker pageLocker(heap->page_lock); - heap_area *area = heap->areas; - if (area == NULL) { - TRACE(("heap %p: no free pages to allocate %lu bytes\n", heap, - size)); - return NULL; - } - - // by design there are only areas in the list that still have - // free pages available - page = area->free_pages; - area->free_pages = page->next; - if (page->next) - page->next->prev = NULL; - - heap_free_pages_removed(heap, area, 1); - - if (page->in_use) - panic("got an in use page from the free pages list\n"); - page->in_use = 1; - - pageLocker.Unlock(); - - page->bin_index = binIndex; - page->free_count = bin->max_free_count; - page->empty_index = 0; - page->free_list = NULL; - page->next = page->prev = NULL; - bin->page_list = page; - } - - // we have a page where we have a free slot - void *address = NULL; - if (page->free_list) { - // there's a previously freed entry we can use - address = page->free_list; - page->free_list = (addr_t *)*page->free_list; - } else { - // the page hasn't been fully allocated so use the next empty_index - address = (void *)(page->area->base + page->index * heap->page_size - + page->empty_index * bin->element_size); - page->empty_index++; - } - - page->free_count--; - if (page->free_count == 0) { - // the page is now full so we remove it from the page_list - bin->page_list = page->next; - if (page->next) - page->next->prev = NULL; - page->next = page->prev = NULL; - } - - binLocker.Unlock(); - #if KERNEL_HEAP_LEAK_CHECK - heap_leak_check_info *info = (heap_leak_check_info *)((addr_t)address - + bin->element_size - sizeof(heap_leak_check_info)); - info->size = size - sizeof(heap_leak_check_info); - info->thread = (gKernelStartup ? 0 : thread_get_current_thread_id()); - info->team = (gKernelStartup ? 0 : team_get_current_team_id()); - info->caller = heap->get_caller(); +static void +heap_add_leak_check_info(addr_t address, size_t allocated, size_t size) +{ + heap_leak_check_info *info = (heap_leak_check_info *)(address + allocated + - sizeof(heap_leak_check_info)); + info->size = size - sizeof(heap_leak_check_info); + info->thread = (gKernelStartup ? 0 : thread_get_current_thread_id()); + info->team = (gKernelStartup ? 0 : team_get_current_team_id()); + info->caller = heap->get_caller(); +} #endif - return address; - } + + +static void * +heap_raw_alloc(heap_allocator *heap, size_t size, size_t alignment) +{ + TRACE(("heap %p: allocate %lu bytes from raw pages with alignment %lu\n", + heap, size, alignment)); uint32 pageCount = (size + heap->page_size - 1) / heap->page_size; - heap_page *firstPage = heap_allocate_contiguous_pages(heap, pageCount); + heap_page *firstPage = heap_allocate_contiguous_pages(heap, pageCount, + alignment); if (firstPage == NULL) { TRACE(("heap %p: found no contiguous pages to allocate %ld bytes\n", heap, size)); return NULL; } + addr_t address = firstPage->area->base + firstPage->index * heap->page_size; #if KERNEL_HEAP_LEAK_CHECK - heap_leak_check_info *info = (heap_leak_check_info *)(firstPage->area->base - + (firstPage->index + pageCount) * heap->page_size - - sizeof(heap_leak_check_info)); - info->size = size - sizeof(heap_leak_check_info); - info->thread = (gKernelStartup ? 0 : thread_get_current_thread_id()); - info->team = (gKernelStartup ? 0 : team_get_current_team_id()); - info->caller = heap->get_caller(); + heap_add_leak_check_info(address, pageCount * heap->page_size, size); #endif - return (void *)(firstPage->area->base + firstPage->index * heap->page_size); + return (void *)address; +} + + +static void * +heap_allocate_from_bin(heap_allocator *heap, uint32 binIndex, size_t size) +{ + heap_bin *bin = &heap->bins[binIndex]; + TRACE(("heap %p: allocate %lu bytes from bin %lu with element_size %lu\n", + heap, size, binIndex, bin->element_size)); + + MutexLocker binLocker(bin->lock); + heap_page *page = bin->page_list; + if (page == NULL) { + MutexLocker pageLocker(heap->page_lock); + heap_area *area = heap->areas; + if (area == NULL) { + TRACE(("heap %p: no free pages to allocate %lu bytes\n", heap, + size)); + return NULL; + } + + // by design there are only areas in the list that still have + // free pages available + page = area->free_pages; + area->free_pages = page->next; + if (page->next) + page->next->prev = NULL; + + heap_free_pages_removed(heap, area, 1); + + if (page->in_use) + panic("got an in use page from the free pages list\n"); + page->in_use = 1; + + pageLocker.Unlock(); + + page->bin_index = binIndex; + page->free_count = bin->max_free_count; + page->empty_index = 0; + page->free_list = NULL; + page->next = page->prev = NULL; + bin->page_list = page; + } + + // we have a page where we have a free slot + void *address = NULL; + if (page->free_list) { + // there's a previously freed entry we can use + address = page->free_list; + page->free_list = (addr_t *)*page->free_list; + } else { + // the page hasn't been fully allocated so use the next empty_index + address = (void *)(page->area->base + page->index * heap->page_size + + page->empty_index * bin->element_size); + page->empty_index++; + } + + page->free_count--; + if (page->free_count == 0) { + // the page is now full so we remove it from the page_list + bin->page_list = page->next; + if (page->next) + page->next->prev = NULL; + page->next = page->prev = NULL; + } + +#if KERNEL_HEAP_LEAK_CHECK + binLocker.Unlock(); + heap_add_leak_check_info((addr_t)address, bin->element_size, size); +#endif + return address; } -#if DEBUG static bool is_valid_alignment(size_t number) { // this cryptic line accepts zero and all powers of two return ((~number + 1) | ((number << 1) - 1)) == ~0UL; } -#endif inline bool @@ -1486,23 +1511,35 @@ heap_memalign(heap_allocator *heap, size_t alignment, size_t size) size += sizeof(heap_leak_check_info); #endif - if (alignment != 0) { - // TODO: The alignment is done by ensuring that the element size - // of the target bin is aligned with the requested alignment. This - // has the problem that it wastes space because a better (smaller) bin - // could possibly be selected. We should pick the best bin and check - // if there is an aligned block in the free list or if a new (page - // aligned) page has to be allocated anyway. - size = ROUNDUP(size, alignment); + void *address = NULL; + if (alignment < B_PAGE_SIZE) { + if (alignment != 0) { + // TODO: The alignment is done by ensuring that the element size + // of the target bin is aligned with the requested alignment. This + // has the problem that it wastes space because a better (smaller) + // bin could possibly be selected. We should pick the best bin and + // check if there is an aligned block in the free list or if a new + // (page aligned) page has to be allocated anyway. + size = ROUNDUP(size, alignment); + for (uint32 i = 0; i < heap->bin_count; i++) { + if (size <= heap->bins[i].element_size + && is_valid_alignment(heap->bins[i].element_size)) { + address = heap_allocate_from_bin(heap, i, size); + break; + } + } + } else { + for (uint32 i = 0; i < heap->bin_count; i++) { + if (size <= heap->bins[i].element_size) { + address = heap_allocate_from_bin(heap, i, size); + break; + } + } + } } - uint32 binIndex; - for (binIndex = 0; binIndex < heap->bin_count; binIndex++) { - if (size <= heap->bins[binIndex].element_size) - break; - } - - void *address = heap_raw_alloc(heap, size, binIndex); + if (address == NULL) + address = heap_raw_alloc(heap, size, alignment); #if KERNEL_HEAP_LEAK_CHECK size -= sizeof(heap_leak_check_info); @@ -2183,9 +2220,9 @@ memalign(size_t alignment, size_t size) address = (void *)((addr_t)address + sizeof(area_allocation_info)); if (alignment != 0) { + address = (void *)ROUNDUP((addr_t)address, alignment); ASSERT((addr_t)address % alignment == 0); ASSERT((addr_t)address + size - 1 < (addr_t)info + areaSize - 1); - address = (void *)ROUNDUP((addr_t)address, alignment); } TRACE(("heap: allocated area %ld for huge allocation of %lu bytes\n",