i2p/haiku-beta6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kernel: Overhaul debug heaps system.

Browse Source 
Instead of the kernel heap being a compile-time parameter, make it
a runtime parameter (if DEBUG_HEAPS is specified, at least; otherwise
we're hardwired to the slab heap as before.)

This is similar to what's done in userland: there LD_PRELOAD and
MALLOC_DEBUG control the heaps, here "kernel_malloc" and then
"guarded_heap_options" kernel settings control the heap.

While at it, apply fixes to the guarded and debug heaps to ensure
a minimum alignment of sizeof(void*), if no specific alignment
is requested.

Tested with a lot of build configurations (DEBUG_HEAPS 0, 1;
DEBUG_HEAPS_DEFAULT slab, guarded; USE_DEBUG_HEAPS_FOR_OBJECT_CACHE
0, 1) and with various parameters specified in the bootloader.
This commit is contained in:
Augustin Cavalier
2026-03-23 23:01:26 -04:00
parent 4fd16356e0
commit a277043c1f
12 changed files with 580 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/config_headers/kernel_debug_config.h
+29 -23
View File
@@ -85,6 +85,9 @@
// VM
// Enables swap support.
#define ENABLE_SWAP_SUPPORT 1
// Enables the vm_page::queue field, i.e. it is tracked which queue the page
// should be in.
#define DEBUG_PAGE_QUEUE 0
@@ -98,35 +101,38 @@
// Enables a global list of all vm_cache structures.
#define DEBUG_CACHE_LIST KDEBUG_LEVEL_2
// Enables swap support.
#define ENABLE_SWAP_SUPPORT 1
// Enables the alternative heaps to be selected at startup.
#define DEBUG_HEAPS KDEBUG_LEVEL_2
// Use the selected allocator as generic memory allocator (malloc()/free()).
#define USE_DEBUG_HEAP_FOR_MALLOC 0
// Heap implementation with additional debugging facilities.
#define USE_GUARDED_HEAP_FOR_MALLOC 0
// Heap implementation that allocates memory so that the end of the
// allocation always coincides with a page end and is followed by a guard
// page which is marked non-present. Out of bounds access (both read and
// write) therefore cause a crash (unhandled page fault). Note that this
// allocator is neither speed nor space efficient, indeed it wastes huge
// amounts of pages and address space so it is quite easy to hit limits.
#define USE_SLAB_ALLOCATOR_FOR_MALLOC 1
// Heap implementation based on the slab allocator (for production use).
// Sets the default heap to be used as the generic memory allocator (malloc/free)
// when DEBUG_HEAPS is enabled. (This value can be overridden via "kernel_malloc".)
//
// The possible options are:
// - "slab": Heap implementation based on the slab allocator (for production use).
//
// - "debug": Heap implementation with additional debugging facilities.
//
// - "guarded": Heap implementation that allocates memory so that the end of the
// allocation always coincides with a page end and is followed by a guard page
// which is marked non-present. Out of bounds access (both read and write)
// therefore cause a crash (page fault). Note that this allocator is neither
// speed nor space efficient, indeed it wastes huge amounts of pages and address
// space so it is quite easy to hit limits.
#define DEBUG_HEAPS_DEFAULT slab
// Replace the object cache with the guarded heap to force debug features. Also
// requires the use of the guarded heap for malloc.
#define USE_GUARDED_HEAP_FOR_OBJECT_CACHE 0
// Replace the object cache with the chosen debug heap to force debug features.
#define USE_DEBUG_HEAPS_FOR_OBJECT_CACHE 0
// If 0, disables memory re-use by default in the guarded heap (can be overridden
// via "guarded_heap_options".) This means freed virtual memory is never reused and
// stays invalid, causing any access to crash. This is a magnitude more space
// inefficient than the guarded heap itself; fully booting may not work at all due
// to address space waste.
#define DEBUG_GUARDED_HEAP_MEMORY_REUSE_DEFAULT 1
// Enables additional sanity checks in the slab allocator's memory manager.
#define DEBUG_SLAB_MEMORY_MANAGER_PARANOID_CHECKS 0
// Disables memory re-use in the guarded heap (freed memory is never reused and
// stays invalid causing every access to crash). Note that this is a magnitude
// more space inefficient than the guarded heap itself. Fully booting may not
// work at all due to address space waste.
#define DEBUG_GUARDED_HEAP_DISABLE_MEMORY_REUSE 0
// When set limits the amount of available RAM (in MB).
//#define LIMIT_AVAILABLE_MEMORY 256
headers/private/kernel/heap.h
+2 -27
View File
@@ -8,37 +8,12 @@
#ifndef _KERNEL_HEAP_H
#define _KERNEL_HEAP_H
#include <OS.h>
#include "kernel_debug_config.h"
#if USE_GUARDED_HEAP_FOR_MALLOC && USE_GUARDED_HEAP_FOR_OBJECT_CACHE
// This requires a lot of up-front memory to boot at all...
#define INITIAL_HEAP_SIZE 128 * 1024 * 1024
// ... and a lot of reserves to keep running.
#define HEAP_GROW_SIZE 128 * 1024 * 1024
#elif USE_GUARDED_HEAP_FOR_MALLOC
#define INITIAL_HEAP_SIZE 32 * 1024 * 1024
#define HEAP_GROW_SIZE 32 * 1024 * 1024
#else
// allocate 16MB initial heap for the kernel
#define INITIAL_HEAP_SIZE 16 * 1024 * 1024
// grow by another 4MB each time the heap runs out of memory
#define HEAP_GROW_SIZE 4 * 1024 * 1024
// allocate a dedicated 1MB area for dynamic growing
#define HEAP_DEDICATED_GROW_SIZE 1 * 1024 * 1024
// use areas for allocations bigger than 1MB
#define HEAP_AREA_USE_THRESHOLD 1 * 1024 * 1024
#endif
// allocation/deallocation flags for {malloc,free}_etc()
#define HEAP_DONT_WAIT_FOR_MEMORY 0x01
#define HEAP_DONT_LOCK_KERNEL_SPACE 0x02
@@ -58,7 +33,7 @@ void* memalign(size_t alignment, size_t size) _ALIGNED_BY_ARG(1);
void deferred_free(void* block);
status_t heap_init(addr_t heapBase, size_t heapSize);
status_t heap_init(struct kernel_args* args);
status_t heap_init_post_area();
status_t heap_init_post_sem();
status_t heap_init_post_thread();
src/system/kernel/debug/Jamfile
+1
View File
@@ -26,6 +26,7 @@ KernelMergeObject kernel_debug.o :
gdb.cpp
guarded_heap.cpp
heap.cpp
heaps.cpp
safemode_settings.cpp
system_profiler.cpp
tracing.cpp
src/system/kernel/debug/guarded_heap.cpp
+59 -188
View File
@@ -13,6 +13,7 @@
#include <debug.h>
#include <heap.h>
#include <malloc.h>
#include <safemode.h>
#include <slab/Slab.h>
#include <team.h>
#include <util/SimpleAllocator.h>
@@ -21,11 +22,12 @@
#include <vm/vm.h>
#include <vm/vm_page.h>
#include "heaps.h"
#include "../vm/VMAddressSpaceLocking.h"
#include "../vm/VMAnonymousNoSwapCache.h"
#if USE_GUARDED_HEAP_FOR_MALLOC
#if DEBUG_HEAPS
#define GUARDED_HEAP_STACK_TRACE_DEPTH 0
@@ -95,6 +97,7 @@ protected:
struct guarded_heap {
mutex lock;
bool reuse_memory;
ConditionVariable memory_added_condition;
GuardedHeapCache* cache;
@@ -122,7 +125,8 @@ static size_t sGuardedHeapEarlySize;
static void*
guarded_heap_allocate_meta(guarded_heap& heap, size_t size, uint32 flags)
{
size_t growSize = ROUNDUP(((HEAP_GROW_SIZE / B_PAGE_SIZE) / 2) * sizeof(GuardedHeapChunk),
size_t growSize = ROUNDUP(((kernel_guarded_heap.grow_size / B_PAGE_SIZE) / 2)
* sizeof(GuardedHeapChunk),
B_PAGE_SIZE);
while (heap.meta_allocator.Available() < (growSize / 2)) {
if ((flags & HEAP_DONT_LOCK_KERNEL_SPACE) != 0)
@@ -172,7 +176,7 @@ guarded_heap_add_area(guarded_heap& heap, size_t minimumPages, uint32 flags)
if (heap.acquiring_pages == thread_get_current_thread_id())
return false;
size_t growPages = HEAP_GROW_SIZE / B_PAGE_SIZE;
size_t growPages = kernel_guarded_heap.grow_size / B_PAGE_SIZE;
if (minimumPages > growPages)
growPages = minimumPages;
@@ -245,11 +249,11 @@ guarded_heap_allocate(guarded_heap& heap, size_t size, size_t alignment,
MutexLocker locker(heap.lock);
if (alignment == 0)
alignment = 1;
alignment = sizeof(void*);
// If we need more address space, allocate some now, while
// there's still space to allocate bookkeeping structures.
if (heap.free_pages_count <= (HEAP_GROW_SIZE / B_PAGE_SIZE / 2))
if (heap.free_pages_count <= (kernel_guarded_heap.grow_size / B_PAGE_SIZE / 2))
guarded_heap_add_area(heap, 0, flags);
// Allocate a spare meta chunk up-front, since this also might
@@ -442,12 +446,10 @@ guarded_heap_free(guarded_heap& heap, void* address, uint32 flags)
vm_unreserve_memory(reservation.count * B_PAGE_SIZE);
vm_page_unreserve_pages(&reservation);
#if DEBUG_GUARDED_HEAP_DISABLE_MEMORY_REUSE
GuardedHeapChunksTree& tree = heap.dead_chunks;
#else
GuardedHeapChunksTree& tree = heap.free_chunks;
heap.free_pages_count += chunk->pages_count;
#endif
GuardedHeapChunksTree& tree = heap.reuse_memory ?
heap.free_chunks : heap.dead_chunks;
if (heap.reuse_memory)
heap.free_pages_count += chunk->pages_count;
GuardedHeapChunk* joinLower = guarded_heap_find_chunk(tree, chunk->base - 1),
*joinUpper = guarded_heap_find_chunk(tree,
@@ -681,9 +683,22 @@ dump_guarded_heap_allocations(int argc, char** argv)
// #pragma mark - Malloc API
status_t
heap_init(addr_t address, size_t size)
static status_t
guarded_heap_init(struct kernel_args* args, addr_t address, size_t size)
{
sGuardedHeap.reuse_memory = DEBUG_GUARDED_HEAP_MEMORY_REUSE_DEFAULT;
char options[32];
size_t optionsSize = sizeof(options);
if (get_safemode_option_early(args, "guarded_heap_options", options, &optionsSize) == B_OK) {
if (strchr(options, 'r') != NULL)
sGuardedHeap.reuse_memory = false;
else if (strchr(options, 'R') != NULL)
sGuardedHeap.reuse_memory = true;
}
dprintf("guarded heap settings: %s\n",
sGuardedHeap.reuse_memory ? "R" : "r");
sGuardedHeap.memory_added_condition.Init(&sGuardedHeap, "guarded heap");
sGuardedHeap.acquiring_pages = sGuardedHeap.acquiring_meta = -1;
@@ -705,8 +720,8 @@ heap_init(addr_t address, size_t size)
}
status_t
heap_init_post_area()
static status_t
guarded_heap_init_post_area()
{
void* address = (void*)sGuardedHeapEarlyMetaBase;
area_id metaAreaId = create_area("guarded heap meta", &address, B_EXACT_ADDRESS,
@@ -724,8 +739,8 @@ heap_init_post_area()
}
status_t
heap_init_post_sem()
static status_t
guarded_heap_init_post_sem()
{
new(&sGuardedHeapCache) GuardedHeapCache;
sGuardedHeapCache.Init();
@@ -828,200 +843,56 @@ heap_init_post_sem()
}
void*
memalign(size_t alignment, size_t size)
static void*
guarded_heap_memalign(size_t alignment, size_t size, uint32 flags)
{
return memalign_etc(alignment, size, 0);
}
void *
memalign_etc(size_t alignment, size_t size, uint32 flags)
{
if (size == 0)
size = 1;
return guarded_heap_allocate(sGuardedHeap, size, alignment, flags);
}
extern "C" int
posix_memalign(void** _pointer, size_t alignment, size_t size)
{
if ((alignment & (sizeof(void*) - 1)) != 0 || _pointer == NULL)
return B_BAD_VALUE;
*_pointer = guarded_heap_allocate(sGuardedHeap, size, alignment, 0);
return 0;
}
void
free_etc(void *address, uint32 flags)
static void
guarded_heap_free_etc(void *address, uint32 flags)
{
guarded_heap_free(sGuardedHeap, address, flags);
}
void*
malloc(size_t size)
{
return memalign_etc(sizeof(void*), size, 0);
}
void
free(void* address)
{
free_etc(address, 0);
}
void*
realloc_etc(void* address, size_t newSize, uint32 flags)
static void*
guarded_heap_realloc_etc(void* address, size_t newSize, uint32 flags)
{
if (newSize == 0) {
free_etc(address, flags);
guarded_heap_free(sGuardedHeap, address, flags);
return NULL;
}
if (address == NULL)
return malloc_etc(newSize, flags);
return guarded_heap_memalign(0, newSize, flags);
return guarded_heap_realloc(sGuardedHeap, address, newSize, flags);
}
void*
realloc(void* address, size_t newSize)
{
return realloc_etc(address, newSize, 0);
}
kernel_heap_implementation kernel_guarded_heap = {
"guarded_heap",
#if USE_DEBUG_HEAPS_FOR_OBJECT_CACHE
// This requires a lot of up-front memory to boot at all...
/* initial_size */ 128 * 1024 * 1024,
// ... and a lot of reserves to keep running.
/* grow_size */ 128 * 1024 * 1024,
#else
/* initial_size */ 32 * 1024 * 1024,
/* grow size */ 32 * 1024 * 1024,
#endif
guarded_heap_init,
guarded_heap_init_post_area,
guarded_heap_init_post_sem,
NULL,
#endif // USE_GUARDED_HEAP_FOR_MALLOC
#if USE_GUARDED_HEAP_FOR_OBJECT_CACHE
// #pragma mark - Slab API
struct ObjectCache {
size_t object_size;
size_t alignment;
void* cookie;
object_cache_constructor constructor;
object_cache_destructor destructor;
guarded_heap_memalign,
guarded_heap_realloc_etc,
guarded_heap_free_etc,
};
object_cache*
create_object_cache(const char* name, size_t object_size, uint32 flags)
{
return create_object_cache_etc(name, object_size, 0, 0, 0, 0, flags,
NULL, NULL, NULL, NULL);
}
object_cache*
create_object_cache_etc(const char*, size_t objectSize, size_t alignment, size_t, size_t,
size_t, uint32, void* cookie, object_cache_constructor ctor, object_cache_destructor dtor,
object_cache_reclaimer)
{
ObjectCache* cache = new ObjectCache;
if (cache == NULL)
return NULL;
cache->object_size = objectSize;
cache->alignment = alignment;
cache->cookie = cookie;
cache->constructor = ctor;
cache->destructor = dtor;
return cache;
}
void
delete_object_cache(object_cache* cache)
{
delete cache;
}
status_t
object_cache_set_minimum_reserve(object_cache* cache, size_t objectCount)
{
return B_OK;
}
void*
object_cache_alloc(object_cache* cache, uint32 flags)
{
void* object = memalign_etc(cache->alignment, cache->object_size, flags);
if (object == NULL)
return NULL;
if (cache->constructor != NULL)
cache->constructor(cache->cookie, object);
return object;
}
void
object_cache_free(object_cache* cache, void* object, uint32 flags)
{
if (cache->destructor != NULL)
cache->destructor(cache->cookie, object);
return free_etc(object, flags);
}
status_t
object_cache_reserve(object_cache* cache, size_t objectCount, uint32 flags)
{
return B_OK;
}
void
object_cache_get_usage(object_cache* cache, size_t* _allocatedMemory)
{
*_allocatedMemory = 0;
}
void
request_memory_manager_maintenance()
{
}
void
slab_init(kernel_args* args)
{
}
void
slab_init_post_area()
{
}
void
slab_init_post_sem()
{
}
void
slab_init_post_thread()
{
}
#endif // USE_GUARDED_HEAP_FOR_OBJECT_CACHE
#endif // DEBUG_HEAPS
src/system/kernel/debug/heap.cpp
+73 -58
View File
@@ -23,6 +23,8 @@
#include <vm/vm.h>
#include <vm/vm_page.h>
#include "heaps.h"
//#define TRACE_HEAP
#ifdef TRACE_HEAP
@@ -32,10 +34,18 @@
#endif
#if DEBUG_HEAPS
#define USE_DEBUG_HEAP_FOR_MALLOC 1
#if !USE_DEBUG_HEAP_FOR_MALLOC
# undef KERNEL_HEAP_LEAK_CHECK
#endif
// allocate a dedicated 1MB area for dynamic growing
#define HEAP_DEDICATED_GROW_SIZE 1 * 1024 * 1024
// use areas for allocations bigger than 1MB
#define HEAP_AREA_USE_THRESHOLD 1 * 1024 * 1024
#if KERNEL_HEAP_LEAK_CHECK
typedef struct heap_leak_check_info_s {
@@ -57,7 +67,6 @@ static int32 sCallerInfoCount = 0;
#endif // KERNEL_HEAP_LEAK_CHECK
#if USE_DEBUG_HEAP_FOR_MALLOC
typedef struct heap_class_s {
const char *name;
uint32 initial_percentage;
@@ -158,7 +167,7 @@ static const heap_class sHeapClasses[HEAP_CLASS_COUNT] = {
B_PAGE_SIZE / 8, /* max allocation size */
B_PAGE_SIZE, /* page size */
8, /* min bin size */
4, /* bin alignment */
sizeof(void*), /* bin alignment */
8, /* min count per page */
16 /* max waste per page */
},
@@ -185,6 +194,9 @@ static const heap_class sHeapClasses[HEAP_CLASS_COUNT] = {
};
static addr_t sInitialBase;
static size_t sInitialSize;
static uint32 sHeapCount;
static heap_allocator *sHeaps[HEAP_CLASS_COUNT * SMP_MAX_CPUS];
static uint32 *sLastGrowRequest[HEAP_CLASS_COUNT * SMP_MAX_CPUS];
@@ -1144,7 +1156,7 @@ heap_remove_area(heap_allocator *heap, heap_area *area)
}
heap_allocator *
static heap_allocator *
heap_create_allocator(const char *name, addr_t base, size_t size,
const heap_class *heapClass, bool allocateOnHeap)
{
@@ -1505,11 +1517,11 @@ inline bool
heap_should_grow(heap_allocator *heap)
{
// suggest growing if there is less than 20% of a grow size available
return heap->total_free_pages * heap->page_size < HEAP_GROW_SIZE / 5;
return heap->total_free_pages * heap->page_size < kernel_debug_heap.grow_size / 5;
}
void *
static void *
heap_memalign(heap_allocator *heap, size_t alignment, size_t size)
{
TRACE(("memalign(alignment = %lu, size = %lu)\n", alignment, size));
@@ -1579,7 +1591,7 @@ heap_memalign(heap_allocator *heap, size_t alignment, size_t size)
}
status_t
static status_t
heap_free(heap_allocator *heap, void *address)
{
if (address == NULL)
@@ -1970,7 +1982,7 @@ heap_grow_thread(void *)
// allocation cannot be fulfilled due to lack of contiguous
// pages)
if (heap_create_new_heap_area(heap, "additional heap",
HEAP_GROW_SIZE) != B_OK)
kernel_debug_heap.grow_size) != B_OK)
dprintf("heap_grower: failed to create new heap area\n");
sLastHandledGrowRequest[i] = sLastGrowRequest[i];
}
@@ -1987,9 +1999,12 @@ heap_grow_thread(void *)
// #pragma mark -
status_t
heap_init(addr_t base, size_t size)
static status_t
debug_heap_init(struct kernel_args*, addr_t base, size_t size)
{
sInitialBase = base;
sInitialSize = size;
for (uint32 i = 0; i < HEAP_CLASS_COUNT; i++) {
size_t partSize = size * sHeapClasses[i].initial_percentage / 100;
sHeaps[i] = heap_create_allocator(sHeapClasses[i].name, base, partSize,
@@ -2040,9 +2055,12 @@ heap_init(addr_t base, size_t size)
}
status_t
heap_init_post_area()
static status_t
debug_heap_init_post_area()
{
create_area("kernel heap", (void**)&sInitialBase, B_EXACT_ADDRESS,
sInitialSize, B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
void *address = NULL;
area_id growHeapArea = create_area("dedicated grow heap", &address,
B_ANY_KERNEL_BLOCK_ADDRESS, HEAP_DEDICATED_GROW_SIZE, B_FULL_LOCK,
@@ -2067,7 +2085,7 @@ heap_init_post_area()
B_PAGE_SIZE / 8, /* max allocation size */
B_PAGE_SIZE, /* page size */
8, /* min bin size */
4, /* bin alignment */
sizeof(void*), /* bin alignment */
8, /* min count per page */
16 /* max waste per page */
};
@@ -2093,8 +2111,8 @@ heap_init_post_area()
}
status_t
heap_init_post_sem()
static status_t
debug_heap_init_post_sem()
{
sHeapGrowSem = create_sem(0, "heap_grow_sem");
if (sHeapGrowSem < 0) {
@@ -2112,13 +2130,9 @@ heap_init_post_sem()
}
#endif // USE_DEBUG_HEAP_FOR_MALLOC
status_t
heap_init_post_thread()
static status_t
debug_heap_init_post_thread()
{
#if USE_DEBUG_HEAP_FOR_MALLOC
sHeapGrowThread = spawn_kernel_thread(heap_grow_thread, "heap grower",
B_URGENT_PRIORITY, NULL);
if (sHeapGrowThread < 0) {
@@ -2131,7 +2145,7 @@ heap_init_post_thread()
(int32)vm_page_num_pages() / 60 / 1024);
for (int32 i = 1; i < heapCount; i++) {
addr_t base = 0;
size_t size = HEAP_GROW_SIZE * HEAP_CLASS_COUNT;
size_t size = kernel_debug_heap.grow_size * HEAP_CLASS_COUNT;
area_id perCPUHeapArea = create_area("per cpu initial heap",
(void **)&base, B_ANY_KERNEL_ADDRESS, size, B_FULL_LOCK,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
@@ -2165,7 +2179,6 @@ heap_init_post_thread()
"If the optional argument \"stats\" is specified, only the allocation\n"
"counts and no individual allocations are printed.\n", 0);
#endif // KERNEL_HEAP_LEAK_CHECK
#endif // USE_DEBUG_HEAP_FOR_MALLOC
return B_OK;
}
@@ -2174,11 +2187,9 @@ heap_init_post_thread()
// #pragma mark - Public API
#if USE_DEBUG_HEAP_FOR_MALLOC
void *
memalign(size_t alignment, size_t size)
static void *
debug_heap_memalign(size_t alignment, size_t size)
{
if (!gKernelStartup && !are_interrupts_enabled()) {
panic("memalign(): called with interrupts disabled\n");
@@ -2270,8 +2281,8 @@ memalign(size_t alignment, size_t size)
}
void *
memalign_etc(size_t alignment, size_t size, uint32 flags)
static void *
debug_heap_memalign_etc(size_t alignment, size_t size, uint32 flags)
{
if ((flags & HEAP_PRIORITY_VIP) != 0)
return heap_memalign(sVIPHeap, alignment, size);
@@ -2281,29 +2292,12 @@ memalign_etc(size_t alignment, size_t size, uint32 flags)
return memalign_nogrow(alignment, size);
}
return memalign(alignment, size);
return debug_heap_memalign(alignment, size);
}
void
free_etc(void *address, uint32 flags)
{
if ((flags & HEAP_PRIORITY_VIP) != 0)
heap_free(sVIPHeap, address);
else
free(address);
}
void *
malloc(size_t size)
{
return memalign_etc(0, size, 0);
}
void
free(void *address)
static void
debug_heap_free(void *address)
{
if (!gKernelStartup && !are_interrupts_enabled()) {
panic("free(): called with interrupts disabled\n");
@@ -2350,8 +2344,18 @@ free(void *address)
}
void *
realloc_etc(void *address, size_t newSize, uint32 flags)
static void
debug_heap_free_etc(void *address, uint32 flags)
{
if ((flags & HEAP_PRIORITY_VIP) != 0)
heap_free(sVIPHeap, address);
else
debug_heap_free(address);
}
static void *
debug_heap_realloc(void *address, size_t newSize, uint32 flags)
{
if (!gKernelStartup && !are_interrupts_enabled()) {
panic("realloc(): called with interrupts disabled\n");
@@ -2359,10 +2363,10 @@ realloc_etc(void *address, size_t newSize, uint32 flags)
}
if (address == NULL)
return malloc_etc(newSize, flags);
return debug_heap_memalign_etc(0, newSize, flags);
if (newSize == 0) {
free_etc(address, flags);
debug_heap_free_etc(address, flags);
return NULL;
}
@@ -2425,11 +2429,22 @@ realloc_etc(void *address, size_t newSize, uint32 flags)
}
void *
realloc(void *address, size_t newSize)
{
return realloc_etc(address, newSize, 0);
}
kernel_heap_implementation kernel_debug_heap = {
"debug_heap",
// allocate 16MB initial heap for the kernel
16 * 1024 * 1024,
// grow by another 4MB each time the heap runs out of memory
4 * 1024 * 1024,
debug_heap_init,
debug_heap_init_post_area,
debug_heap_init_post_sem,
debug_heap_init_post_thread,
debug_heap_memalign_etc,
debug_heap_realloc,
debug_heap_free_etc,
};
#endif // USE_DEBUG_HEAP_FOR_MALLOC
#endif // DEBUG_HEAPS
src/system/kernel/debug/heaps.cpp
+287
View File
@@ -0,0 +1,287 @@
/*
* Copyright 2026, Haiku, Inc. All rights reserved.
* Distributed under the terms of the MIT License.
*/
#include "kernel_debug_config.h"
#if DEBUG_HEAPS
#include <stdlib.h>
#include <heap.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <slab/Slab.h>
#include <safemode.h>
#include "heaps.h"
//#define TRACE_HEAPS
#ifdef TRACE_HEAPS
# define TRACE(x) dprintf x
#else
# define TRACE(x) ;
#endif
#define HEAP_SYMBOL_NAME(NAME) kernel_##NAME##_heap
#define HEAP_SYMBOL(NAME) HEAP_SYMBOL_NAME(NAME)
static kernel_heap_implementation* sHeap = &HEAP_SYMBOL(DEBUG_HEAPS_DEFAULT);
// #pragma mark -
status_t
heap_init(struct kernel_args* args)
{
char buffer[32];
size_t bufferSize = sizeof(buffer);
if (get_safemode_option_early(args, "kernel_malloc", buffer, &bufferSize) == B_OK) {
if (strcmp(buffer, "guarded") == 0)
sHeap = &kernel_guarded_heap;
else if (strcmp(buffer, "debug") == 0)
sHeap = &kernel_debug_heap;
#if !USE_DEBUG_HEAPS_FOR_OBJECT_CACHE
else if (strcmp(buffer, "slab") == 0)
sHeap = &kernel_slab_heap;
#endif
else
panic("unknown or unavailable kernel heap '%s'!", buffer);
}
dprintf("kernel malloc: using %s\n", sHeap->name);
addr_t heapBase = 0;
size_t heapSize = sHeap->initial_size;
if (heapSize != 0) {
// try to accomodate low memory systems
while (heapSize > (vm_page_num_pages() * B_PAGE_SIZE) / 8)
heapSize /= 2;
if (heapSize < 1024 * 1024)
panic("heap_init: go buy some RAM please.");
// map in the new heap and initialize it
heapBase = vm_allocate_early(args, heapSize, heapSize,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0);
TRACE(("heap at 0x%lx\n", heapBase));
}
return sHeap->init(args, heapBase, heapSize);
}
status_t
heap_init_post_area()
{
if (sHeap->init_post_area == NULL)
return B_OK;
return sHeap->init_post_area();
}
status_t
heap_init_post_sem()
{
if (sHeap->init_post_sem == NULL)
return B_OK;
return sHeap->init_post_sem();
}
status_t
heap_init_post_thread()
{
if (sHeap->init_post_thread == NULL)
return B_OK;
return sHeap->init_post_thread();
}
void*
memalign(size_t alignment, size_t size)
{
return sHeap->memalign(alignment, size, 0);
}
void*
memalign_etc(size_t alignment, size_t size, uint32 flags)
{
return sHeap->memalign(alignment, size, flags);
}
void
free_etc(void* address, uint32 flags)
{
return sHeap->free(address, flags);
}
void
free(void *address)
{
return sHeap->free(address, 0);
}
void*
malloc(size_t size)
{
return sHeap->memalign(0, size, 0);
}
void*
realloc_etc(void* address, size_t newSize, uint32 flags)
{
return sHeap->realloc(address, newSize, flags);
}
void*
realloc(void *address, size_t newSize)
{
return sHeap->realloc(address, newSize, 0);
}
extern "C" int
posix_memalign(void** _pointer, size_t alignment, size_t size)
{
if ((alignment & (sizeof(void*) - 1)) != 0 || _pointer == NULL)
return B_BAD_VALUE;
*_pointer = sHeap->memalign(alignment, size, 0);
return 0;
}
#if USE_DEBUG_HEAPS_FOR_OBJECT_CACHE
// #pragma mark - Slab API
struct ObjectCache {
size_t object_size;
size_t alignment;
void* cookie;
object_cache_constructor constructor;
object_cache_destructor destructor;
};
object_cache*
create_object_cache(const char* name, size_t object_size, uint32 flags)
{
return create_object_cache_etc(name, object_size, 0, 0, 0, 0, flags,
NULL, NULL, NULL, NULL);
}
object_cache*
create_object_cache_etc(const char*, size_t objectSize, size_t alignment, size_t, size_t,
size_t, uint32, void* cookie, object_cache_constructor ctor, object_cache_destructor dtor,
object_cache_reclaimer)
{
ObjectCache* cache = new ObjectCache;
if (cache == NULL)
return NULL;
cache->object_size = objectSize;
cache->alignment = alignment;
cache->cookie = cookie;
cache->constructor = ctor;
cache->destructor = dtor;
return cache;
}
void
delete_object_cache(object_cache* cache)
{
delete cache;
}
status_t
object_cache_set_minimum_reserve(object_cache* cache, size_t objectCount)
{
return B_OK;
}
void*
object_cache_alloc(object_cache* cache, uint32 flags)
{
void* object = sHeap->memalign(cache->alignment, cache->object_size, flags);
if (object == NULL)
return NULL;
if (cache->constructor != NULL)
cache->constructor(cache->cookie, object);
return object;
}
void
object_cache_free(object_cache* cache, void* object, uint32 flags)
{
if (cache->destructor != NULL)
cache->destructor(cache->cookie, object);
return sHeap->free(object, flags);
}
status_t
object_cache_reserve(object_cache* cache, size_t objectCount, uint32 flags)
{
return B_OK;
}
void
object_cache_get_usage(object_cache* cache, size_t* _allocatedMemory)
{
*_allocatedMemory = 0;
}
void
request_memory_manager_maintenance()
{
}
void
slab_init(kernel_args* args)
{
}
void
slab_init_post_area()
{
}
void
slab_init_post_sem()
{
}
void
slab_init_post_thread()
{
}
#endif // USE_DEBUG_HEAPS_FOR_OBJECT_CACHE
#endif // DEBUG_HEAPS
src/system/kernel/debug/heaps.h
+32
View File
@@ -0,0 +1,32 @@
/*
* Copyright 2026, Haiku, Inc. All rights reserved.
* Distributed under terms of the MIT License.
*/
#ifndef _KERNEL_DEBUG_HEAPS_H
#define _KERNEL_DEBUG_HEAPS_H
#include <OS.h>
struct kernel_heap_implementation {
const char* name;
const size_t initial_size;
const size_t grow_size;
status_t (*init)(struct kernel_args* args, addr_t base, size_t size);
status_t (*init_post_area)();
status_t (*init_post_sem)();
status_t (*init_post_thread)();
void* (*memalign)(size_t alignment, size_t size, uint32 flags);
void* (*realloc)(void* address, size_t newSize, uint32 flags);
void (*free)(void* address, uint32 flags);
};
extern kernel_heap_implementation kernel_slab_heap;
extern kernel_heap_implementation kernel_guarded_heap;
extern kernel_heap_implementation kernel_debug_heap;
#endif // _KERNEL_DEBUG_HEAPS_H
src/system/kernel/slab/MemoryManager.cpp
+1 -1
View File
@@ -468,7 +468,7 @@ MemoryManager::Init(kernel_args* args)
sFreeAreaCount = 0;
sMaintenanceNeeded = false;
#if USE_DEBUG_HEAP_FOR_MALLOC || USE_GUARDED_HEAP_FOR_MALLOC
#if DEBUG_HEAPS
// Allocate one area immediately. Otherwise, we might try to allocate before
// post-area initialization but after page initialization, during which time
// we can't actually reserve pages.
src/system/kernel/slab/Slab.cpp
+2 -6
View File
@@ -35,7 +35,7 @@
#include "slab_private.h"
#if !USE_GUARDED_HEAP_FOR_OBJECT_CACHE
#if !USE_DEBUG_HEAPS_FOR_OBJECT_CACHE
typedef DoublyLinkedList<ObjectCache> ObjectCacheList;
@@ -1372,8 +1372,6 @@ slab_init(kernel_args* args)
MemoryManager::Init(args);
new (&sObjectCaches) ObjectCacheList();
block_allocator_init_boot();
}
@@ -1431,8 +1429,6 @@ slab_init_post_sem()
register_low_resource_handler(object_cache_low_memory, NULL,
B_KERNEL_RESOURCE_PAGES | B_KERNEL_RESOURCE_MEMORY
| B_KERNEL_RESOURCE_ADDRESS_SPACE, 5);
block_allocator_init_rest();
}
@@ -1457,4 +1453,4 @@ slab_init_post_thread()
RANGE_MARKER_FUNCTION_END(Slab)
#endif // !USE_GUARDED_HEAP_FOR_OBJECT_CACHE
#endif // !USE_DEBUG_HEAPS_FOR_OBJECT_CACHE
src/system/kernel/slab/allocator.cpp
+92 -56
View File
@@ -23,7 +23,15 @@
#include "MemoryManager.h"
#if USE_SLAB_ALLOCATOR_FOR_MALLOC
#if !USE_DEBUG_HEAPS_FOR_OBJECT_CACHE
#if DEBUG_HEAPS
#include "../debug/heaps.h"
#define SLAB_PUBLIC_NAME(NAME) slab_##NAME
#else
#define SLAB_PUBLIC_NAME(NAME) NAME
#endif
//#define TEST_ALL_CACHES_DURING_BOOT
@@ -170,8 +178,13 @@ block_free(void* block, uint32 flags)
}
void
block_allocator_init_boot()
#if DEBUG_HEAPS
status_t
slab_heap_init(struct kernel_args*, addr_t, size_t)
#else
status_t
heap_init(struct kernel_args*)
#endif
{
for (size_t index = 0; index < kNumBlockSizes; index++) {
char name[32];
@@ -194,11 +207,13 @@ block_allocator_init_boot()
if (sBlockCaches[index] == NULL)
panic("allocator: failed to init block cache");
}
return B_OK;
}
void
block_allocator_init_rest()
status_t
SLAB_PUBLIC_NAME(heap_init_post_sem)()
{
#ifdef TEST_ALL_CACHES_DURING_BOOT
for (int index = 0; kBlockSizes[index] != 0; index++) {
@@ -206,38 +221,23 @@ block_allocator_init_rest()
0);
}
#endif
return B_OK;
}
// #pragma mark - public API
void*
memalign(size_t alignment, size_t size)
{
return block_alloc(size, alignment, 0);
}
void *
memalign_etc(size_t alignment, size_t size, uint32 flags)
SLAB_PUBLIC_NAME(memalign_etc)(size_t alignment, size_t size, uint32 flags)
{
return block_alloc(size, alignment, flags & CACHE_ALLOC_FLAGS);
}
int
posix_memalign(void** _pointer, size_t alignment, size_t size)
{
if ((alignment & (sizeof(void*) - 1)) != 0 || _pointer == NULL)
return B_BAD_VALUE;
*_pointer = block_alloc(size, alignment, 0);
return 0;
}
void
free_etc(void *address, uint32 flags)
SLAB_PUBLIC_NAME(free_etc)(void *address, uint32 flags)
{
if ((flags & CACHE_DONT_LOCK_KERNEL_SPACE) != 0) {
deferred_free(address);
@@ -249,21 +249,7 @@ free_etc(void *address, uint32 flags)
void*
malloc(size_t size)
{
return block_alloc(size, 0, 0);
}
void
free(void* address)
{
block_free(address, 0);
}
void*
realloc_etc(void* address, size_t newSize, uint32 flags)
SLAB_PUBLIC_NAME(realloc_etc)(void* address, size_t newSize, uint32 flags)
{
if (newSize == 0) {
block_free(address, flags);
@@ -295,6 +281,41 @@ realloc_etc(void* address, size_t newSize, uint32 flags)
}
#if DEBUG_HEAPS
kernel_heap_implementation kernel_slab_heap = {
"slab_heap",
0, 0,
slab_heap_init,
NULL,
slab_heap_init_post_sem,
NULL,
slab_memalign_etc,
slab_realloc_etc,
slab_free_etc,
};
#else
void*
malloc(size_t size)
{
return block_alloc(size, 0, 0);
}
void
free(void* address)
{
block_free(address, 0);
}
void*
realloc(void* address, size_t newSize)
{
@@ -302,7 +323,37 @@ realloc(void* address, size_t newSize)
}
#else
void*
memalign(size_t alignment, size_t size)
{
return block_alloc(size, alignment, 0);
}
int
posix_memalign(void** _pointer, size_t alignment, size_t size)
{
if ((alignment & (sizeof(void*) - 1)) != 0 || _pointer == NULL)
return B_BAD_VALUE;
*_pointer = block_alloc(size, alignment, 0);
return 0;
}
status_t
heap_init_post_thread()
{
return B_OK;
}
#endif
RANGE_MARKER_FUNCTION_END(slab_allocator)
#else // USE_DEBUG_HEAPS_FOR_OBJECT_CACHE
void*
@@ -313,19 +364,4 @@ block_alloc_early(size_t size)
}
void
block_allocator_init_boot()
{
}
void
block_allocator_init_rest()
{
}
#endif // USE_SLAB_ALLOCATOR_FOR_MALLOC
RANGE_MARKER_FUNCTION_END(slab_allocator)
#endif // USE_DEBUG_HEAPS_FOR_OBJECT_CACHE
src/system/kernel/slab/slab_private.h
-2
View File
@@ -20,8 +20,6 @@ static const size_t kMinObjectAlignment = 8;
void request_memory_manager_maintenance();
void* block_alloc_early(size_t size);
void block_allocator_init_boot();
void block_allocator_init_rest();
static inline void*
src/system/kernel/vm/vm.cpp
+2 -25
View File
@@ -3876,21 +3876,7 @@ vm_init(kernel_args* args)
vm_page_init_num_pages(args);
slab_init(args);
#if USE_DEBUG_HEAP_FOR_MALLOC || USE_GUARDED_HEAP_FOR_MALLOC
size_t heapSize = INITIAL_HEAP_SIZE;
// try to accomodate low memory systems
while (heapSize > (vm_page_num_pages() * B_PAGE_SIZE) / 8)
heapSize /= 2;
if (heapSize < 1024 * 1024)
panic("vm_init: go buy some RAM please.");
// map in the new heap and initialize it
addr_t heapBase = vm_allocate_early(args, heapSize, heapSize,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0);
TRACE(("heap at 0x%lx\n", heapBase));
heap_init(heapBase, heapSize);
#endif
heap_init(args);
// initialize the free page list and physical page mapper
vm_page_init(args);
@@ -3907,7 +3893,7 @@ vm_init(kernel_args* args)
VMAddressSpace::Init();
reserve_boot_loader_ranges(args);
#if USE_DEBUG_HEAP_FOR_MALLOC || USE_GUARDED_HEAP_FOR_MALLOC
#if DEBUG_HEAPS
heap_init_post_area();
#endif
@@ -3920,12 +3906,6 @@ vm_init(kernel_args* args)
// allocate areas to represent stuff that already exists
#if USE_DEBUG_HEAP_FOR_MALLOC
address = (void*)ROUNDDOWN(heapBase, B_PAGE_SIZE);
create_area("kernel heap", &address, B_EXACT_ADDRESS, heapSize,
B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
#endif
allocate_kernel_args(args);
create_preloaded_image_areas(args->kernel_image);
@@ -4006,10 +3986,7 @@ vm_init_post_sem(kernel_args* args)
arch_vm_translation_map_init_post_sem(args);
slab_init_post_sem();
#if USE_DEBUG_HEAP_FOR_MALLOC || USE_GUARDED_HEAP_FOR_MALLOC
heap_init_post_sem();
#endif
return B_OK;
}