boot loader: Optimize heap implementation

* Increase general allocation alignment from 4 to 8 byte. That was even
  incorrect.
* Use a splay tree instead of a singly linked list to manage the free
  chunks. That increases the size of the per-chunk structure to manage
  the free chunks, i.e. the of minimally allocatable memory size (from
  align(sizeof(void*)) to align(3 * sizeof(void*))), but make finding
  and inserting chunks much faster.

Fixes #10063 respectively improves the situation significantly.

src/system/boot/loader/heap.cpp

@@ -5,9 +5,6 @@
 
 
 #include <boot/heap.h>
-#include <boot/platform.h>
-
-#include <algorithm>
 
 #ifdef HEAP_TEST
 #	include <stdio.h>
@@ -15,6 +12,11 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <algorithm>
+
+#include <boot/platform.h>
+#include <util/SplayTree.h>
+
 
 //#define TRACE_HEAP
 #ifdef TRACE_HEAP
@@ -46,72 +48,169 @@
 	// if defined, the maximum heap usage is determined and printed before
 	// entering the kernel
 
-class FreeChunk {
+
+const static size_t kAlignment = 8;
+	// all memory chunks will be a multiple of this
+
+
+class Chunk {
 public:
-			void				SetTo(size_t size, FreeChunk* next);
+	size_t CompleteSize() const
+	{
+		return fSize;
+	}
 
-			uint32				Size() const;
+protected:
-			uint32				CompleteSize() const { return fSize; }
+	union {
-
+		size_t	fSize;
-			FreeChunk*			Next() const { return fNext; }
+		char	fAlignment[kAlignment];
-			void				SetNext(FreeChunk* next) { fNext = next; }
+	};
-
-			FreeChunk*			Split(uint32 splitSize);
-			bool				IsTouching(FreeChunk* link);
-			FreeChunk*			Join(FreeChunk* link);
-			void				Remove(FreeChunk* previous = NULL);
-			void				Enqueue();
-
-			void*				AllocatedAddress() const;
-	static	FreeChunk*			SetToAllocated(void* allocated);
-	static	addr_t				NextOffset() { return sizeof(uint32); }
-
-private:
-			uint32				fSize;
-			FreeChunk*			fNext;
 };
 
 
-const static uint32 kAlignment = 4;
+class FreeChunk;
-	// all memory chunks will be a multiple of this
+
+
+struct FreeChunkData : SplayTreeLink<FreeChunk> {
+
+	FreeChunk* Next() const
+	{
+		return fNext;
+	}
+
+	FreeChunk** NextLink()
+	{
+		return &fNext;
+	}
+
+protected:
+	FreeChunk*	fNext;
+};
+
+
+class FreeChunk : public Chunk, public FreeChunkData {
+public:
+			void				SetTo(size_t size);
+
+			size_t				Size() const;
+
+			FreeChunk*			Split(size_t splitSize);
+			bool				IsTouching(FreeChunk* link);
+			FreeChunk*			Join(FreeChunk* link);
+
+			void*				AllocatedAddress() const;
+	static	FreeChunk*			SetToAllocated(void* allocated);
+};
+
+
+struct FreeChunkKey {
+	FreeChunkKey(size_t size)
+		:
+		fSize(size),
+		fChunk(NULL)
+	{
+	}
+
+	FreeChunkKey(const FreeChunk* chunk)
+		:
+		fSize(chunk->Size()),
+		fChunk(chunk)
+	{
+	}
+
+	int Compare(const FreeChunk* chunk) const
+	{
+		size_t chunkSize = chunk->Size();
+		if (chunkSize != fSize)
+			return fSize < chunkSize ? -1 : 1;
+
+		if (fChunk == chunk)
+			return 0;
+		return fChunk < chunk ? -1 : 1;
+	}
+
+private:
+	size_t				fSize;
+	const FreeChunk*	fChunk;
+};
+
+
+struct FreeChunkTreeDefinition {
+	typedef FreeChunkKey	KeyType;
+	typedef FreeChunk		NodeType;
+
+	static FreeChunkKey GetKey(const FreeChunk* node)
+	{
+		return FreeChunkKey(node);
+	}
+
+	static SplayTreeLink<FreeChunk>* GetLink(FreeChunk* node)
+	{
+		return node;
+	}
+
+	static int Compare(const FreeChunkKey& key, const FreeChunk* node)
+	{
+		return key.Compare(node);
+	}
+
+	static FreeChunk** GetListLink(FreeChunk* node)
+	{
+		return node->NextLink();
+	}
+};
+
+
+typedef IteratableSplayTree<FreeChunkTreeDefinition> FreeChunkTree;
+
 
 static void* sHeapBase;
-static uint32 /*sHeapSize,*/ sMaxHeapSize, sAvailable, sMaxHeapUsage;
+static size_t /*sHeapSize,*/ sMaxHeapSize, sAvailable, sMaxHeapUsage;
-static FreeChunk sFreeAnchor;
+static FreeChunkTree sFreeChunkTree;
+
+static uint64 sTotalMallocCycles = 0;
+static uint64 sTotalFreeCycles = 0;
+
+
+static inline size_t
+align(size_t size)
+{
+	return (size + kAlignment - 1) & ~(kAlignment - 1);
+}
 
 
 void
-FreeChunk::SetTo(size_t size, FreeChunk* next)
+FreeChunk::SetTo(size_t size)
 {
 	fSize = size;
-	fNext = next;
+	fNext = NULL;
 }
 
 
 /*!	Returns the amount of bytes that can be allocated
 	in this chunk.
 */
-uint32
+size_t
 FreeChunk::Size() const
 {
-	return fSize - FreeChunk::NextOffset();
+	return (addr_t)this + fSize - (addr_t)AllocatedAddress();
 }
 
 
-/*!	Splits the upper half at the requested location
+/*!	Splits the upper half at the requested location and returns it. This chunk
-	and returns it.
+	will no longer be a valid FreeChunk object; only its fSize will be valid.
-*/
+ */
 FreeChunk*
-FreeChunk::Split(uint32 splitSize)
+FreeChunk::Split(size_t splitSize)
 {
-	splitSize = (splitSize - 1 + kAlignment) & ~(kAlignment - 1);
+	splitSize = align(splitSize);
 
-	FreeChunk* chunk
+	FreeChunk* chunk = (FreeChunk*)((addr_t)AllocatedAddress() + splitSize);
-		= (FreeChunk*)((uint8*)this + FreeChunk::NextOffset() + splitSize);
+	size_t newSize = (addr_t)chunk - (addr_t)this;
-	chunk->fSize = fSize - splitSize - FreeChunk::NextOffset();
+	chunk->fSize = fSize - newSize;
-	chunk->fNext = fNext;
+	chunk->fNext = NULL;
 
-	fSize = splitSize + FreeChunk::NextOffset();
+	fSize = newSize;
 
 	return chunk;
 }
@@ -153,58 +252,17 @@ FreeChunk::Join(FreeChunk* chunk)
 }
 
 
-void
-FreeChunk::Remove(FreeChunk* previous)
-{
-	if (previous == NULL) {
-		// find the previous chunk in the list
-		FreeChunk* chunk = sFreeAnchor.fNext;
-
-		while (chunk != NULL && chunk != this) {
-			previous = chunk;
-			chunk = chunk->fNext;
-		}
-
-		if (chunk == NULL)
-			panic("try to remove chunk that's not in list");
-	}
-
-	previous->fNext = fNext;
-	fNext = NULL;
-}
-
-
-void
-FreeChunk::Enqueue()
-{
-	FreeChunk* chunk = sFreeAnchor.fNext;
-	FreeChunk* last = &sFreeAnchor;
-	while (chunk && chunk->Size() < fSize) {
-		last = chunk;
-		chunk = chunk->fNext;
-	}
-
-	fNext = chunk;
-	last->fNext = this;
-
-#ifdef DEBUG_ALLOCATIONS
-	memset((uint8*)this + sizeof(FreeChunk), 0xde,
-		fSize - sizeof(FreeChunk));
-#endif
-}
-
-
 void*
 FreeChunk::AllocatedAddress() const
 {
-	return (void*)&fNext;
+	return (void*)static_cast<const FreeChunkData*>(this);
 }
 
 
 FreeChunk*
 FreeChunk::SetToAllocated(void* allocated)
 {
-	return (FreeChunk*)((uint8*)allocated - FreeChunk::NextOffset());
+	return static_cast<FreeChunk*>((FreeChunkData*)allocated);
 }
 
 
@@ -237,17 +295,17 @@ heap_init(stage2_args* args)
 
 	sHeapBase = base;
 	sMaxHeapSize = (uint8*)top - (uint8*)base;
-	sAvailable = sMaxHeapSize - FreeChunk::NextOffset();
-#ifdef DEBUG_MAX_HEAP_USAGE
-	sMaxHeapUsage = sMaxHeapSize - sAvailable;
-#endif
 
 	// declare the whole heap as one chunk, and add it
 	// to the free list
-
 	FreeChunk* chunk = (FreeChunk*)base;
-	chunk->SetTo(sMaxHeapSize, NULL);
+	chunk->SetTo(sMaxHeapSize);
-	sFreeAnchor.SetTo(0, chunk);
+	sFreeChunkTree.Insert(chunk);
+
+	sAvailable = chunk->Size();
+#ifdef DEBUG_MAX_HEAP_USAGE
+	sMaxHeapUsage = sMaxHeapSize - sAvailable;
+#endif
 
 	return B_OK;
 }
@@ -274,11 +332,8 @@ grow_heap(uint32 bytes)
 void
 dump_chunks(void)
 {
-	FreeChunk* chunk = sFreeAnchor.Next();
+	FreeChunk* chunk = sFreeChunkTree.FindMin();
-	FreeChunk* last = &sFreeAnchor;
 	while (chunk != NULL) {
-		last = chunk;
-
 		printf("\t%p: chunk size = %ld, end = %p, next = %p\n", chunk,
 			chunk->Size(), (uint8*)chunk + chunk->CompleteSize(),
 			chunk->Next());
@@ -290,7 +345,7 @@ dump_chunks(void)
 uint32
 heap_available(void)
 {
-	return sAvailable;
+	return (uint32)sAvailable;
 }
 
 
@@ -301,19 +356,17 @@ malloc(size_t size)
 		return NULL;
 
 	// align the size requirement to a kAlignment bytes boundary
-	size = (size - 1 + kAlignment) & ~(size_t)(kAlignment - 1);
+	if (size < sizeof(FreeChunkData))
+		size = sizeof(FreeChunkData);
+	size = align(size);
 
 	if (size > sAvailable) {
 		dprintf("malloc(): Out of memory!\n");
 		return NULL;
 	}
 
-	FreeChunk* chunk = sFreeAnchor.Next();
+	FreeChunk* chunk = sFreeChunkTree.FindClosest(FreeChunkKey(size), true,
-	FreeChunk* last = &sFreeAnchor;
+		true);
-	while (chunk && chunk->Size() < size) {
-		last = chunk;
-		chunk = chunk->Next();
-	}
 
 	if (chunk == NULL) {
 		// could not find a free chunk as large as needed
@@ -321,30 +374,25 @@ malloc(size_t size)
 		return NULL;
 	}
 
-	if (chunk->Size() > size + sizeof(FreeChunk) + kAlignment) {
+	sFreeChunkTree.Remove(chunk);
-		// if this chunk is bigger than the requested size,
+	sAvailable -= chunk->Size();
-		// we split it to form two chunks (with a minimal
-		// size of kAlignment allocatable bytes).
 
+	void* allocatedAddress = chunk->AllocatedAddress();
+
+	// If this chunk is bigger than the requested size and there's enough space
+	// left over for a new chunk, we split it.
+	if (chunk->Size() >= size + align(sizeof(FreeChunk))) {
 		FreeChunk* freeChunk = chunk->Split(size);
-		last->SetNext(freeChunk);
+		sFreeChunkTree.Insert(freeChunk);
-
+		sAvailable += freeChunk->Size();
-		// re-enqueue the free chunk at the correct position
-		freeChunk->Remove(last);
-		freeChunk->Enqueue();
-	} else {
-		// remove the chunk from the free list
-
-		last->SetNext(chunk->Next());
 	}
 
-	sAvailable -= size + sizeof(uint32);
 #ifdef DEBUG_MAX_HEAP_USAGE
 	sMaxHeapUsage = std::max(sMaxHeapUsage, sMaxHeapSize - sAvailable);
 #endif
 
-	TRACE("malloc(%lu) -> %p\n", size, chunk->AllocatedAddress());
+	TRACE("malloc(%lu) -> %p\n", size, allocatedAddress);
-	return chunk->AllocatedAddress();
+	return allocatedAddress;
 }
 
 
@@ -398,54 +446,47 @@ free(void* allocated)
 	FreeChunk* freedChunk = FreeChunk::SetToAllocated(allocated);
 
 #ifdef DEBUG_ALLOCATIONS
-	if (freedChunk->CompleteSize() > sMaxHeapSize) {
+	if (freedChunk->Size() > sMaxHeapSize - sAvailable) {
-		panic("freed chunk %p clobbered (%lx)!\n", freedChunk,
+		panic("freed chunk %p clobbered (%#zx)!\n", freedChunk,
 			freedChunk->Size());
 	}
 	{
-		FreeChunk* chunk = sFreeAnchor.Next();
+		FreeChunk* chunk = sFreeChunkTree.FindMin();
 		while (chunk) {
-			if (chunk->CompleteSize() > sMaxHeapSize || freedChunk == chunk)
+			if (chunk->Size() > sAvailable || freedChunk == chunk)
-				panic("invalid chunk in free list, or double free\n");
+				panic("invalid chunk in free list (%p (%zu)), or double free\n",
+					chunk, chunk->Size());
 			chunk = chunk->Next();
 		}
 	}
 #endif
 
-	sAvailable += freedChunk->CompleteSize();
-#ifdef DEBUG_MAX_HEAP_USAGE
-	sMaxHeapUsage = std::max(sMaxHeapUsage, sMaxHeapSize - sAvailable);
-#endif
-
 	// try to join the new free chunk with an existing one
 	// it may be joined with up to two chunks
 
-	FreeChunk* chunk = sFreeAnchor.Next();
+	FreeChunk* chunk = sFreeChunkTree.FindMin();
-	FreeChunk* last = &sFreeAnchor;
 	int32 joinCount = 0;
 
 	while (chunk) {
-		if (chunk->IsTouching(freedChunk)) {
+		FreeChunk* nextChunk = chunk->Next();
-			// almost "insert" it into the list before joining
-			// because the next pointer is inherited by the chunk
-			freedChunk->SetNext(chunk->Next());
-			freedChunk = chunk->Join(freedChunk);
 
-			// remove the joined chunk from the list
+		if (chunk->IsTouching(freedChunk)) {
-			last->SetNext(freedChunk->Next());
+			sFreeChunkTree.Remove(chunk);
-			chunk = last;
+			sAvailable -= chunk->Size();
+
+			freedChunk = chunk->Join(freedChunk);
 
 			if (++joinCount == 2)
 				break;
 		}
 
-		last = chunk;
+		chunk = nextChunk;
-		chunk = chunk->Next();
 	}
 
-	// enqueue the link at the right position; the
+	sFreeChunkTree.Insert(freedChunk);
-	// free link queue is ordered by size
+	sAvailable += freedChunk->Size();
-
+#ifdef DEBUG_MAX_HEAP_USAGE
-	freedChunk->Enqueue();
+	sMaxHeapUsage = std::max(sMaxHeapUsage, sMaxHeapSize - sAvailable);
+#endif
 }