ltrace: Place stubs in exectable memory and improve efficiency.

* Implement the TODO about needing to place stubs in executable
   memory. This fixes basic ltrace operations.

 * Place traces behind a TRACE ifdef.

 * Get rid of the "patched table" and find the PatchEntry
   using fixed offsets, to improve efficiency.

 * Add a "bool trace" to PatchEntry. This paves the way for
   tracing only specific methods.

Using LD_PRELOAD_ADDONS=...ltrace_stub.so, trace entries are now
printed on x86_64. However, it seems that the trace stub is clobbering
something, as BWindows don't work correctly when traced.

src/bin/debug/ltrace/ltrace_stub.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright 2008, Ingo Weinhold, [email protected].
+ * Copyright 2026, Haiku, Inc. All rights reserved.
  * Distributed under the terms of the MIT License.
  */
 
@@ -13,43 +14,112 @@
 #include <runtime_loader.h>
 #include <util/OpenHashTable.h>
 
+#include <syscalls.h>
+
 #include "arch/ltrace_stub.h"
 
 
-//#define TRACE_PRINTF debug_printf
+//#define TRACE_STUB
-#define TRACE_PRINTF ktrace_printf
+#ifdef TRACE_STUB
+#   define TRACE(x...) ktrace_printf(x)
+#else
+#   define TRACE(x...) ;
+#endif
 
 
 static void* function_call_callback(const void* stub, const void* args);
 
 
-struct PatchEntry {
+// TODO: Per-image_t patch memory (so it can be freed,
-	PatchEntry*	originalTableLink;
+// or marked RX-only after symbol resolution completes?)
-	PatchEntry*	patchedTableLink;
+static const size_t kPatchMemoryChunkSize = B_PAGE_SIZE * 16;
+static const size_t kPatchMemoryReserveSize = 1 * 1024 * 1024;
+static uint8* sPatchMemoryChunk = NULL;
+static size_t sPatchMemoryChunkRemaining = 0;
+static area_id sPatchMemoryChunkArea = 0;
+static size_t sPatchMemoryChunkAreaSize = 0;
 
-	void*		originalFunction;
+
-	void*		patchedFunction;
+static uint8*
-	const char*	functionName;
+patch_malloc(size_t size)
+{
+	if (sPatchMemoryChunkRemaining < size) {
+		status_t status = B_NO_MEMORY;
+		if (sPatchMemoryChunkArea != 0) {
+			// Try to resize the previous area instead of creating a new one.
+			status = _kern_resize_area(sPatchMemoryChunkArea,
+				sPatchMemoryChunkAreaSize + kPatchMemoryChunkSize);
+			if (status == B_OK) {
+				sPatchMemoryChunkAreaSize += kPatchMemoryChunkSize;
+				sPatchMemoryChunkRemaining += kPatchMemoryChunkSize;
+			}
+		}
+		if (status != B_OK) {
+			void* reservedBase;
+			status = _kern_reserve_address_range((addr_t*)&reservedBase,
+				B_RANDOMIZED_ANY_ADDRESS, kPatchMemoryReserveSize);
+			if (status != B_OK)
+				return NULL;
+
+			void* base = reservedBase;
+			area_id area = _kern_create_area("ltrace patches", &base,
+				B_EXACT_ADDRESS, size, B_NO_LOCK,
+				B_READ_AREA | B_WRITE_AREA | B_EXECUTE_AREA);
+			if (area < 0) {
+				_kern_unreserve_address_range((addr_t)reservedBase,
+					kPatchMemoryReserveSize);
+				return NULL;
+			}
+
+			sPatchMemoryChunk = (uint8*)base;
+			sPatchMemoryChunkArea = area;
+			sPatchMemoryChunkRemaining = sPatchMemoryChunkAreaSize = size;
+		}
+	}
+
+	uint8* allocation = sPatchMemoryChunk;
+	sPatchMemoryChunk += size;
+	sPatchMemoryChunkRemaining -= size;
+	return allocation;
+}
+
+
+struct PatchEntry {
+	PatchEntry*	original_table_link;
+
+	void*		original_function;
+	const char*	function_name;
+
+	bool		trace;
 
 	static PatchEntry* Create(const char* name, void* function)
 	{
-		// TODO memory should be executable, use mmap with PROT_EXEC
+		void* memory = patch_malloc(_ALIGN(sizeof(PatchEntry))
-		void* memory = malloc(_ALIGN(sizeof(PatchEntry))
 			+ arch_call_stub_size());
 		if (memory == NULL)
 			return NULL;
 
 		PatchEntry* entry = new(memory) PatchEntry;
 
-		void* stub = (uint8*)memory + _ALIGN(sizeof(PatchEntry));
+		void* stub = (uint8*)memory + OffsetToStub();
 		arch_init_call_stub(stub, &function_call_callback, function);
 
-		entry->originalFunction = function;
+		entry->original_function = function;
-		entry->patchedFunction = stub;
+		entry->function_name = name;
-		entry->functionName = name;
+		entry->trace = true;
 
 		return entry;
 	}
+
+	static size_t OffsetToStub()
+	{
+		return _ALIGN(sizeof(PatchEntry));
+	}
+
+	void* Stub()
+	{
+		return (uint8*)this + OffsetToStub();
+	}
 };
 
 
@@ -64,43 +134,17 @@ struct OriginalTableDefinition {
 
 	size_t Hash(PatchEntry* value) const
 	{
-		return HashKey(value->originalFunction);
+		return HashKey(value->original_function);
 	}
 
 	bool Compare(const void* key, PatchEntry* value) const
 	{
-		return value->originalFunction == key;
+		return value->original_function == key;
 	}
 
 	PatchEntry*& GetLink(PatchEntry* value) const
 	{
-		return value->originalTableLink;
+		return value->original_table_link;
-	}
-};
-
-
-struct PatchedTableDefinition {
-	typedef const void*	KeyType;
-	typedef PatchEntry	ValueType;
-
-	size_t HashKey(const void* key) const
-	{
-		return (addr_t)key >> 2;
-	}
-
-	size_t Hash(PatchEntry* value) const
-	{
-		return HashKey(value->patchedFunction);
-	}
-
-	bool Compare(const void* key, PatchEntry* value) const
-	{
-		return value->patchedFunction == key;
-	}
-
-	PatchEntry*& GetLink(PatchEntry* value) const
-	{
-		return value->patchedTableLink;
 	}
 };
 
@@ -109,18 +153,15 @@ static rld_export* sRuntimeLoaderInterface;
 static runtime_loader_add_on_export* sRuntimeLoaderAddOnInterface;
 
 static BOpenHashTable<OriginalTableDefinition> sOriginalTable;
-static BOpenHashTable<PatchedTableDefinition> sPatchedTable;
 
 
 static void*
 function_call_callback(const void* stub, const void* _args)
 {
-	PatchEntry* entry = sPatchedTable.Lookup(stub);
+	PatchEntry* entry = (PatchEntry*)((uint8*)stub - PatchEntry::OffsetToStub());
-	if (entry == NULL)
+
-{
+	if (!entry->trace)
-TRACE_PRINTF("function_call_callback(): CALLED FOR UNKNOWN FUNCTION!\n");
+		return entry->original_function;
-		return NULL;
-}
 
 	char buffer[1024];
 	size_t bufferSize = sizeof(buffer);
@@ -128,15 +169,15 @@ TRACE_PRINTF("function_call_callback(): CALLED FOR UNKNOWN FUNCTION!\n");
 
 	const ulong* args = (const ulong*)_args;
 	written += snprintf(buffer, bufferSize, "ltrace: %s(",
-		entry->functionName);
+		entry->function_name);
 	for (int32 i = 0; i < 5; i++) {
 		written += snprintf(buffer + written, bufferSize - written, "%s%#lx",
 			i == 0 ? "" : ", ", args[i]);
 	}
-	written += snprintf(buffer + written, bufferSize - written, ")");
+	written += snprintf(buffer + written, bufferSize - written, ")\n");
-	TRACE_PRINTF("%s\n", buffer);
+	write(0, buffer, written);
 
-	return entry->originalFunction;
+	return entry->original_function;
 }
 
 
@@ -144,7 +185,7 @@ static void
 symbol_patcher(void* cookie, image_t* rootImage, image_t* image,
 	const char* name, image_t** foundInImage, void** symbol, int32* type)
 {
-	TRACE_PRINTF("symbol_patcher(%p, %p, %p, \"%s\", %p, %p, %" B_PRId32 ")\n",
+	TRACE("symbol_patcher(%p, %p, %p, \"%s\", %p, %p, %" B_PRId32 ")\n",
 		cookie, rootImage, image, name, *foundInImage, *symbol, *type);
 
 	// patch functions only
@@ -155,7 +196,7 @@ symbol_patcher(void* cookie, image_t* rootImage, image_t* image,
 	PatchEntry* entry = sOriginalTable.Lookup(*symbol);
 	if (entry != NULL) {
 		*foundInImage = NULL;
-		*symbol = entry->patchedFunction;
+		*symbol = entry->Stub();
 		return;
 	}
 
@@ -164,12 +205,11 @@ symbol_patcher(void* cookie, image_t* rootImage, image_t* image,
 		return;
 
 	sOriginalTable.Insert(entry);
-	sPatchedTable.Insert(entry);
 
-	TRACE_PRINTF("  -> patching to %p\n", entry->patchedFunction);
+	TRACE("  -> patching to %p\n", entry->Stub());
 
 	*foundInImage = NULL;
-	*symbol = entry->patchedFunction;
+	*symbol = entry->Stub();
 }
 
 
@@ -177,24 +217,23 @@ static void
 ltrace_stub_init(rld_export* standardInterface,
 	runtime_loader_add_on_export* addOnInterface)
 {
-	TRACE_PRINTF("ltrace_stub_init(%p, %p)\n", standardInterface, addOnInterface);
+	TRACE("ltrace_stub_init(%p, %p)\n", standardInterface, addOnInterface);
 	sRuntimeLoaderInterface = standardInterface;
 	sRuntimeLoaderAddOnInterface = addOnInterface;
 
 	sOriginalTable.Init();
-	sPatchedTable.Init();
 }
 
 
 static void
 ltrace_stub_image_loaded(image_t* image)
 {
-	TRACE_PRINTF("ltrace_stub_image_loaded(%p): \"%s\" (%" B_PRId32 ")\n",
+	TRACE("ltrace_stub_image_loaded(%p): \"%s\" (%" B_PRId32 ")\n",
 		image, image->path, image->id);
 
 	if (sRuntimeLoaderAddOnInterface->register_undefined_symbol_patcher(image,
 			symbol_patcher, (void*)(addr_t)0xc0011eaf) != B_OK) {
-		TRACE_PRINTF("  failed to install symbol patcher\n");
+		TRACE("  failed to install symbol patcher\n");
 	}
 }
 
@@ -202,7 +241,7 @@ ltrace_stub_image_loaded(image_t* image)
 static void
 ltrace_stub_image_relocated(image_t* image)
 {
-	TRACE_PRINTF("ltrace_stub_image_relocated(%p): \"%s\" (%" B_PRId32 ")\n",
+	TRACE("ltrace_stub_image_relocated(%p): \"%s\" (%" B_PRId32 ")\n",
 		image, image->path, image->id);
 }
 
@@ -210,7 +249,7 @@ ltrace_stub_image_relocated(image_t* image)
 static void
 ltrace_stub_image_initialized(image_t* image)
 {
-	TRACE_PRINTF("ltrace_stub_image_initialized(%p): \"%s\" (%" B_PRId32 ")\n",
+	TRACE("ltrace_stub_image_initialized(%p): \"%s\" (%" B_PRId32 ")\n",
 		image, image->path, image->id);
 }
 
@@ -218,7 +257,7 @@ ltrace_stub_image_initialized(image_t* image)
 static void
 ltrace_stub_image_uninitializing(image_t* image)
 {
-	TRACE_PRINTF("ltrace_stub_image_uninitializing(%p): \"%s\" (%" B_PRId32
+	TRACE("ltrace_stub_image_uninitializing(%p): \"%s\" (%" B_PRId32
 		")\n",image, image->path, image->id);
 }
 
@@ -226,7 +265,7 @@ ltrace_stub_image_uninitializing(image_t* image)
 static void
 ltrace_stub_image_unloading(image_t* image)
 {
-	TRACE_PRINTF("ltrace_stub_image_unloading(%p): \"%s\" (%" B_PRId32 ")\n",
+	TRACE("ltrace_stub_image_unloading(%p): \"%s\" (%" B_PRId32 ")\n",
 		image, image->path, image->id);
 }