BMessage: Allocate more temporary data on the stack if possible.

Some methods already did, but used a very large size of 16KB.
Reduce that to 4KB, do it in more places, and use BStackOrHeapArray
where possible.

Change-Id: Ia2d64582e76da6c1850e502107c07cb10bcd8226
Reviewed-on: https://review.haiku-os.org/c/haiku/+/8969
Reviewed-by: waddlesplash <[email protected]>

src/kits/app/Message.cpp

@@ -28,6 +28,7 @@
 #include <Point.h>
 #include <String.h>
 #include <StringList.h>
+#include <StackOrHeapArray.h>
 
 #include <assert.h>
 #include <ctype.h>
@@ -114,6 +115,7 @@ handle_reply(port_id replyPort, int32* _code, bigtime_t timeout,
 	BMessage* reply)
 {
 	DEBUG_FUNCTION_ENTER2;
+
 	ssize_t size;
 	do {
 		size = port_buffer_size_etc(replyPort, B_RELATIVE_TIMEOUT, timeout);
@@ -122,22 +124,19 @@ handle_reply(port_id replyPort, int32* _code, bigtime_t timeout,
 	if (size < 0)
 		return size;
 
-	status_t result;
+	BStackOrHeapArray<char, 4096> buffer(size);
-	char* buffer = (char*)malloc(size);
+	if (!buffer.IsValid())
-	if (buffer == NULL)
 		return B_NO_MEMORY;
 
+	status_t result;
 	do {
 		result = read_port(replyPort, _code, buffer, size);
 	} while (result == B_INTERRUPTED);
 
-	if (result < 0 || *_code != kPortMessageCode) {
+	if (result < 0 || *_code != kPortMessageCode)
-		free(buffer);
 		return result < 0 ? result : B_ERROR;
-	}
 
 	result = reply->Unflatten(buffer);
-	free(buffer);
 	return result;
 }
 
@@ -2133,7 +2132,9 @@ BMessage::_SendMessage(port_id port, team_id portOwner, int32 token,
 	bigtime_t timeout, bool replyRequired, BMessenger& replyTo) const
 {
 	DEBUG_FUNCTION_ENTER;
+
 	ssize_t size = 0;
+	char stackBuffer[4096];
 	char* buffer = NULL;
 	message_header* header = NULL;
 	status_t result = B_OK;
@@ -2199,13 +2200,17 @@ BMessage::_SendMessage(port_id port, team_id portOwner, int32 token,
 #endif
 	} else {
 		size = FlattenedSize();
-		buffer = (char*)malloc(size);
+		if (size > (ssize_t)sizeof(stackBuffer)) {
-		if (buffer == NULL)
+			buffer = (char*)malloc(size);
-			return B_NO_MEMORY;
+			if (buffer == NULL)
+				return B_NO_MEMORY;
+		} else
+			buffer = stackBuffer;
 
 		result = Flatten(buffer, size);
 		if (result != B_OK) {
-			free(buffer);
+			if (buffer != stackBuffer)
+				free(buffer);
 			return result;
 		}
 
@@ -2269,7 +2274,8 @@ BMessage::_SendMessage(port_id port, team_id portOwner, int32 token,
 		direct->Release();
 	}
 
-	free(buffer);
+	if (buffer != stackBuffer)
+		free(buffer);
 	return result;
 }
 
@@ -2751,25 +2757,16 @@ BMessage::AddMessage(const char* name, const BMessage* message)
 	// copying an extra buffer. Functions can be added that return a direct
 	// pointer into the message.
 
-	char stackBuffer[16384];
 	ssize_t size = message->FlattenedSize();
-
+	BStackOrHeapArray<char, 4096> buffer(size);
-	char* buffer;
+	if (!buffer.IsValid())
-	if (size > (ssize_t)sizeof(stackBuffer)) {
+		return B_NO_MEMORY;
-		buffer = (char*)malloc(size);
-		if (buffer == NULL)
-			return B_NO_MEMORY;
-	} else
-		buffer = stackBuffer;
 
 	status_t error = message->Flatten(buffer, size);
 
 	if (error >= B_OK)
 		error = AddData(name, B_MESSAGE_TYPE, buffer, size, false);
 
-	if (buffer != stackBuffer)
-		free(buffer);
-
 	return error;
 }
 
@@ -2787,25 +2784,16 @@ BMessage::AddFlat(const char* name, const BFlattenable* object, int32 count)
 	if (object == NULL)
 		return B_BAD_VALUE;
 
-	char stackBuffer[16384];
 	ssize_t size = object->FlattenedSize();
-
+	BStackOrHeapArray<char, 4096> buffer(size);
-	char* buffer;
+	if (!buffer.IsValid())
-	if (size > (ssize_t)sizeof(stackBuffer)) {
+		return B_NO_MEMORY;
-		buffer = (char*)malloc(size);
-		if (buffer == NULL)
-			return B_NO_MEMORY;
-	} else
-		buffer = stackBuffer;
 
 	status_t error = object->Flatten(buffer, size);
 
 	if (error >= B_OK)
 		error = AddData(name, object->TypeCode(), buffer, size, false);
 
-	if (buffer != stackBuffer)
-		free(buffer);
-
 	return error;
 }