* Fully switch to the Haiku message format so that it is also used for on-disk

storage.
* Remove reduntant information from the header (field_count vs. fields_size).
* Remove checksums previously used to validate the message passing by area
  mechanism.
* Move variables that are purely used by the instance out of the header and into
  the BMessage object.
* Use more sensible types for the different message fields.
* Reduce some field sizes to realistic values.
* Make size_t values into uint32 values so the message format will not change
  when later moving to 64 bits.
* Pack the structures used for flat message storage so it doesn't change
  because of padding.
* Fix message passing by area. It never worked because the created area was
  never actually filled with any data!
* Some more allocation checks with graceful fallbacks (should be all now).
* Some more checks for negative index values (should also be all now).
* Make printing more inline with how the rest of the class works and make some
  of the output more consistent.
* Also add the new unsigned types to PrintToStream() output.
* Fix printing of unknown types and invalid BMessages, it would always have
  printed only the first entry respectively the same error.
* Added some clarifying comments.
* Cleanup.


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@32039 a95241bf-73f2-0310-859d-f6bbb57e9c96

headers/os/app/Message.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2007, Haiku Inc. All Rights Reserved.
+ * Copyright 2005-2009, Haiku Inc. All Rights Reserved.
  * Distributed under the terms of the MIT License.
  *
  * Authors:
@@ -291,10 +291,12 @@ class BMessage {
 
 		status_t		_FlattenToArea(message_header **_header) const;
 		status_t		_CopyForWrite();
-		status_t		_Reference(message_header *header);
+		status_t		_Reference();
 		status_t		_Dereference();
 
-		status_t		_ResizeData(int32 offset, int32 change);
+		status_t		_ValidateMessage();
+
+		status_t		_ResizeData(uint32 offset, int32 change);
 
 		uint32			_HashName(const char* name) const;
 		status_t		_FindField(const char* name, type_code type,
@@ -303,9 +305,6 @@ class BMessage {
 							bool isFixedSize, field_header** _result);
 		status_t		_RemoveField(field_header* field);
 
-		ssize_t			_NativeFlattenedSize() const;
-		status_t		_NativeFlatten(char *buffer, ssize_t size) const;
-		status_t		_NativeFlatten(BDataIO *stream, ssize_t *size = NULL) const;
 		void			_PrintToStream(const char* indent) const;
 
 	private:
@@ -313,12 +312,15 @@ class BMessage {
 		field_header*	fFields;
 		uint8*			fData;
 
+		uint32			fFieldsAvailable;
+		size_t			fDataAvailable;
+
 		mutable	BMessage* fOriginal;
 
 		BMessage*		fQueueLink;
 			// fQueueLink is used by BMessageQueue to build a linked list
 
-		uint32			fReserved[11];
+		uint32			fReserved[9];
 
 						// deprecated
 						BMessage(BMessage *message);

headers/private/app/MessagePrivate.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2007, Haiku Inc. All rights reserved.
+ * Copyright 2005-2009, Haiku Inc. All rights reserved.
  * Distributed under the terms of the MIT License.
  *
  * Authors:
@@ -14,10 +14,9 @@
 #include <TokenSpace.h>
 
 
-#define MESSAGE_BODY_HASH_TABLE_SIZE	10
+#define MESSAGE_BODY_HASH_TABLE_SIZE	5
 #define MAX_DATA_PREALLOCATION			B_PAGE_SIZE * 10
 #define MAX_FIELD_PREALLOCATION			50
-#define MAX_ITEM_PREALLOCATION			B_PAGE_SIZE
 
 
 static const int32 kPortMessageCode = 'pjpp';
@@ -30,8 +29,8 @@ enum {
 	MESSAGE_FLAG_IS_REPLY = 0x0008,
 	MESSAGE_FLAG_WAS_DELIVERED = 0x0010,
 	MESSAGE_FLAG_HAS_SPECIFIERS = 0x0020,
-	MESSAGE_FLAG_WAS_DROPPED = 0x0080,
+	MESSAGE_FLAG_WAS_DROPPED = 0x0040,
-	MESSAGE_FLAG_PASS_BY_AREA = 0x0100
+	MESSAGE_FLAG_PASS_BY_AREA = 0x0080
 };
 
 
@@ -42,15 +41,14 @@ enum {
 
 
 struct BMessage::field_header {
-	uint32		flags;
+	uint16		flags;
+	uint16		name_length;
 	type_code	type;
-	int32		name_length;
+	uint32		count;
-	int32		count;
+	uint32		data_size;
-	ssize_t		data_size;
+	uint32		offset;
-	ssize_t		allocated;
-	int32		offset;
 	int32		next_field;
-};
+} _PACKED;
 
 
 struct BMessage::message_header {
@@ -58,14 +56,6 @@ struct BMessage::message_header {
 	uint32		what;
 	uint32		flags;
 
-	ssize_t		fields_size;
-	ssize_t		data_size;
-	ssize_t		fields_available;
-	ssize_t		data_available;
-
-	uint32		fields_checksum;
-	uint32		data_checksum;
-
 	int32		target;
 	int32		current_specifier;
 	area_id		message_area;
@@ -76,8 +66,9 @@ struct BMessage::message_header {
 	team_id		reply_team;
 
 	// body info
-	int32		field_count;
+	uint32		data_size;
-	int32		hash_table_size;
+	uint32		field_count;
+	uint32		hash_table_size;
 	int32		hash_table[MESSAGE_BODY_HASH_TABLE_SIZE];
 
 	/*	The hash table does contain indexes into the field list and
@@ -86,18 +77,20 @@ struct BMessage::message_header {
 		The hash table must be reevaluated when we remove a field
 		though.
 	*/
-};
+} _PACKED;
 
 
 class BMessage::Private {
 	public:
 		Private(BMessage *msg)
-			: fMessage(msg)
+			:
+			fMessage(msg)
 		{
 		}
 
 		Private(BMessage &msg)
-			: fMessage(&msg)
+			:
+			fMessage(&msg)
 		{
 		}
 
@@ -175,24 +168,6 @@ class BMessage::Private {
 			return fMessage->fData;
 		}
 
-		ssize_t
-		NativeFlattenedSize() const
-		{
-			return fMessage->_NativeFlattenedSize();
-		}
-
-		status_t
-		NativeFlatten(char *buffer, ssize_t size) const
-		{
-			return fMessage->_NativeFlatten(buffer, size);
-		}
-
-		status_t
-		NativeFlatten(BDataIO *stream, ssize_t *size) const
-		{
-			return fMessage->_NativeFlatten(stream, size);
-		}
-
 		status_t
 		FlattenToArea(message_header **header) const
 		{

src/kits/app/MessageAdapter.cpp

@@ -316,7 +316,7 @@ MessageAdapter::_R5FlattenedSize(const BMessage *from)
 
 	uint8 *data = messagePrivate.GetMessageData();
 	BMessage::field_header *field = messagePrivate.GetMessageFields();
-	for (int32 i = 0; i < header->field_count; i++, field++) {
+	for (uint32 i = 0; i < header->field_count; i++, field++) {
 		// flags and type
 		flattenedSize += 1 + sizeof(type_code);
 
@@ -343,7 +343,7 @@ MessageAdapter::_R5FlattenedSize(const BMessage *from)
 		else {
 			uint8 *source = data + field->offset + field->name_length;
 
-			for (int32 i = 0; i < field->count; i++) {
+			for (uint32 i = 0; i < field->count; i++) {
 				ssize_t itemSize = *(ssize_t *)source + sizeof(ssize_t);
 				flattenedSize += pad_to_8(itemSize);
 				source += itemSize;
@@ -414,7 +414,7 @@ MessageAdapter::_FlattenR5Message(uint32 format, const BMessage *from,
 
 	// collect and add the data
 	BMessage::field_header *field = messagePrivate.GetMessageFields();
-	for (int32 i = 0; i < header->field_count; i++, field++) {
+	for (uint32 i = 0; i < header->field_count; i++, field++) {
 		flags = R5_FIELD_FLAG_VALID;
 
 		if (field->count == 1)
@@ -466,7 +466,7 @@ MessageAdapter::_FlattenR5Message(uint32 format, const BMessage *from,
 			pointer += field->data_size;
 		} else {
 			uint8 *previous = pointer;
-			for (int32 i = 0; i < field->count; i++) {
+			for (uint32 i = 0; i < field->count; i++) {
 				ssize_t itemSize = *(ssize_t *)source + sizeof(ssize_t);
 				memcpy(pointer, source, itemSize);
 				ssize_t paddedSize = pad_to_8(itemSize);

src/kits/app/MessageUtils.cpp

@@ -44,14 +44,21 @@ CalculateChecksum(const uint8 *buffer, int32 size)
 status_t
 entry_ref_flatten(char *buffer, size_t *size, const entry_ref *ref)
 {
+	if (*size < sizeof(ref->device) + sizeof(ref->directory))
+		return B_BUFFER_OVERFLOW;
+
 	memcpy((void *)buffer, (const void *)&ref->device, sizeof(ref->device));
 	buffer += sizeof(ref->device);
 	memcpy((void *)buffer, (const void *)&ref->directory, sizeof(ref->directory));
 	buffer += sizeof (ref->directory);
+	*size -= sizeof(ref->device) + sizeof(ref->directory);
 
 	size_t nameLength = 0;
 	if (ref->name) {
 		nameLength = strlen(ref->name) + 1;
+		if (*size < nameLength)
+			return B_BUFFER_OVERFLOW;
+
 		memcpy((void *)buffer, (const void *)ref->name, nameLength);
 	}
 
@@ -74,7 +81,7 @@ entry_ref_unflatten(entry_ref *ref, const char *buffer, size_t size)
 	buffer += sizeof(ref->directory);
 
 	if (ref->device != -1 && size > sizeof(ref->device)
-		+ sizeof(ref->directory)) {
+			+ sizeof(ref->directory)) {
 		ref->set_name(buffer);
 		if (ref->name == NULL) {
 			*ref = entry_ref();

src/kits/interface/View.cpp

@@ -1335,10 +1335,10 @@ BView::DragMessage(BMessage* message, BBitmap* image,
 
 	// TODO: create area and flatten message into that area!
 	// send area info over port, not the actual message!
-	int32 bufferSize = privateMessage.NativeFlattenedSize();
+	int32 bufferSize = message->FlattenedSize();
 	char* buffer = new(std::nothrow) char[bufferSize];
 	if (buffer != NULL) {
-		privateMessage.NativeFlatten(buffer, bufferSize);
+		message->Flatten(buffer, bufferSize);
 
 		fOwner->fLink->StartMessage(AS_VIEW_DRAG_IMAGE);
 		fOwner->fLink->Attach<int32>(image->_ServerToken());

src/servers/registrar/MessageDeliverer.cpp

@@ -608,7 +608,7 @@ MessageDeliverer::DeliverMessage(BMessage *message, MessagingTargetSet &targets,
 
 	// flatten the message
 	BMallocIO mallocIO;
-	status_t error = BMessage::Private(message).NativeFlatten(&mallocIO, NULL);
+	status_t error = message->Flatten(&mallocIO, NULL);
 	if (error < B_OK)
 		return error;