Port endian conversion fixes to build version of BMessage.

git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@21904 a95241bf-73f2-0310-859d-f6bbb57e9c96

headers/build/private/app/MessageUtils.h

@@ -136,6 +136,14 @@ byte_swap(int64 &data)
 }
 
 
+template<>
+inline void
+byte_swap(uint64 &data)
+{
+	data = __swap_int64(data);
+}
+
+
 template<>
 inline void
 byte_swap(int32 &data)
@@ -144,6 +152,14 @@ byte_swap(int32 &data)
 }
 
 
+template<>
+inline void
+byte_swap(uint32 &data)
+{
+	data = __swap_int32(data);
+}
+
+
 template<>
 inline void
 byte_swap(int16 &data)
@@ -152,6 +168,14 @@ byte_swap(int16 &data)
 }
 
 
+template<>
+inline void
+byte_swap(uint16 &data)
+{
+	data = __swap_int16(data);
+}
+
+
 template<>
 inline void
 byte_swap(entry_ref &data)

src/build/libbe/app/MessageAdapter.cpp

@@ -43,23 +43,27 @@ struct r5_message_header {
 	uint8	flags;
 } _PACKED;
 
+
 struct dano_section_header {
 	uint32		code;
 	ssize_t		size;
 	uint8		data[0];
 } _PACKED;
 
+
 struct dano_message_header {
 	int32		what;
 	int32		padding;
 } _PACKED;
 
+
 typedef struct offset_table_s {
 	int32		indexTable;
 	int32		endOfData;
 	int64		padding;
 } OffsetTable;
 
+
 struct dano_single_item {
 	type_code	type;
 	ssize_t		item_size;
@@ -67,6 +71,7 @@ struct dano_single_item {
 	char		name[0];
 } _PACKED;
 
+
 struct dano_fixed_size_array {
 	type_code	type;
 	ssize_t		size_per_item;
@@ -161,7 +166,6 @@ MessageAdapter::Unflatten(uint32 format, BMessage *into, const char *buffer)
 	try {
 		switch (format) {
 			case MESSAGE_FORMAT_R5:
-			case MESSAGE_FORMAT_R5_SWAPPED:
 			{
 				r5_message_header *header = (r5_message_header *)buffer;
 				BMemoryIO stream(buffer + sizeof(uint32),
@@ -169,6 +173,14 @@ MessageAdapter::Unflatten(uint32 format, BMessage *into, const char *buffer)
 				return _UnflattenR5Message(format, into, &stream);
 			}
 
+			case MESSAGE_FORMAT_R5_SWAPPED:
+			{
+				r5_message_header *header = (r5_message_header *)buffer;
+				BMemoryIO stream(buffer + sizeof(uint32),
+					__swap_int32(header->flattened_size) - sizeof(uint32));
+				return _UnflattenR5Message(format, into, &stream);
+			}
+
 			case MESSAGE_FORMAT_DANO:
 			case MESSAGE_FORMAT_DANO_SWAPPED:
 			{
@@ -329,7 +341,7 @@ MessageAdapter::_FlattenR5Message(uint32 format, const BMessage *from,
 	r5header->flags = flags;
 
 	// store the header size - used for the checksum later
-	ssize_t headerSize = (addr_t)pointer - (addr_t)buffer;
+	ssize_t headerSize = (uint32)pointer - (uint32)buffer;
 
 	// collect and add the data
 	BMessage::field_header *field = messagePrivate.GetMessageFields();
@@ -405,7 +417,7 @@ MessageAdapter::_FlattenR5Message(uint32 format, const BMessage *from,
 	pointer++;
 
 	// calculate the flattened size from the pointers
-	r5header->flattened_size = (addr_t)pointer - (addr_t)buffer;
+	r5header->flattened_size = (uint32)pointer - (uint32)buffer;
 	r5header->checksum = CalculateChecksum((uint8 *)(buffer + 8),
 		headerSize - 8);
 
@@ -515,24 +527,50 @@ MessageAdapter::_UnflattenR5Message(uint32 format, BMessage *into,
 		if (fixedSize)
 			itemSize = dataSize / itemCount;
 
-		for (int32 i = 0; i < itemCount; i++) {
+		if (fixedSize) {
-			if (!fixedSize) {
+			for (int32 i = 0; i < itemCount; i++) {
+				// ToDo: what if we are swapped? need B_INT32_TYPEs and the
+				// like be swapped here?
+				result = into->AddData(nameBuffer, type, pointer, itemSize,
+					fixedSize, itemCount);
+
+				if (result < B_OK) {
+					free(buffer);
+					return result;
+				}
+
+				pointer += itemSize;
+			}
+		} else if (format == MESSAGE_FORMAT_R5_SWAPPED) {
+			for (int32 i = 0; i < itemCount; i++) {
+				itemSize = __swap_int32(*(ssize_t *)pointer);
+				pointer += sizeof(ssize_t);
+
+				result = into->AddData(nameBuffer, type, pointer, itemSize,
+					fixedSize, itemCount);
+
+				if (result < B_OK) {
+					free(buffer);
+					return result;
+				}
+
+				pointer += pad_to_8(itemSize + sizeof(ssize_t)) - sizeof(ssize_t);
+			}
+		} else {
+			for (int32 i = 0; i < itemCount; i++) {
 				itemSize = *(ssize_t *)pointer;
 				pointer += sizeof(ssize_t);
-			}
 
-			result = into->AddData(nameBuffer, type, pointer, itemSize,
+				result = into->AddData(nameBuffer, type, pointer, itemSize,
-				fixedSize, itemCount);
+					fixedSize, itemCount);
 
-			if (result < B_OK) {
+				if (result < B_OK) {
-				free(buffer);
+					free(buffer);
-				return result;
+					return result;
-			}
+				}
 
-			if (fixedSize)
-				pointer += itemSize;
-			else
 				pointer += pad_to_8(itemSize + sizeof(ssize_t)) - sizeof(ssize_t);
+			}
 		}
 
 		free(buffer);