Refactoring step towards implementation of RepositoryWriter:

* pulled commonly useful parts out of PackageWriterImpl into 
  WriterImplBase
* moved CachedStringTable and related methods into a separate class,
  StringCache, in order to support having more than one string cache
  per package file
* made package attribute section use a string cache, too, as that's
  going to be very useful for repositories
* instead of writing package attributes directly, we now collect
  corresponding PackageAttributes and write those later
* adjusted package reader accordingly


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@40376 a95241bf-73f2-0310-859d-f6bbb57e9c96

headers/os/package/hpkg/PackageWriter.h

@@ -35,6 +35,7 @@ public:
 									uint64 uncompressedMainSize,
 									uint64 uncompressedTOCSize) = 0;
 	virtual void				OnPackageAttributesSizeInfo(
+									uint32 stringCount,
 									uint32 uncompressedSize) = 0;
 	virtual void				OnPackageSizeInfo(uint32 headerSize,
 									uint64 heapSize, uint64 tocSize,

headers/os/package/hpkg/RepositoryWriter.h

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2011, Haiku, Inc.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _PACKAGE__HPKG__REPOSITORY_WRITER_H_
+#define _PACKAGE__HPKG__REPOSITORY_WRITER_H_
+
+
+#include <SupportDefs.h>
+
+#include <package/hpkg/ErrorOutput.h>
+
+
+namespace BPackageKit {
+
+namespace BHPKG {
+
+
+namespace BPrivate {
+	class RepositoryWriterImpl;
+}
+using BPrivate::RepositoryWriterImpl;
+
+
+class BRepositoryWriterListener : public BErrorOutput {
+public:
+	virtual	void				PrintErrorVarArgs(const char* format,
+									va_list args) = 0;
+
+	virtual	void				OnPackageAdded(
+									const BPackageInfo& packageInfo) = 0;
+
+	virtual void				OnPackageAttributesSizeInfo(
+									uint32 uncompressedSize) = 0;
+	virtual void				OnRepositorySizeInfo(uint32 headerSize,
+									uint32 packageAttributesSize,
+									uint64 totalSize) = 0;
+};
+
+
+class BRepositoryWriter {
+public:
+public:
+								BRepositoryWriter(
+									BRepositoryWriterListener* listener);
+								~BRepositoryWriter();
+
+			status_t			Init(const char* fileName);
+			status_t			AddPackage(const BPackageInfo& packageInfo);
+			status_t			Finish();
+
+private:
+			RepositoryWriterImpl*	fImpl;
+};
+
+
+}	// namespace BHPKG
+
+}	// namespace BPackageKit
+
+
+#endif	// _PACKAGE__HPKG__REPOSITORY_WRITER_H_

headers/private/package/hpkg/PackageReaderImpl.h

@@ -65,17 +65,22 @@ private:
 				uint8*			data;
 				uint64			offset;
 				uint64			currentOffset;
+				uint64			stringsLength;
+				uint64			stringsCount;
+				char**			strings;
 				const char*		name;
 
 				SectionInfo(const char* _name)
 					:
 					data(NULL),
+					strings(NULL),
 					name(_name)
 				{
 				}
 
 				~SectionInfo()
 				{
+					delete[] strings;
 					delete[] data;
 				}
 			};
@@ -90,7 +95,8 @@ private:
 									const SectionInfo& section) const;
 
 			status_t			_ParseTOCAttributeTypes();
-			status_t			_ParseTOCStrings();
+
+			status_t			_ParseStrings();
 
 			status_t			_ParseContent(AttributeHandlerContext* context,
 									AttributeHandler* rootAttributeHandler);
@@ -153,15 +159,12 @@ private:
 
 			uint64				fTOCAttributeTypesLength;
 			uint64				fTOCAttributeTypesCount;
-			uint64				fTOCStringsLength;
-			uint64				fTOCStringsCount;
 
 			SectionInfo			fTOCSection;
 			SectionInfo			fPackageAttributesSection;
 			SectionInfo*		fCurrentSection;
 
 			AttributeTypeReference* fAttributeTypes;
-			char**				fStrings;
 
 			AttributeHandlerList* fAttributeHandlerStack;
 

headers/private/package/hpkg/PackageWriterImpl.h

@@ -10,9 +10,9 @@
 #include <util/DoublyLinkedList.h>
 #include <util/OpenHashTable.h>
 
-#include <package/PackageInfo.h>
 #include <package/hpkg/PackageWriter.h>
 #include <package/hpkg/Strings.h>
+#include <package/hpkg/WriterImplBase.h>
 
 
 namespace BPackageKit {
@@ -29,7 +29,7 @@ namespace BPrivate {
 
 struct hpkg_header;
 
-class PackageWriterImpl {
+class PackageWriterImpl : public WriterImplBase {
 public:
 								PackageWriterImpl(
 									BPackageWriterListener* listener);
@@ -40,18 +40,13 @@ public:
 			status_t			Finish();
 
 private:
-			struct AttributeValue;
 			struct AttributeTypeKey;
 			struct AttributeType;
 			struct AttributeTypeHashDefinition;
 			struct Attribute;
 			struct AttributeTypeUsageGreater;
 			struct Entry;
-			struct DataWriter;
-			struct DummyDataWriter;
-			struct FDDataWriter;
 			struct SubPathAdder;
-			struct ZlibDataWriter;
 
 			typedef BOpenHashTable<AttributeTypeHashDefinition>
 				AttributeTypeTable;
@@ -66,33 +61,25 @@ private:
 									const char* name, size_t nameLength,
 									bool isImplicit);
 
-			void				_WriteTOC(hpkg_header& header);
+			void				_RegisterPackageInfo(
-			int32				_WriteTOCSections(uint64& _attributeTypesSize,
+									PackageAttributeList& attributeList,
-									uint64& _stringsSize, uint64& _mainSize);
+									const BPackageInfo& packageInfo);
-			void				_WriteAttributeTypes();
+			void				_RegisterPackageVersion(
-			int32				_WriteCachedStrings();
+									PackageAttributeList& attributeList,
-			void				_WriteAttributeChildren(Attribute* attribute);
-
-			void				_WritePackageAttributes(hpkg_header& header);
-
-			void				_WriteAttributeValue(
-									const AttributeValue& value,
-									uint8 encoding);
-			void				_WriteUnsignedLEB128(uint64 value);
-	inline	void				_WriteString(const char* string);
-
-			void				_WritePackageVersion(
 									const BPackageVersion& version);
-			void				_WritePackageResolvableExpressionList(
+			void				_RegisterPackageResolvableExpressionList(
+									PackageAttributeList& attributeList,
 									const BObjectList<
 										BPackageResolvableExpression>& list,
 									uint8 id);
 
-	template<typename Type>
+			void				_WriteTOC(hpkg_header& header);
-	inline	void				_Write(const Type& value);
+			int32				_WriteTOCSections(uint64& _attributeTypesSize,
+									uint64& _stringsSize, uint64& _mainSize);
+			void				_WriteAttributeTypes();
+			void				_WriteAttributeChildren(Attribute* attribute);
 
-			void				_WriteBuffer(const void* buffer, size_t size,
+			void				_WritePackageAttributes(hpkg_header& header);
-									off_t offset);
 
 			void				_AddEntry(int dirFD, Entry* entry,
 									const char* fileName, char* pathBuffer);
@@ -111,38 +98,30 @@ private:
 			Attribute*			_AddDataAttribute(const char* attributeName,
 									uint64 dataSize, const uint8* data);
 
-			CachedString*		_GetCachedString(const char* value);
 			AttributeType*		_GetAttributeType(const char* attributeName,
 									uint8 type);
 
 			status_t			_AddData(BDataReader& dataReader, off_t size);
+
 			status_t			_WriteUncompressedData(BDataReader& dataReader,
 									off_t size, uint64 writeOffset);
-			status_t			_WriteZlibCompressedData(BDataReader& dataReader,
+			status_t			_WriteZlibCompressedData(
+									BDataReader& dataReader,
 									off_t size, uint64 writeOffset,
 									uint64& _compressedSize);
 
 private:
 			BPackageWriterListener*	fListener;
 
-			const char*			fFileName;
-			int					fFD;
-			bool				fFinished;
 			off_t				fHeapOffset;
 			off_t				fHeapEnd;
-			void*				fDataBuffer;
-			size_t				fDataBufferSize;
-
-			BPackageInfo		fPackageInfo;
-
-			DataWriter*			fDataWriter;
 
 			Entry*				fRootEntry;
 
 			Attribute*			fRootAttribute;
 			Attribute*			fTopAttribute;
 
-			CachedStringTable*	fCachedStrings;
+			StringCache			fStringCache;
 			AttributeTypeTable*	fAttributeTypes;
 };
 

headers/private/package/hpkg/Strings.h

@@ -6,6 +6,8 @@
 #define _PACKAGE__HPKG__PRIVATE__STRINGS_H_
 
 
+#include <new>
+
 #include <util/OpenHashTable.h>
 
 
@@ -86,6 +88,15 @@ struct CachedStringUsageGreater {
 };
 
 
+struct StringCache : public CachedStringTable {
+	StringCache();
+	~StringCache();
+
+	CachedString* Get(const char* value);
+
+};
+
+
 }	// namespace BPrivate
 
 }	// namespace BHPKG

headers/private/package/hpkg/WriterImplBase.h

@@ -0,0 +1,234 @@
+/*
+ * Copyright 2011, Oliver Tappe <[email protected]>
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _PACKAGE__HPKG__PRIVATE__WRITER_IMPL_BASE_H_
+#define _PACKAGE__HPKG__PRIVATE__WRITER_IMPL_BASE_H_
+
+
+#include <util/DoublyLinkedList.h>
+
+#include <package/hpkg/haiku_package.h>
+
+#include <package/hpkg/DataOutput.h>
+#include <package/hpkg/Strings.h>
+#include <package/hpkg/ZlibCompressor.h>
+
+#include <package/PackageInfo.h>
+
+
+namespace BPackageKit {
+
+namespace BHPKG {
+
+
+class BDataReader;
+class BErrorOutput;
+
+
+namespace BPrivate {
+
+
+struct hpkg_header;
+
+class WriterImplBase {
+public:
+								WriterImplBase(BErrorOutput* errorOutput);
+								~WriterImplBase();
+
+protected:
+
+
+			struct AttributeValue {
+				union {
+					int64			signedInt;
+					uint64			unsignedInt;
+					CachedString*	string;
+					struct {
+						uint64		size;
+						union {
+							uint64	offset;
+							uint8	raw[B_HPKG_MAX_INLINE_DATA_SIZE];
+						};
+					} data;
+				};
+				uint8		type;
+				int8		encoding;
+
+				AttributeValue();
+				~AttributeValue();
+
+				void SetTo(int8 value);
+				void SetTo(uint8 value);
+				void SetTo(int16 value);
+				void SetTo(uint16 value);
+				void SetTo(int32 value);
+				void SetTo(uint32 value);
+				void SetTo(int64 value);
+				void SetTo(uint64 value);
+				void SetTo(CachedString* value);
+				void SetToData(uint64 size, uint64 offset);
+				void SetToData(uint64 size, const void* rawData);
+				uint8 ApplicableEncoding() const;
+
+			private:
+				static uint8 _ApplicableIntEncoding(uint64 value);
+			};
+
+
+			struct PackageAttribute :
+					public DoublyLinkedListLinkImpl<PackageAttribute>,
+					public AttributeValue {
+				HPKGPackageAttributeID 				id;
+				DoublyLinkedList<PackageAttribute>	children;
+
+				PackageAttribute(HPKGPackageAttributeID id_, uint8 type,
+					uint8 encoding);
+				~PackageAttribute();
+
+				void AddChild(PackageAttribute* child);
+
+			private:
+				void _DeleteChildren();
+			};
+
+
+			struct DataWriter {
+				DataWriter();
+				virtual ~DataWriter();
+
+				uint64 BytesWritten() const;
+
+				virtual status_t WriteDataNoThrow(const void* buffer,
+					size_t size) = 0;
+
+				void WriteDataThrows(const void* buffer, size_t size);
+
+			protected:
+				uint64	fBytesWritten;
+			};
+
+
+			struct FDDataWriter : DataWriter {
+				FDDataWriter(int fd, off_t offset, BErrorOutput* errorOutput);
+
+				virtual status_t WriteDataNoThrow(const void* buffer,
+					size_t size);
+
+				off_t Offset() const;
+
+			private:
+				int				fFD;
+				off_t			fOffset;
+				BErrorOutput*	fErrorOutput;
+			};
+
+
+			struct ZlibDataWriter : DataWriter, private BDataOutput {
+				ZlibDataWriter(DataWriter* dataWriter);
+
+				void Init();
+
+				void Finish();
+
+				virtual status_t WriteDataNoThrow(const void* buffer,
+					size_t size);
+
+			private:
+				// BDataOutput
+				virtual status_t WriteData(const void* buffer, size_t size);
+
+			private:
+				DataWriter*		fDataWriter;
+				ZlibCompressor	fCompressor;
+			};
+
+
+			typedef DoublyLinkedList<PackageAttribute> PackageAttributeList;
+
+protected:
+			status_t			Init(const char* fileName, const char* type);
+
+			int32				WriteCachedStrings(const StringCache& cache,
+									uint32 minUsageCount);
+
+			void				WritePackageAttributes(
+									const PackageAttributeList& attributes);
+			void				WritePackageVersion(
+									const BPackageVersion& version);
+			void				WritePackageResolvableExpressionList(
+									const BObjectList<
+										BPackageResolvableExpression>& list,
+									uint8 id);
+
+			void				WriteAttributeValue(const AttributeValue& value,
+									uint8 encoding);
+			void				WriteUnsignedLEB128(uint64 value);
+	inline	void				WriteString(const char* string);
+
+	template<typename Type>
+	inline	void				Write(const Type& value);
+
+			void				WriteBuffer(const void* buffer, size_t size,
+									off_t offset);
+
+protected:
+			BErrorOutput*		fErrorOutput;
+			const char*			fFileName;
+			int					fFD;
+			bool				fFinished;
+			void*				fDataBuffer;
+			const size_t		fDataBufferSize;
+
+			DataWriter*			fDataWriter;
+
+			StringCache			fPackageStringCache;
+			DoublyLinkedList<PackageAttribute>	fPackageAttributes;
+};
+
+
+inline uint64
+WriterImplBase::DataWriter::BytesWritten() const
+{
+	return fBytesWritten;
+}
+
+
+inline void
+WriterImplBase::DataWriter::WriteDataThrows(const void* buffer, size_t size)
+{
+	status_t error = WriteDataNoThrow(buffer, size);
+	if (error != B_OK)
+		throw status_t(error);
+}
+
+inline off_t
+WriterImplBase::FDDataWriter::Offset() const
+{
+	return fOffset;
+}
+
+
+template<typename Type>
+inline void
+WriterImplBase::Write(const Type& value)
+{
+	fDataWriter->WriteDataThrows(&value, sizeof(Type));
+}
+
+
+inline void
+WriterImplBase::WriteString(const char* string)
+{
+	fDataWriter->WriteDataThrows(string, strlen(string) + 1);
+}
+
+
+}	// namespace BPrivate
+
+}	// namespace BHPKG
+
+}	// namespace BPackageKit
+
+
+#endif	// _PACKAGE__HPKG__PRIVATE__WRITER_IMPL_BASE_H_

headers/private/package/hpkg/haiku_package.h

@@ -29,6 +29,8 @@ struct hpkg_header {
 	uint32	attributes_compression;
 	uint32	attributes_length_compressed;
 	uint32	attributes_length_uncompressed;
+	uint32	attributes_strings_length;
+	uint32	attributes_strings_count;
 
 	// TOC section
 	uint32	toc_compression;

src/bin/package/command_create.cpp

@@ -63,12 +63,14 @@ public:
 			uncompressedTOCSize);
 	}
 
-	virtual void OnPackageAttributesSizeInfo(uint32 uncompressedSize)
+	virtual void OnPackageAttributesSizeInfo(uint32 stringCount,
+		uint32 uncompressedSize)
 	{
 		if (fQuiet || !fVerbose)
 			return;
 
 		printf("----- Package Attribute Info -----\n");
+		printf("string count:            %10ld\n", stringCount);
 		printf("package attributes size: %10ld (uncompressed)\n",
 			uncompressedSize);
 	}

src/kits/package/Jamfile

@@ -26,6 +26,7 @@ HPKG_SOURCES =
 	PackageWriter.cpp
 	PackageWriterImpl.cpp
 	Strings.cpp
+	WriterImplBase.cpp
 
 	# compression
 	ZlibCompressionBase.cpp

src/kits/package/hpkg/PackageReaderImpl.cpp

@@ -620,7 +620,6 @@ PackageReaderImpl::PackageReaderImpl(BErrorOutput* errorOutput)
 	fPackageAttributesSection("package attributes section"),
 	fCurrentSection(NULL),
 	fAttributeTypes(NULL),
-	fStrings(NULL),
 	fScratchBuffer(NULL),
 	fScratchBufferSize(0)
 {
@@ -633,7 +632,6 @@ PackageReaderImpl::~PackageReaderImpl()
 		close(fFD);
 
 	delete[] fScratchBuffer;
-	delete[] fStrings;
 	delete[] fAttributeTypes;
 }
 
@@ -713,6 +711,10 @@ PackageReaderImpl::Init(int fd, bool keepFD)
 		= B_BENDIAN_TO_HOST_INT32(header.attributes_length_compressed);
 	fPackageAttributesSection.uncompressedLength
 		= B_BENDIAN_TO_HOST_INT32(header.attributes_length_uncompressed);
+	fPackageAttributesSection.stringsLength
+		= B_BENDIAN_TO_HOST_INT32(header.attributes_strings_length);
+	fPackageAttributesSection.stringsCount
+		= B_BENDIAN_TO_HOST_INT32(header.attributes_strings_count);
 
 	if (const char* errorString = _CheckCompression(
 		fPackageAttributesSection)) {
@@ -739,14 +741,16 @@ PackageReaderImpl::Init(int fd, bool keepFD)
 		= B_BENDIAN_TO_HOST_INT64(header.toc_attribute_types_length);
 	fTOCAttributeTypesCount
 		= B_BENDIAN_TO_HOST_INT64(header.toc_attribute_types_count);
-	fTOCStringsLength = B_BENDIAN_TO_HOST_INT64(header.toc_strings_length);
+	fTOCSection.stringsLength
-	fTOCStringsCount = B_BENDIAN_TO_HOST_INT64(header.toc_strings_count);
+		= B_BENDIAN_TO_HOST_INT64(header.toc_strings_length);
+	fTOCSection.stringsCount
+		= B_BENDIAN_TO_HOST_INT64(header.toc_strings_count);
 
 	if (fTOCAttributeTypesLength > fTOCSection.uncompressedLength
-		|| fTOCStringsLength
+		|| fTOCSection.stringsLength
 			> fTOCSection.uncompressedLength - fTOCAttributeTypesLength
 		|| fTOCAttributeTypesCount > fTOCAttributeTypesLength
-		|| fTOCStringsCount > fTOCStringsLength) {
+		|| fTOCSection.stringsCount > fTOCSection.stringsLength) {
 		fErrorOutput->PrintError("Error: Invalid package file: Invalid TOC "
 			"subsections description\n");
 		return B_BAD_DATA;
@@ -829,10 +833,10 @@ PackageReaderImpl::Init(int fd, bool keepFD)
 	fCurrentSection->currentOffset += fTOCAttributeTypesLength;
 
 	// strings
-	error = _ParseTOCStrings();
+	error = _ParseStrings();
 	if (error != B_OK)
 		return error;
-	fCurrentSection->currentOffset += fTOCStringsLength;
+	fCurrentSection->currentOffset += fCurrentSection->stringsLength;
 
 	return B_OK;
 }
@@ -851,6 +855,12 @@ PackageReaderImpl::ParseContent(BPackageContentHandler* contentHandler)
 	fCurrentSection = &fPackageAttributesSection;
 	fCurrentSection->currentOffset = 0;
 
+	// strings
+	status_t error = _ParseStrings();
+	if (error != B_OK)
+		return error;
+	fCurrentSection->currentOffset += fCurrentSection->stringsLength;
+
 	return _ParsePackageAttributes(&context);
 }
 
@@ -916,7 +926,6 @@ PackageReaderImpl::_ParseTOCAttributeTypes()
 			if (position + 1 != sectionEnd) {
 				fErrorOutput->PrintError("Error: Excess bytes in TOC attribute "
 					"types section\n");
-TRACE("position: %p, sectionEnd: %p\n", position, sectionEnd);
 				return B_BAD_DATA;
 			}
 
@@ -941,28 +950,30 @@ TRACE("position: %p, sectionEnd: %p\n", position, sectionEnd);
 		fAttributeTypes[index].type = type;
 		fAttributeTypes[index].standardIndex = _GetStandardIndex(type);
 		index++;
-TRACE("type: %u, \"%s\"\n", type->type, type->name);
 	}
 }
 
 
 status_t
-PackageReaderImpl::_ParseTOCStrings()
+PackageReaderImpl::_ParseStrings()
 {
 	// allocate table
-	fStrings = new(std::nothrow) char*[fTOCStringsCount];
+	fCurrentSection->strings
-	if (fStrings == NULL) {
+		= new(std::nothrow) char*[fCurrentSection->stringsCount];
+	if (fCurrentSection->strings == NULL) {
 		fErrorOutput->PrintError("Error: Out of memory!\n");
 		return B_NO_MEMORY;
 	}
 
 	// parse the section and fill the table
-	char* position = (char*)fTOCSection.data + fTOCSection.currentOffset;
+	char* position
-	char* sectionEnd = position + fTOCStringsLength;
+		= (char*)fCurrentSection->data + fCurrentSection->currentOffset;
+	char* sectionEnd = position + fCurrentSection->stringsLength;
 	uint32 index = 0;
 	while (true) {
 		if (position >= sectionEnd) {
-			fErrorOutput->PrintError("Error: Malformed TOC strings section\n");
+			fErrorOutput->PrintError("Error: Malformed %s strings section\n",
+				fCurrentSection->name);
 			return B_BAD_DATA;
 		}
 
@@ -970,29 +981,31 @@ PackageReaderImpl::_ParseTOCStrings()
 
 		if (stringLength == 0) {
 			if (position + 1 != sectionEnd) {
-				fErrorOutput->PrintError("Error: Excess bytes in TOC strings "
+				fErrorOutput->PrintError(
-					"section\n");
+					"Error: %ld excess bytes in %s strings section\n",
-TRACE("position: %p, sectionEnd: %p\n", position, sectionEnd);
+					sectionEnd - (position + 1), fCurrentSection->name);
 				return B_BAD_DATA;
 			}
 
-			if (index != fTOCStringsCount) {
+			if (index != fCurrentSection->stringsCount) {
-				fErrorOutput->PrintError("Error: Invalid TOC strings section: "
+				fErrorOutput->PrintError("Error: Invalid %s strings section: "
-					"Less strings than specified in the header\n");
+					"Less strings (%lld) than specified in the header (%lld)\n",
+					fCurrentSection->name, index,
+					fCurrentSection->stringsCount);
 				return B_BAD_DATA;
 			}
 
 			return B_OK;
 		}
 
-		if (index >= fTOCStringsCount) {
+		if (index >= fCurrentSection->stringsCount) {
-			fErrorOutput->PrintError("Error: Invalid TOC strings section: "
+			fErrorOutput->PrintError("Error: Invalid %s strings section: "
-				"More strings than specified in the header\n");
+				"More strings (%lld) than specified in the header (%lld)\n",
+				fCurrentSection->name, index, fCurrentSection->stringsCount);
 			return B_BAD_DATA;
 		}
 
-		fStrings[index++] = position;
+		fCurrentSection->strings[index++] = position;
-TRACE("string: \"%s\"\n", position);
 		position += stringLength + 1;
 	}
 }
@@ -1003,7 +1016,8 @@ PackageReaderImpl::_ParseContent(AttributeHandlerContext* context,
 	AttributeHandler* rootAttributeHandler)
 {
 	// parse the TOC
-	fTOCSection.currentOffset = fTOCAttributeTypesLength + fTOCStringsLength;
+	fTOCSection.currentOffset = fTOCAttributeTypesLength
+		+ fTOCSection.stringsLength;
 
 	// prepare attribute handler context
 	context->heapOffset = fHeapOffset;
@@ -1491,9 +1505,9 @@ PackageReaderImpl::_ReadAttributeValue(uint8 type, uint8 encoding,
 				}
 				default:
 				{
-					fErrorOutput->PrintError("Error: Invalid TOC section: "
+					fErrorOutput->PrintError("Error: Invalid %s section: "
-						"invalid encoding %d for int value type %d\n", encoding,
+						"invalid encoding %d for int value type %d\n",
-						type);
+						fCurrentSection->name, encoding, type);
 					return B_BAD_VALUE;
 				}
 			}
@@ -1517,13 +1531,15 @@ PackageReaderImpl::_ReadAttributeValue(uint8 type, uint8 encoding,
 				if (error != B_OK)
 					return error;
 
-				if (index > fTOCStringsCount) {
+				if (index > fCurrentSection->stringsCount) {
-					fErrorOutput->PrintError("Error: Invalid TOC section: "
+					fErrorOutput->PrintError("Error: Invalid %s section: "
-						"string reference out of bounds\n");
+						"string reference (%lld) out of bounds (%lld)\n",
+						fCurrentSection->name, index,
+						fCurrentSection->stringsCount);
 					return B_BAD_DATA;
 				}
 
-				_value.SetTo(fStrings[index]);
+				_value.SetTo(fCurrentSection->strings[index]);
 			} else if (encoding == B_HPKG_ATTRIBUTE_ENCODING_STRING_INLINE) {
 				const char* string;
 				status_t error = _ReadString(string);
@@ -1532,8 +1548,8 @@ PackageReaderImpl::_ReadAttributeValue(uint8 type, uint8 encoding,
 
 				_value.SetTo(string);
 			} else {
-				fErrorOutput->PrintError("Error: Invalid TOC section: invalid "
+				fErrorOutput->PrintError("Error: Invalid %s section: invalid "
-					"string encoding (%u)\n", encoding);
+					"string encoding (%u)\n", fCurrentSection->name, encoding);
 				return B_BAD_DATA;
 			}
 
@@ -1554,16 +1570,16 @@ PackageReaderImpl::_ReadAttributeValue(uint8 type, uint8 encoding,
 					return error;
 
 				if (offset > fHeapSize || size > fHeapSize - offset) {
-					fErrorOutput->PrintError("Error: Invalid TOC section: "
+					fErrorOutput->PrintError("Error: Invalid %s section: "
-						"invalid data reference\n");
+						"invalid data reference\n", fCurrentSection->name);
 					return B_BAD_DATA;
 				}
 
 				_value.SetToData(size, fHeapOffset + offset);
 			} else if (encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) {
 				if (size > B_HPKG_MAX_INLINE_DATA_SIZE) {
-					fErrorOutput->PrintError("Error: Invalid TOC section: "
+					fErrorOutput->PrintError("Error: Invalid %s section: "
-						"inline data too long\n");
+						"inline data too long\n", fCurrentSection->name);
 					return B_BAD_DATA;
 				}
 
@@ -1573,8 +1589,8 @@ PackageReaderImpl::_ReadAttributeValue(uint8 type, uint8 encoding,
 					return error;
 				_value.SetToData(size, buffer);
 			} else {
-				fErrorOutput->PrintError("Error: Invalid TOC section: invalid "
+				fErrorOutput->PrintError("Error: Invalid %s section: invalid "
-					"raw encoding (%u)\n", encoding);
+					"raw encoding (%u)\n", fCurrentSection->name, encoding);
 				return B_BAD_DATA;
 			}
 
@@ -1582,8 +1598,8 @@ PackageReaderImpl::_ReadAttributeValue(uint8 type, uint8 encoding,
 		}
 
 		default:
-			fErrorOutput->PrintError("Error: Invalid TOC section: invalid "
+			fErrorOutput->PrintError("Error: Invalid %s section: invalid "
-				"value type: %d\n", type);
+				"value type: %d\n", fCurrentSection->name, type);
 			return B_BAD_DATA;
 	}
 }

src/kits/package/hpkg/Strings.cpp

@@ -34,6 +34,42 @@ hash_string(const char* string)
 }
 
 
+StringCache::StringCache()
+{
+}
+
+
+StringCache::~StringCache()
+{
+	CachedString* cachedString = Clear(true);
+	while (cachedString != NULL) {
+		CachedString* next = cachedString->next;
+		delete cachedString;
+		cachedString = next;
+	}
+}
+
+
+CachedString*
+StringCache::Get(const char* value)
+{
+	CachedString* string = Lookup(value);
+	if (string != NULL) {
+		string->usageCount++;
+		return string;
+	}
+
+	string = new CachedString;
+	if (!string->Init(value)) {
+		delete string;
+		throw std::bad_alloc();
+	}
+
+	Insert(string);
+	return string;
+}
+
+
 }	// namespace BPrivate
 
 }	// namespace BHPKG

src/kits/package/hpkg/WriterImplBase.cpp

@@ -0,0 +1,533 @@
+/*
+ * Copyright 2011, Oliver Tappe <zooey@hirschkaefer.de>
+ * Distributed under the terms of the MIT License.
+ */
+
+
+#include <package/hpkg/WriterImplBase.h>
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <new>
+
+#include <ByteOrder.h>
+
+#include <AutoDeleter.h>
+
+#include <package/hpkg/haiku_package.h>
+
+#include <package/hpkg/DataReader.h>
+#include <package/hpkg/ErrorOutput.h>
+
+
+namespace BPackageKit {
+
+namespace BHPKG {
+
+namespace BPrivate {
+
+
+// #pragma mark - AttributeValue
+
+
+WriterImplBase::AttributeValue::AttributeValue()
+	:
+	type(B_HPKG_ATTRIBUTE_TYPE_INVALID),
+	encoding(-1)
+{
+}
+
+
+WriterImplBase::AttributeValue::~AttributeValue()
+{
+}
+
+
+void
+WriterImplBase::AttributeValue::SetTo(int8 value)
+{
+	signedInt = value;
+	type = B_HPKG_ATTRIBUTE_TYPE_INT;
+}
+
+
+void
+WriterImplBase::AttributeValue::SetTo(uint8 value)
+{
+	unsignedInt = value;
+	type = B_HPKG_ATTRIBUTE_TYPE_UINT;
+}
+
+
+void
+WriterImplBase::AttributeValue::SetTo(int16 value)
+{
+	signedInt = value;
+	type = B_HPKG_ATTRIBUTE_TYPE_INT;
+}
+
+
+void
+WriterImplBase::AttributeValue::SetTo(uint16 value)
+{
+	unsignedInt = value;
+	type = B_HPKG_ATTRIBUTE_TYPE_UINT;
+}
+
+
+void
+WriterImplBase::AttributeValue::SetTo(int32 value)
+{
+	signedInt = value;
+	type = B_HPKG_ATTRIBUTE_TYPE_INT;
+}
+
+
+void
+WriterImplBase::AttributeValue::SetTo(uint32 value)
+{
+	unsignedInt = value;
+	type = B_HPKG_ATTRIBUTE_TYPE_UINT;
+}
+
+
+void
+WriterImplBase::AttributeValue::SetTo(int64 value)
+{
+	signedInt = value;
+	type = B_HPKG_ATTRIBUTE_TYPE_INT;
+}
+
+
+void
+WriterImplBase::AttributeValue::SetTo(uint64 value)
+{
+	unsignedInt = value;
+	type = B_HPKG_ATTRIBUTE_TYPE_UINT;
+}
+
+
+void
+WriterImplBase::AttributeValue::SetTo(CachedString* value)
+{
+	string = value;
+	type = B_HPKG_ATTRIBUTE_TYPE_STRING;
+}
+
+
+void
+WriterImplBase::AttributeValue::SetToData(uint64 size, uint64 offset)
+{
+	data.size = size;
+	data.offset = offset;
+	type = B_HPKG_ATTRIBUTE_TYPE_RAW;
+	encoding = B_HPKG_ATTRIBUTE_ENCODING_RAW_HEAP;
+}
+
+
+void
+WriterImplBase::AttributeValue::SetToData(uint64 size, const void* rawData)
+{
+	data.size = size;
+	if (size > 0)
+		memcpy(data.raw, rawData, size);
+	type = B_HPKG_ATTRIBUTE_TYPE_RAW;
+	encoding = B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE;
+}
+
+
+uint8
+WriterImplBase::AttributeValue::ApplicableEncoding() const
+{
+	switch (type) {
+		case B_HPKG_ATTRIBUTE_TYPE_INT:
+			return _ApplicableIntEncoding(signedInt >= 0
+				? (uint64)signedInt << 1
+				: (uint64)(-(signedInt + 1) << 1));
+		case B_HPKG_ATTRIBUTE_TYPE_UINT:
+			return _ApplicableIntEncoding(unsignedInt);
+		case B_HPKG_ATTRIBUTE_TYPE_STRING:
+			return string->index >= 0
+				? B_HPKG_ATTRIBUTE_ENCODING_STRING_TABLE
+				: B_HPKG_ATTRIBUTE_ENCODING_STRING_INLINE;
+		case B_HPKG_ATTRIBUTE_TYPE_RAW:
+			return encoding;
+		default:
+			return 0;
+	}
+}
+
+
+/*static*/ uint8
+WriterImplBase::AttributeValue::_ApplicableIntEncoding(uint64 value)
+{
+	if (value <= 0xff)
+		return B_HPKG_ATTRIBUTE_ENCODING_INT_8_BIT;
+	if (value <= 0xffff)
+		return B_HPKG_ATTRIBUTE_ENCODING_INT_16_BIT;
+	if (value <= 0xffffffff)
+		return B_HPKG_ATTRIBUTE_ENCODING_INT_32_BIT;
+
+	return B_HPKG_ATTRIBUTE_ENCODING_INT_64_BIT;
+}
+
+
+// #pragma mark - DataWriter
+
+
+WriterImplBase::DataWriter::DataWriter()
+	:
+	fBytesWritten(0)
+{
+}
+
+
+WriterImplBase::DataWriter::~DataWriter()
+{
+}
+
+
+// #pragma mark - FDDataWriter
+
+
+WriterImplBase::FDDataWriter::FDDataWriter(int fd, off_t offset,
+	BErrorOutput* errorOutput)
+	:
+	fFD(fd),
+	fOffset(offset),
+	fErrorOutput(errorOutput)
+{
+}
+
+
+status_t
+WriterImplBase::FDDataWriter::WriteDataNoThrow(const void* buffer, size_t size)
+{
+	ssize_t bytesWritten = pwrite(fFD, buffer, size, fOffset);
+	if (bytesWritten < 0) {
+		fErrorOutput->PrintError(
+			"WriteDataNoThrow(%p, %lu) failed to write data: %s\n", buffer,
+			size, strerror(errno));
+		return errno;
+	}
+	if ((size_t)bytesWritten != size) {
+		fErrorOutput->PrintError(
+			"WriteDataNoThrow(%p, %lu) failed to write all data\n", buffer,
+			size);
+		return B_ERROR;
+	}
+
+	fOffset += size;
+	fBytesWritten += size;
+	return B_OK;
+}
+
+
+// #pragma mark - ZlibDataWriter
+
+
+WriterImplBase::ZlibDataWriter::ZlibDataWriter(DataWriter* dataWriter)
+	:
+	fDataWriter(dataWriter),
+	fCompressor(this)
+{
+}
+
+
+void
+WriterImplBase::ZlibDataWriter::Init()
+{
+	status_t error = fCompressor.Init();
+	if (error != B_OK)
+		throw status_t(error);
+}
+
+
+void
+WriterImplBase::ZlibDataWriter::Finish()
+{
+	status_t error = fCompressor.Finish();
+	if (error != B_OK)
+		throw status_t(error);
+}
+
+
+status_t
+WriterImplBase::ZlibDataWriter::WriteDataNoThrow(const void* buffer,
+	size_t size)
+{
+	status_t error = fCompressor.CompressNext(buffer, size);
+	if (error == B_OK)
+		fBytesWritten += size;
+	return error;
+}
+
+
+status_t
+WriterImplBase::ZlibDataWriter::WriteData(const void* buffer, size_t size)
+{
+	return fDataWriter->WriteDataNoThrow(buffer, size);
+}
+
+
+// #pragma mark - PackageAttribute
+
+WriterImplBase::PackageAttribute::PackageAttribute(HPKGPackageAttributeID id_,
+	uint8 type_, uint8 encoding_)
+	:
+	id(id_)
+{
+	type = type_;
+	encoding = encoding_;
+}
+
+
+WriterImplBase::PackageAttribute::~PackageAttribute()
+{
+	_DeleteChildren();
+}
+
+
+void
+WriterImplBase::PackageAttribute::AddChild(PackageAttribute* child)
+{
+	children.Add(child);
+}
+
+
+void
+WriterImplBase::PackageAttribute::_DeleteChildren()
+{
+	while (PackageAttribute* child = children.RemoveHead())
+		delete child;
+}
+
+
+// #pragma mark - WriterImplBase
+
+
+WriterImplBase::WriterImplBase(BErrorOutput* errorOutput)
+	:
+	fErrorOutput(errorOutput),
+	fFileName(NULL),
+	fFD(-1),
+	fFinished(false),
+	fDataBuffer(NULL),
+	fDataBufferSize(2 * B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB),
+	fDataWriter(NULL)
+{
+}
+
+
+WriterImplBase::~WriterImplBase()
+{
+	if (fFD >= 0)
+		close(fFD);
+
+	if (!fFinished && fFileName != NULL)
+		unlink(fFileName);
+}
+
+
+status_t
+WriterImplBase::Init(const char* fileName, const char* type)
+{
+	if (fPackageStringCache.Init() != B_OK)
+		throw std::bad_alloc();
+
+	// allocate data buffer
+	fDataBuffer = malloc(fDataBufferSize);
+	if (fDataBuffer == NULL)
+		throw std::bad_alloc();
+
+	// open file
+	fFD = open(fileName, O_WRONLY | O_CREAT | O_TRUNC,
+		S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+	if (fFD < 0) {
+		fErrorOutput->PrintError("Failed to open %s file \"%s\": %s\n", type,
+			fileName, strerror(errno));
+		return errno;
+	}
+
+	fFileName = fileName;
+
+	return B_OK;
+}
+
+
+int32
+WriterImplBase::WriteCachedStrings(const StringCache& cache,
+	uint32 minUsageCount)
+{
+	// create an array of the cached strings
+	int32 count = cache.CountElements();
+	CachedString** cachedStrings = new CachedString*[count];
+	ArrayDeleter<CachedString*> cachedStringsDeleter(cachedStrings);
+
+	int32 index = 0;
+	for (CachedStringTable::Iterator it = cache.GetIterator();
+			CachedString* string = it.Next();) {
+		cachedStrings[index++] = string;
+	}
+
+	// sort it by descending usage count
+	std::sort(cachedStrings, cachedStrings + count, CachedStringUsageGreater());
+
+	// assign the indices and write entries to disk
+	int32 stringsWritten = 0;
+	for (int32 i = 0; i < count; i++) {
+		CachedString* cachedString = cachedStrings[i];
+
+		// empty strings must be stored inline, as they can't be distinguished
+		// from the end-marker!
+		if (strlen(cachedString->string) == 0)
+			continue;
+
+		// strings that are used only once are better stored inline
+		if (cachedString->usageCount < minUsageCount)
+			break;
+
+		WriteString(cachedString->string);
+
+		cachedString->index = stringsWritten++;
+	}
+
+	// write a terminating 0 byte
+	Write<uint8>(0);
+
+	return stringsWritten;
+}
+
+
+void
+WriterImplBase::WritePackageAttributes(
+	const PackageAttributeList& packageAttributes)
+{
+	DoublyLinkedList<PackageAttribute>::ConstIterator it
+		= packageAttributes.GetIterator();
+	while (PackageAttribute* attribute = it.Next()) {
+		uint8 encoding = attribute->ApplicableEncoding();
+
+		// write tag
+		WriteUnsignedLEB128(HPKG_PACKAGE_ATTRIBUTE_TAG_COMPOSE(
+			attribute->id, attribute->type, encoding,
+			!attribute->children.IsEmpty()));
+
+		// write value
+		WriteAttributeValue(*attribute, encoding);
+
+		if (!attribute->children.IsEmpty())
+			WritePackageAttributes(attribute->children);
+	}
+
+	WriteUnsignedLEB128(0);
+}
+
+
+void
+WriterImplBase::WriteAttributeValue(const AttributeValue& value, uint8 encoding)
+{
+	switch (value.type) {
+		case B_HPKG_ATTRIBUTE_TYPE_INT:
+		case B_HPKG_ATTRIBUTE_TYPE_UINT:
+		{
+			uint64 intValue = value.type == B_HPKG_ATTRIBUTE_TYPE_INT
+				? (uint64)value.signedInt : value.unsignedInt;
+
+			switch (encoding) {
+				case B_HPKG_ATTRIBUTE_ENCODING_INT_8_BIT:
+					Write<uint8>((uint8)intValue);
+					break;
+				case B_HPKG_ATTRIBUTE_ENCODING_INT_16_BIT:
+					Write<uint16>(
+						B_HOST_TO_BENDIAN_INT16((uint16)intValue));
+					break;
+				case B_HPKG_ATTRIBUTE_ENCODING_INT_32_BIT:
+					Write<uint32>(
+						B_HOST_TO_BENDIAN_INT32((uint32)intValue));
+					break;
+				case B_HPKG_ATTRIBUTE_ENCODING_INT_64_BIT:
+					Write<uint64>(
+						B_HOST_TO_BENDIAN_INT64((uint64)intValue));
+					break;
+				default:
+				{
+					fErrorOutput->PrintError("WriteAttributeValue(): invalid "
+						"encoding %d for int value type %d\n", encoding,
+						value.type);
+					throw status_t(B_BAD_VALUE);
+				}
+			}
+
+			break;
+		}
+
+		case B_HPKG_ATTRIBUTE_TYPE_STRING:
+		{
+			if (encoding == B_HPKG_ATTRIBUTE_ENCODING_STRING_TABLE)
+				WriteUnsignedLEB128(value.string->index);
+			else
+				WriteString(value.string->string);
+			break;
+		}
+
+		case B_HPKG_ATTRIBUTE_TYPE_RAW:
+		{
+			WriteUnsignedLEB128(value.data.size);
+			if (encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_HEAP)
+				WriteUnsignedLEB128(value.data.offset);
+			else
+				fDataWriter->WriteDataThrows(value.data.raw, value.data.size);
+			break;
+		}
+
+		default:
+			fErrorOutput->PrintError(
+				"WriteAttributeValue(): invalid value type: %d\n", value.type);
+			throw status_t(B_BAD_VALUE);
+	}
+}
+
+
+void
+WriterImplBase::WriteUnsignedLEB128(uint64 value)
+{
+	uint8 bytes[10];
+	int32 count = 0;
+	do {
+		uint8 byte = value & 0x7f;
+		value >>= 7;
+		bytes[count++] = byte | (value != 0 ? 0x80 : 0);
+	} while (value != 0);
+
+	fDataWriter->WriteDataThrows(bytes, count);
+}
+
+
+void
+WriterImplBase::WriteBuffer(const void* buffer, size_t size, off_t offset)
+{
+	ssize_t bytesWritten = pwrite(fFD, buffer, size, offset);
+	if (bytesWritten < 0) {
+		fErrorOutput->PrintError(
+			"WriteBuffer(%p, %lu) failed to write data: %s\n", buffer, size,
+			strerror(errno));
+		throw status_t(errno);
+	}
+	if ((size_t)bytesWritten != size) {
+		fErrorOutput->PrintError(
+			"WriteBuffer(%p, %lu) failed to write all data\n", buffer, size);
+		throw status_t(B_ERROR);
+	}
+}
+
+
+}	// namespace BPrivate
+
+}	// namespace BHPKG
+
+}	// namespace BPackageKit