package kit: Actually add support for B_HPKG_COMPRESSION_NONE

Until now we always declared in the HPKG header that the package file is
zlib compressed. For uncompressed files we would just store all
individual chunks uncompressed. Now we handle completely uncompressed
files slightly differently: We don't write the redundant chunk size
table anymore. The size savings are minor, but it makes the uncompressed
format read-streamable which may be handy.

headers/os/package/hpkg/PackageWriter.h

@@ -52,11 +52,15 @@ public:
 			uint32				Flags() const;
 			void				SetFlags(uint32 flags);
 
+			uint32				Compression() const;
+			void				SetCompression(uint32 compression);
+
 			int32				CompressionLevel() const;
 			void				SetCompressionLevel(int32 compressionLevel);
 
 private:
 			uint32				fFlags;
+			uint32				fCompression;
 			int32				fCompressionLevel;
 };
 

headers/private/package/hpkg/PackageFileHeapAccessorBase.h

@@ -92,10 +92,6 @@ public:
 			void				SetFile(BPositionIO* file)
 									{ fFile = file; }
 
-			uint64				HeapOverhead(uint64 uncompressedSize) const;
-									// additional bytes needed when storing
-									// the given amount of data
-
 	// BAbstractBufferedDataReader
 	virtual	status_t			ReadDataToOutput(off_t offset,
 									size_t size, BDataIO* output);
@@ -152,6 +148,8 @@ public:
 								OffsetArray();
 								~OffsetArray();
 
+			bool				InitUncompressedChunksOffsets(
+									size_t totalChunkCount);
 			bool				InitChunksOffsets(size_t totalChunkCount,
 									size_t baseIndex, const uint16* chunkSizes,
 									size_t chunkCount);
@@ -162,6 +160,10 @@ public:
 
 			uint64				operator[](size_t index) const;
 
+private:
+	static	uint32*				_AllocateOffsetArray(size_t totalChunkCount,
+									size_t offset32BitChunkCount);
+
 private:
 			uint32*				fOffsets;
 									// - NULL, if chunkCount <= 1

headers/private/package/hpkg/PackageReaderImpl.h

@@ -22,6 +22,7 @@ class BPackageEntryAttribute;
 namespace BPrivate {
 
 
+struct hpkg_header;
 class PackageWriterImpl;
 
 
@@ -34,7 +35,7 @@ public:
 			status_t			Init(const char* fileName, uint32 flags);
 			status_t			Init(int fd, bool keepFD, uint32 flags);
 			status_t			Init(BPositionIO* file, bool keepFile,
-									uint32 flags);
+									uint32 flags, hpkg_header* _header = NULL);
 			status_t			ParseContent(
 									BPackageContentHandler* contentHandler);
 			status_t			ParseContent(BLowLevelPackageContentHandler*

headers/private/package/hpkg/WriterImplBase.h

@@ -101,8 +101,11 @@ protected:
 			typedef DoublyLinkedList<PackageAttribute> PackageAttributeList;
 
 protected:
-			status_t			Init(const char* fileName, size_t headerSize,
+			status_t			Init(const char* fileName,
 									const BPackageWriterParameters& parameters);
+			status_t			InitHeapReader(size_t headerSize);
+
+			void				SetCompression(uint32 compression);
 
 			void				RegisterPackageInfo(
 									PackageAttributeList& attributeList,

src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp

@@ -43,6 +43,45 @@ PackageFileHeapAccessorBase::OffsetArray::~OffsetArray()
 }
 
 
+bool
+PackageFileHeapAccessorBase::OffsetArray::InitUncompressedChunksOffsets(
+	size_t totalChunkCount)
+{
+	if (totalChunkCount <= 1)
+		return true;
+
+	const size_t max32BitChunks = (uint64(1) << 32) / kChunkSize;
+	size_t actual32BitChunks = totalChunkCount;
+	if (totalChunkCount - 1 > max32BitChunks) {
+		actual32BitChunks = max32BitChunks;
+		fOffsets = _AllocateOffsetArray(totalChunkCount, max32BitChunks);
+	} else
+		fOffsets = _AllocateOffsetArray(totalChunkCount, 0);
+
+	if (fOffsets == NULL)
+		return false;
+
+	{
+		uint32 offset = kChunkSize;
+		for (size_t i = 1; i < actual32BitChunks; i++, offset += kChunkSize)
+			fOffsets[i] = offset;
+
+	}
+
+	if (actual32BitChunks < totalChunkCount) {
+		uint64 offset = actual32BitChunks * kChunkSize;
+		uint32* offsets = fOffsets + actual32BitChunks;
+		for (size_t i = actual32BitChunks; i < totalChunkCount;
+				i++, offset += kChunkSize) {
+			*offsets++ = (uint32)offset;
+			*offsets++ = uint32(offset >> 32);
+		}
+	}
+
+	return true;
+}
+
+
 bool
 PackageFileHeapAccessorBase::OffsetArray::InitChunksOffsets(
 	size_t totalChunkCount, size_t baseIndex, const uint16* chunkSizes,
@@ -52,12 +91,9 @@ PackageFileHeapAccessorBase::OffsetArray::InitChunksOffsets(
 		return true;
 
 	if (fOffsets == NULL) {
-		fOffsets = new(std::nothrow) uint32[totalChunkCount];
+		fOffsets = _AllocateOffsetArray(totalChunkCount, totalChunkCount);
 		if (fOffsets == NULL)
 			return false;
-		fOffsets[0] = 0;
-			// Value that serves as a marker that all offsets are 32 bit. We'll
-			// replace it, when necessary.
 	}
 
 	uint64 offset = (*this)[baseIndex];
@@ -74,14 +110,13 @@ PackageFileHeapAccessorBase::OffsetArray::InitChunksOffsets(
 		} else {
 			if (fOffsets[0] == 0) {
 				// Not scaled to allow for 64 bit offsets yet. Do that.
-				fOffsets[0] = index;
+				uint32* newOffsets = _AllocateOffsetArray(totalChunkCount,
-				uint32* newOffsets = new(std::nothrow) uint32[
+					index);
-					2 * totalChunkCount - fOffsets[0]];
 				if (newOffsets == NULL)
 					return false;
 
-				memcpy(newOffsets, fOffsets,
+				fOffsets[0] = index;
-					sizeof(newOffsets[0]) * fOffsets[0]);
+				memcpy(newOffsets, fOffsets, sizeof(newOffsets[0]) * index);
 
 				delete[] fOffsets;
 				fOffsets = newOffsets;
@@ -117,6 +152,24 @@ PackageFileHeapAccessorBase::OffsetArray::Init(size_t totalChunkCount,
 }
 
 
+/*static*/ uint32*
+PackageFileHeapAccessorBase::OffsetArray::_AllocateOffsetArray(
+	size_t totalChunkCount, size_t offset32BitChunkCount)
+{
+	uint32* offsets = new(std::nothrow) uint32[
+		2 * totalChunkCount - offset32BitChunkCount];
+
+	if (offsets != NULL) {
+		offsets[0] = offset32BitChunkCount == totalChunkCount
+			? 0 : offset32BitChunkCount;
+			// 0 means that all offsets are 32 bit. Otherwise it's the index of
+			// the first 64 bit offset.
+	}
+
+	return offsets;
+}
+
+
 // #pragma mark - PackageFileHeapAccessorBase
 
 
@@ -143,16 +196,6 @@ PackageFileHeapAccessorBase::~PackageFileHeapAccessorBase()
 }
 
 
-uint64
-PackageFileHeapAccessorBase::HeapOverhead(uint64 uncompressedSize) const
-{
-	// Determine number of chunks and the size of the chunk size table. Note
-	// that the size of the last chunk is not saved, since its size is implied.
-	size_t chunkCount = (uncompressedSize + kChunkSize - 1) / kChunkSize;
-	return chunkCount > 1 ? (chunkCount - 1) * 2 : 0;
-}
-
-
 status_t
 PackageFileHeapAccessorBase::ReadDataToOutput(off_t offset, size_t size,
 	BDataIO* output)

src/kits/package/hpkg/PackageFileHeapReader.cpp

@@ -61,6 +61,23 @@ PackageFileHeapReader::Init()
 	if (chunkCount == 0)
 		return B_OK;
 
+	// If no compression is used at all, the chunk size table is omitted. Handle
+	// this case.
+	if (fDecompressionAlgorithm == NULL) {
+		if (fUncompressedHeapSize != fCompressedHeapSize) {
+			fErrorOutput->PrintError(
+				"Compressed and uncompressed heap sizes (%" B_PRIu64 " vs. "
+				"%" B_PRIu64 ") don't match for uncompressed heap.\n",
+				fCompressedHeapSize, fUncompressedHeapSize);
+			return B_BAD_DATA;
+		}
+
+		if (!fOffsets.InitUncompressedChunksOffsets(chunkCount))
+			return B_NO_MEMORY;
+
+		return B_OK;
+	}
+
 	size_t chunkSizeTableSize = (chunkCount - 1) * 2; 
 	if (fCompressedHeapSize <= chunkSizeTableSize) {
 		fErrorOutput->PrintError(

src/kits/package/hpkg/PackageFileHeapWriter.cpp

@@ -443,6 +443,10 @@ PackageFileHeapWriter::Finish()
 
 	// write chunk sizes table 
 
+	// We don't need to do that, if we don't use any compression.
+	if (fCompressionAlgorithm == NULL)
+		return B_OK;
+
 	// We don't need to write the last chunk size, since it is implied by the
 	// total size minus the sum of all other chunk sizes.
 	ssize_t offsetCount = fOffsets.Count();

src/kits/package/hpkg/PackageReaderImpl.cpp

@@ -346,7 +346,8 @@ PackageReaderImpl::Init(int fd, bool keepFD, uint32 flags)
 
 
 status_t
-PackageReaderImpl::Init(BPositionIO* file, bool keepFile, uint32 flags)
+PackageReaderImpl::Init(BPositionIO* file, bool keepFile, uint32 flags,
+	hpkg_header* _header)
 {
 	hpkg_header header;
 	status_t error = inherited::Init<hpkg_header, B_HPKG_MAGIC, B_HPKG_VERSION,
@@ -381,6 +382,9 @@ PackageReaderImpl::Init(BPositionIO* file, bool keepFile, uint32 flags)
 	if (error != B_OK)
 		return error;
 
+	if (_header != NULL)
+		*_header = header;
+
 	return B_OK;
 }
 

src/kits/package/hpkg/PackageWriter.cpp

@@ -24,6 +24,7 @@ namespace BHPKG {
 BPackageWriterParameters::BPackageWriterParameters()
 	:
 	fFlags(0),
+	fCompression(B_HPKG_COMPRESSION_ZLIB),
 	fCompressionLevel(B_HPKG_COMPRESSION_LEVEL_BEST)
 {
 }
@@ -48,6 +49,20 @@ BPackageWriterParameters::SetFlags(uint32 flags)
 }
 
 
+uint32
+BPackageWriterParameters::Compression() const
+{
+	return fCompression;
+}
+
+
+void
+BPackageWriterParameters::SetCompression(uint32 compression)
+{
+	fCompression = compression;
+}
+
+
 int32
 BPackageWriterParameters::CompressionLevel() const
 {

src/kits/package/hpkg/PackageWriterImpl.cpp

@@ -627,8 +627,7 @@ status_t
 PackageWriterImpl::_Init(const char* fileName,
 	const BPackageWriterParameters& parameters)
 {
-	status_t result = inherited::Init(fileName, sizeof(hpkg_header),
+	status_t result = inherited::Init(fileName, parameters);
-		parameters);
 	if (result != B_OK)
 		return result;
 
@@ -648,7 +647,8 @@ PackageWriterImpl::_Init(const char* fileName,
 	// in update mode, parse the TOC
 	if ((Flags() & B_HPKG_WRITER_UPDATE_PACKAGE) != 0) {
 		PackageReaderImpl packageReader(fListener);
-		result = packageReader.Init(File(), false, 0);
+		hpkg_header header;
+		result = packageReader.Init(File(), false, 0, &header);
 		if (result != B_OK)
 			return result;
 
@@ -660,6 +660,14 @@ PackageWriterImpl::_Init(const char* fileName,
 		if (result != B_OK)
 			return result;
 
+		// While the compression level can change, we have to reuse the
+		// compression algorithm at least.
+		SetCompression(B_BENDIAN_TO_HOST_INT16(header.heap_compression));
+
+		result = InitHeapReader(fHeapOffset);
+		if (result != B_OK)
+			return result;
+
 		fHeapWriter->Reinit(packageReader.RawHeapReader());
 
 		// Remove the old packages attributes and TOC section from the heap.
@@ -673,6 +681,10 @@ PackageWriterImpl::_Init(const char* fileName,
 				tocSection.uncompressedLength)) {
 			throw std::bad_alloc();
 		}
+	} else {
+		result = InitHeapReader(fHeapOffset);
+		if (result != B_OK)
+			return result;
 	}
 
 	return B_OK;
@@ -706,7 +718,8 @@ PackageWriterImpl::_Finish()
 
 	uint64 compressedHeapSize = fHeapWriter->CompressedHeapSize();
 
-	header.heap_compression = B_HOST_TO_BENDIAN_INT16(B_HPKG_COMPRESSION_ZLIB);
+	header.heap_compression = B_HOST_TO_BENDIAN_INT16(
+		Parameters().Compression());
 	header.heap_chunk_size = B_HOST_TO_BENDIAN_INT32(fHeapWriter->ChunkSize());
 	header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(compressedHeapSize);
 	header.heap_size_uncompressed = B_HOST_TO_BENDIAN_INT64(
@@ -763,20 +776,24 @@ PackageWriterImpl::_Recompress(PackageReaderImpl* reader)
 	// for streaming an uncompressed package.
 	uint64 uncompressedHeapSize
 		= reader->RawHeapReader()->UncompressedHeapSize();
-	uint64 compressedHeapSize = uncompressedHeapSize
+	uint64 compressedHeapSize = uncompressedHeapSize;
-		+ fHeapWriter->HeapOverhead(uncompressedHeapSize);
 
 	off_t totalSize = fHeapWriter->HeapOffset() + (off_t)compressedHeapSize;
 
-	header.heap_compression = B_HOST_TO_BENDIAN_INT16(B_HPKG_COMPRESSION_ZLIB);
+	header.heap_compression = B_HOST_TO_BENDIAN_INT16(
+		Parameters().Compression());
 	header.heap_chunk_size = B_HOST_TO_BENDIAN_INT32(fHeapWriter->ChunkSize());
-	header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(compressedHeapSize);
 	header.heap_size_uncompressed
 		= B_HOST_TO_BENDIAN_INT64(uncompressedHeapSize);
-	header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize);
 
-	if (Parameters().CompressionLevel() == 0)
+	if (Parameters().Compression() == B_HPKG_COMPRESSION_NONE) {
+		header.heap_size_compressed
+			= B_HOST_TO_BENDIAN_INT64(compressedHeapSize);
+		header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize);
+
+		// write the header
 		RawWriteBuffer(&header, sizeof(hpkg_header), 0);
+	}
 
 	// copy the heap data
 	uint64 bytesCompressed;
@@ -791,7 +808,7 @@ PackageWriterImpl::_Recompress(PackageReaderImpl* reader)
 		return error;
 
 	// If compression is enabled, update and write the header.
-	if (Parameters().CompressionLevel() != 0) {
+	if (Parameters().Compression() != B_HPKG_COMPRESSION_NONE) {
 		compressedHeapSize = fHeapWriter->CompressedHeapSize();
 		totalSize = fHeapWriter->HeapOffset() + (off_t)compressedHeapSize;
 		header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(compressedHeapSize);

src/kits/package/hpkg/ReaderImplBase.cpp

@@ -807,22 +807,26 @@ status_t
 ReaderImplBase::InitHeapReader(uint32 compression, uint32 chunkSize,
 	off_t offset, uint64 compressedSize, uint64 uncompressedSize)
 {
-	if (compression != B_HPKG_COMPRESSION_ZLIB) {
+	DecompressionAlgorithmOwner* decompressionAlgorithm = NULL;
-		fErrorOutput->PrintError("Error: Invalid heap compression\n");
+	BReference<DecompressionAlgorithmOwner> decompressionAlgorithmReference;
-		return B_BAD_DATA;
-	}
 
-	DecompressionAlgorithmOwner* decompressionAlgorithm
+	switch (compression) {
-		= DecompressionAlgorithmOwner::Create(
+		case B_HPKG_COMPRESSION_NONE:
-			new(std::nothrow) BZlibCompressionAlgorithm,
+			break;
-			new(std::nothrow) BZlibDecompressionParameters);
+		case B_HPKG_COMPRESSION_ZLIB:
-	BReference<DecompressionAlgorithmOwner> decompressionAlgorithmReference(
+			decompressionAlgorithm = DecompressionAlgorithmOwner::Create(
-		decompressionAlgorithm, true);
+				new(std::nothrow) BZlibCompressionAlgorithm,
-
+				new(std::nothrow) BZlibDecompressionParameters);
-	if (decompressionAlgorithm == NULL
+			decompressionAlgorithmReference.SetTo(decompressionAlgorithm, true);
-		|| decompressionAlgorithm->algorithm == NULL
+			if (decompressionAlgorithm == NULL
-		|| decompressionAlgorithm->parameters == NULL) {
+				|| decompressionAlgorithm->algorithm == NULL
-		return B_NO_MEMORY;
+				|| decompressionAlgorithm->parameters == NULL) {
+				return B_NO_MEMORY;
+			}
+			break;
+		default:
+			fErrorOutput->PrintError("Error: Invalid heap compression\n");
+			return B_BAD_DATA;
 	}
 
 	fRawHeapReader = new(std::nothrow) PackageFileHeapReader(fErrorOutput,

src/kits/package/hpkg/RepositoryWriterImpl.cpp

@@ -264,8 +264,11 @@ RepositoryWriterImpl::Finish()
 status_t
 RepositoryWriterImpl::_Init(const char* fileName)
 {
-	return inherited::Init(fileName, sizeof(hpkg_repo_header),
+	status_t error = inherited::Init(fileName, BPackageWriterParameters());
-		BPackageWriterParameters());
+	if (error != B_OK)
+		return error;
+
+	return InitHeapReader(sizeof(hpkg_repo_header));
 }
 
 
@@ -291,7 +294,8 @@ RepositoryWriterImpl::_Finish()
 	uint64 compressedHeapSize = fHeapWriter->CompressedHeapSize();
 	uint64 totalSize = fHeapWriter->HeapOffset() + compressedHeapSize;
 
-	header.heap_compression = B_HOST_TO_BENDIAN_INT16(B_HPKG_COMPRESSION_ZLIB);
+	header.heap_compression = B_HOST_TO_BENDIAN_INT16(
+		Parameters().Compression());
 	header.heap_chunk_size = B_HOST_TO_BENDIAN_INT32(fHeapWriter->ChunkSize());
 	header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(compressedHeapSize);
 	header.heap_size_uncompressed = B_HOST_TO_BENDIAN_INT64(

src/kits/package/hpkg/WriterImplBase.cpp

@@ -251,7 +251,7 @@ WriterImplBase::~WriterImplBase()
 
 
 status_t
-WriterImplBase::Init(const char* fileName, size_t headerSize,
+WriterImplBase::Init(const char* fileName,
 	const BPackageWriterParameters& parameters)
 {
 	fParameters = parameters;
@@ -276,37 +276,49 @@ WriterImplBase::Init(const char* fileName, size_t headerSize,
 	fFile = file;
 	fFileName = fileName;
 
-	DecompressionAlgorithmOwner* decompressionAlgorithm
+	return B_OK;
-		= DecompressionAlgorithmOwner::Create(
+}
-			new(std::nothrow) BZlibCompressionAlgorithm,
-			new(std::nothrow) BZlibDecompressionParameters);
-	BReference<DecompressionAlgorithmOwner> decompressionAlgorithmReference(
-		decompressionAlgorithm, true);
 
-	if (decompressionAlgorithm == NULL
-		|| decompressionAlgorithm->algorithm == NULL
-		|| decompressionAlgorithm->parameters == NULL) {
-		throw std::bad_alloc();
-	}
 
+status_t
+WriterImplBase::InitHeapReader(size_t headerSize)
+{
+	// allocate the compression/decompression algorithm
 	CompressionAlgorithmOwner* compressionAlgorithm = NULL;
+	BReference<CompressionAlgorithmOwner> compressionAlgorithmReference;
 
-	if (fParameters.CompressionLevel() != B_HPKG_COMPRESSION_LEVEL_NONE) {
+	DecompressionAlgorithmOwner* decompressionAlgorithm = NULL;
-		compressionAlgorithm = CompressionAlgorithmOwner::Create(
+	BReference<DecompressionAlgorithmOwner> decompressionAlgorithmReference;
-			new(std::nothrow) BZlibCompressionAlgorithm,
-			new(std::nothrow) BZlibCompressionParameters(
-				fParameters.CompressionLevel()));
 
-		if (compressionAlgorithm == NULL
+	switch (fParameters.Compression()) {
-			|| compressionAlgorithm->algorithm == NULL
+		case B_HPKG_COMPRESSION_NONE:
-			|| compressionAlgorithm->parameters == NULL) {
+			break;
-			throw std::bad_alloc();
+		case B_HPKG_COMPRESSION_ZLIB:
-		}
+			compressionAlgorithm = CompressionAlgorithmOwner::Create(
+				new(std::nothrow) BZlibCompressionAlgorithm,
+				new(std::nothrow) BZlibCompressionParameters(
+					fParameters.CompressionLevel()));
+			compressionAlgorithmReference.SetTo(compressionAlgorithm, true);
+
+			decompressionAlgorithm = DecompressionAlgorithmOwner::Create(
+				new(std::nothrow) BZlibCompressionAlgorithm,
+				new(std::nothrow) BZlibDecompressionParameters);
+			decompressionAlgorithmReference.SetTo(decompressionAlgorithm, true);
+
+			if (compressionAlgorithm == NULL
+				|| compressionAlgorithm->algorithm == NULL
+				|| compressionAlgorithm->parameters == NULL
+				|| decompressionAlgorithm == NULL
+				|| decompressionAlgorithm->algorithm == NULL
+				|| decompressionAlgorithm->parameters == NULL) {
+				throw std::bad_alloc();
+			}
+			break;
+		default:
+			fErrorOutput->PrintError("Error: Invalid heap compression\n");
+			return B_BAD_VALUE;
 	}
 
-	BReference<CompressionAlgorithmOwner> compressionAlgorithmReference(
-		compressionAlgorithm, true);
-
 	// create heap writer
 	fHeapWriter = new PackageFileHeapWriter(fErrorOutput, fFile, headerSize,
 		compressionAlgorithm, decompressionAlgorithm);
@@ -316,6 +328,13 @@ WriterImplBase::Init(const char* fileName, size_t headerSize,
 }
 
 
+void
+WriterImplBase::SetCompression(uint32 compression)
+{
+	fParameters.SetCompression(compression);
+}
+
+
 void
 WriterImplBase::RegisterPackageInfo(PackageAttributeList& attributeList,
 	const BPackageInfo& packageInfo)