From 8f5130edfa454cc7ee21e4c3aef2a151f7c3c567 Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Sat, 12 Jul 2014 23:12:21 +0200 Subject: [PATCH] package kit: Actually add support for B_HPKG_COMPRESSION_NONE Until now we always declared in the HPKG header that the package file is zlib compressed. For uncompressed files we would just store all individual chunks uncompressed. Now we handle completely uncompressed files slightly differently: We don't write the redundant chunk size table anymore. The size savings are minor, but it makes the uncompressed format read-streamable which may be handy. --- headers/os/package/hpkg/PackageWriter.h | 4 + .../hpkg/PackageFileHeapAccessorBase.h | 10 ++- .../private/package/hpkg/PackageReaderImpl.h | 3 +- headers/private/package/hpkg/WriterImplBase.h | 5 +- .../hpkg/PackageFileHeapAccessorBase.cpp | 81 ++++++++++++++----- .../package/hpkg/PackageFileHeapReader.cpp | 17 ++++ .../package/hpkg/PackageFileHeapWriter.cpp | 4 + src/kits/package/hpkg/PackageReaderImpl.cpp | 6 +- src/kits/package/hpkg/PackageWriter.cpp | 15 ++++ src/kits/package/hpkg/PackageWriterImpl.cpp | 39 ++++++--- src/kits/package/hpkg/ReaderImplBase.cpp | 34 ++++---- .../package/hpkg/RepositoryWriterImpl.cpp | 10 ++- src/kits/package/hpkg/WriterImplBase.cpp | 69 ++++++++++------ 13 files changed, 217 insertions(+), 80 deletions(-) diff --git a/headers/os/package/hpkg/PackageWriter.h b/headers/os/package/hpkg/PackageWriter.h index 996f83f418..32b6ee9508 100644 --- a/headers/os/package/hpkg/PackageWriter.h +++ b/headers/os/package/hpkg/PackageWriter.h @@ -52,11 +52,15 @@ public: uint32 Flags() const; void SetFlags(uint32 flags); + uint32 Compression() const; + void SetCompression(uint32 compression); + int32 CompressionLevel() const; void SetCompressionLevel(int32 compressionLevel); private: uint32 fFlags; + uint32 fCompression; int32 fCompressionLevel; }; diff --git a/headers/private/package/hpkg/PackageFileHeapAccessorBase.h b/headers/private/package/hpkg/PackageFileHeapAccessorBase.h index b302188344..5af32a092c 100644 --- a/headers/private/package/hpkg/PackageFileHeapAccessorBase.h +++ b/headers/private/package/hpkg/PackageFileHeapAccessorBase.h @@ -92,10 +92,6 @@ public: void SetFile(BPositionIO* file) { fFile = file; } - uint64 HeapOverhead(uint64 uncompressedSize) const; - // additional bytes needed when storing - // the given amount of data - // BAbstractBufferedDataReader virtual status_t ReadDataToOutput(off_t offset, size_t size, BDataIO* output); @@ -152,6 +148,8 @@ public: OffsetArray(); ~OffsetArray(); + bool InitUncompressedChunksOffsets( + size_t totalChunkCount); bool InitChunksOffsets(size_t totalChunkCount, size_t baseIndex, const uint16* chunkSizes, size_t chunkCount); @@ -162,6 +160,10 @@ public: uint64 operator[](size_t index) const; +private: + static uint32* _AllocateOffsetArray(size_t totalChunkCount, + size_t offset32BitChunkCount); + private: uint32* fOffsets; // - NULL, if chunkCount <= 1 diff --git a/headers/private/package/hpkg/PackageReaderImpl.h b/headers/private/package/hpkg/PackageReaderImpl.h index 51a3999ae3..37cdb344b5 100644 --- a/headers/private/package/hpkg/PackageReaderImpl.h +++ b/headers/private/package/hpkg/PackageReaderImpl.h @@ -22,6 +22,7 @@ class BPackageEntryAttribute; namespace BPrivate { +struct hpkg_header; class PackageWriterImpl; @@ -34,7 +35,7 @@ public: status_t Init(const char* fileName, uint32 flags); status_t Init(int fd, bool keepFD, uint32 flags); status_t Init(BPositionIO* file, bool keepFile, - uint32 flags); + uint32 flags, hpkg_header* _header = NULL); status_t ParseContent( BPackageContentHandler* contentHandler); status_t ParseContent(BLowLevelPackageContentHandler* diff --git a/headers/private/package/hpkg/WriterImplBase.h b/headers/private/package/hpkg/WriterImplBase.h index 2f2e3266b4..c4d975f979 100644 --- a/headers/private/package/hpkg/WriterImplBase.h +++ b/headers/private/package/hpkg/WriterImplBase.h @@ -101,8 +101,11 @@ protected: typedef DoublyLinkedList PackageAttributeList; protected: - status_t Init(const char* fileName, size_t headerSize, + status_t Init(const char* fileName, const BPackageWriterParameters& parameters); + status_t InitHeapReader(size_t headerSize); + + void SetCompression(uint32 compression); void RegisterPackageInfo( PackageAttributeList& attributeList, diff --git a/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp b/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp index f8bd2da482..a93663690e 100644 --- a/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp +++ b/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp @@ -43,6 +43,45 @@ PackageFileHeapAccessorBase::OffsetArray::~OffsetArray() } +bool +PackageFileHeapAccessorBase::OffsetArray::InitUncompressedChunksOffsets( + size_t totalChunkCount) +{ + if (totalChunkCount <= 1) + return true; + + const size_t max32BitChunks = (uint64(1) << 32) / kChunkSize; + size_t actual32BitChunks = totalChunkCount; + if (totalChunkCount - 1 > max32BitChunks) { + actual32BitChunks = max32BitChunks; + fOffsets = _AllocateOffsetArray(totalChunkCount, max32BitChunks); + } else + fOffsets = _AllocateOffsetArray(totalChunkCount, 0); + + if (fOffsets == NULL) + return false; + + { + uint32 offset = kChunkSize; + for (size_t i = 1; i < actual32BitChunks; i++, offset += kChunkSize) + fOffsets[i] = offset; + + } + + if (actual32BitChunks < totalChunkCount) { + uint64 offset = actual32BitChunks * kChunkSize; + uint32* offsets = fOffsets + actual32BitChunks; + for (size_t i = actual32BitChunks; i < totalChunkCount; + i++, offset += kChunkSize) { + *offsets++ = (uint32)offset; + *offsets++ = uint32(offset >> 32); + } + } + + return true; +} + + bool PackageFileHeapAccessorBase::OffsetArray::InitChunksOffsets( size_t totalChunkCount, size_t baseIndex, const uint16* chunkSizes, @@ -52,12 +91,9 @@ PackageFileHeapAccessorBase::OffsetArray::InitChunksOffsets( return true; if (fOffsets == NULL) { - fOffsets = new(std::nothrow) uint32[totalChunkCount]; + fOffsets = _AllocateOffsetArray(totalChunkCount, totalChunkCount); if (fOffsets == NULL) return false; - fOffsets[0] = 0; - // Value that serves as a marker that all offsets are 32 bit. We'll - // replace it, when necessary. } uint64 offset = (*this)[baseIndex]; @@ -74,14 +110,13 @@ PackageFileHeapAccessorBase::OffsetArray::InitChunksOffsets( } else { if (fOffsets[0] == 0) { // Not scaled to allow for 64 bit offsets yet. Do that. - fOffsets[0] = index; - uint32* newOffsets = new(std::nothrow) uint32[ - 2 * totalChunkCount - fOffsets[0]]; + uint32* newOffsets = _AllocateOffsetArray(totalChunkCount, + index); if (newOffsets == NULL) return false; - memcpy(newOffsets, fOffsets, - sizeof(newOffsets[0]) * fOffsets[0]); + fOffsets[0] = index; + memcpy(newOffsets, fOffsets, sizeof(newOffsets[0]) * index); delete[] fOffsets; fOffsets = newOffsets; @@ -117,6 +152,24 @@ PackageFileHeapAccessorBase::OffsetArray::Init(size_t totalChunkCount, } +/*static*/ uint32* +PackageFileHeapAccessorBase::OffsetArray::_AllocateOffsetArray( + size_t totalChunkCount, size_t offset32BitChunkCount) +{ + uint32* offsets = new(std::nothrow) uint32[ + 2 * totalChunkCount - offset32BitChunkCount]; + + if (offsets != NULL) { + offsets[0] = offset32BitChunkCount == totalChunkCount + ? 0 : offset32BitChunkCount; + // 0 means that all offsets are 32 bit. Otherwise it's the index of + // the first 64 bit offset. + } + + return offsets; +} + + // #pragma mark - PackageFileHeapAccessorBase @@ -143,16 +196,6 @@ PackageFileHeapAccessorBase::~PackageFileHeapAccessorBase() } -uint64 -PackageFileHeapAccessorBase::HeapOverhead(uint64 uncompressedSize) const -{ - // Determine number of chunks and the size of the chunk size table. Note - // that the size of the last chunk is not saved, since its size is implied. - size_t chunkCount = (uncompressedSize + kChunkSize - 1) / kChunkSize; - return chunkCount > 1 ? (chunkCount - 1) * 2 : 0; -} - - status_t PackageFileHeapAccessorBase::ReadDataToOutput(off_t offset, size_t size, BDataIO* output) diff --git a/src/kits/package/hpkg/PackageFileHeapReader.cpp b/src/kits/package/hpkg/PackageFileHeapReader.cpp index 9af47d08db..fb1449b63f 100644 --- a/src/kits/package/hpkg/PackageFileHeapReader.cpp +++ b/src/kits/package/hpkg/PackageFileHeapReader.cpp @@ -61,6 +61,23 @@ PackageFileHeapReader::Init() if (chunkCount == 0) return B_OK; + // If no compression is used at all, the chunk size table is omitted. Handle + // this case. + if (fDecompressionAlgorithm == NULL) { + if (fUncompressedHeapSize != fCompressedHeapSize) { + fErrorOutput->PrintError( + "Compressed and uncompressed heap sizes (%" B_PRIu64 " vs. " + "%" B_PRIu64 ") don't match for uncompressed heap.\n", + fCompressedHeapSize, fUncompressedHeapSize); + return B_BAD_DATA; + } + + if (!fOffsets.InitUncompressedChunksOffsets(chunkCount)) + return B_NO_MEMORY; + + return B_OK; + } + size_t chunkSizeTableSize = (chunkCount - 1) * 2; if (fCompressedHeapSize <= chunkSizeTableSize) { fErrorOutput->PrintError( diff --git a/src/kits/package/hpkg/PackageFileHeapWriter.cpp b/src/kits/package/hpkg/PackageFileHeapWriter.cpp index 1a17ef1647..3a5800ccd8 100644 --- a/src/kits/package/hpkg/PackageFileHeapWriter.cpp +++ b/src/kits/package/hpkg/PackageFileHeapWriter.cpp @@ -443,6 +443,10 @@ PackageFileHeapWriter::Finish() // write chunk sizes table + // We don't need to do that, if we don't use any compression. + if (fCompressionAlgorithm == NULL) + return B_OK; + // We don't need to write the last chunk size, since it is implied by the // total size minus the sum of all other chunk sizes. ssize_t offsetCount = fOffsets.Count(); diff --git a/src/kits/package/hpkg/PackageReaderImpl.cpp b/src/kits/package/hpkg/PackageReaderImpl.cpp index 4bc00739dc..1ff2bf2d29 100644 --- a/src/kits/package/hpkg/PackageReaderImpl.cpp +++ b/src/kits/package/hpkg/PackageReaderImpl.cpp @@ -346,7 +346,8 @@ PackageReaderImpl::Init(int fd, bool keepFD, uint32 flags) status_t -PackageReaderImpl::Init(BPositionIO* file, bool keepFile, uint32 flags) +PackageReaderImpl::Init(BPositionIO* file, bool keepFile, uint32 flags, + hpkg_header* _header) { hpkg_header header; status_t error = inherited::InitReinit(packageReader.RawHeapReader()); // Remove the old packages attributes and TOC section from the heap. @@ -673,6 +681,10 @@ PackageWriterImpl::_Init(const char* fileName, tocSection.uncompressedLength)) { throw std::bad_alloc(); } + } else { + result = InitHeapReader(fHeapOffset); + if (result != B_OK) + return result; } return B_OK; @@ -706,7 +718,8 @@ PackageWriterImpl::_Finish() uint64 compressedHeapSize = fHeapWriter->CompressedHeapSize(); - header.heap_compression = B_HOST_TO_BENDIAN_INT16(B_HPKG_COMPRESSION_ZLIB); + header.heap_compression = B_HOST_TO_BENDIAN_INT16( + Parameters().Compression()); header.heap_chunk_size = B_HOST_TO_BENDIAN_INT32(fHeapWriter->ChunkSize()); header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(compressedHeapSize); header.heap_size_uncompressed = B_HOST_TO_BENDIAN_INT64( @@ -763,20 +776,24 @@ PackageWriterImpl::_Recompress(PackageReaderImpl* reader) // for streaming an uncompressed package. uint64 uncompressedHeapSize = reader->RawHeapReader()->UncompressedHeapSize(); - uint64 compressedHeapSize = uncompressedHeapSize - + fHeapWriter->HeapOverhead(uncompressedHeapSize); + uint64 compressedHeapSize = uncompressedHeapSize; off_t totalSize = fHeapWriter->HeapOffset() + (off_t)compressedHeapSize; - header.heap_compression = B_HOST_TO_BENDIAN_INT16(B_HPKG_COMPRESSION_ZLIB); + header.heap_compression = B_HOST_TO_BENDIAN_INT16( + Parameters().Compression()); header.heap_chunk_size = B_HOST_TO_BENDIAN_INT32(fHeapWriter->ChunkSize()); - header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(compressedHeapSize); header.heap_size_uncompressed = B_HOST_TO_BENDIAN_INT64(uncompressedHeapSize); - header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize); - if (Parameters().CompressionLevel() == 0) + if (Parameters().Compression() == B_HPKG_COMPRESSION_NONE) { + header.heap_size_compressed + = B_HOST_TO_BENDIAN_INT64(compressedHeapSize); + header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize); + + // write the header RawWriteBuffer(&header, sizeof(hpkg_header), 0); + } // copy the heap data uint64 bytesCompressed; @@ -791,7 +808,7 @@ PackageWriterImpl::_Recompress(PackageReaderImpl* reader) return error; // If compression is enabled, update and write the header. - if (Parameters().CompressionLevel() != 0) { + if (Parameters().Compression() != B_HPKG_COMPRESSION_NONE) { compressedHeapSize = fHeapWriter->CompressedHeapSize(); totalSize = fHeapWriter->HeapOffset() + (off_t)compressedHeapSize; header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(compressedHeapSize); diff --git a/src/kits/package/hpkg/ReaderImplBase.cpp b/src/kits/package/hpkg/ReaderImplBase.cpp index eb301f7602..bb3bb1b05e 100644 --- a/src/kits/package/hpkg/ReaderImplBase.cpp +++ b/src/kits/package/hpkg/ReaderImplBase.cpp @@ -807,22 +807,26 @@ status_t ReaderImplBase::InitHeapReader(uint32 compression, uint32 chunkSize, off_t offset, uint64 compressedSize, uint64 uncompressedSize) { - if (compression != B_HPKG_COMPRESSION_ZLIB) { - fErrorOutput->PrintError("Error: Invalid heap compression\n"); - return B_BAD_DATA; - } + DecompressionAlgorithmOwner* decompressionAlgorithm = NULL; + BReference decompressionAlgorithmReference; - DecompressionAlgorithmOwner* decompressionAlgorithm - = DecompressionAlgorithmOwner::Create( - new(std::nothrow) BZlibCompressionAlgorithm, - new(std::nothrow) BZlibDecompressionParameters); - BReference decompressionAlgorithmReference( - decompressionAlgorithm, true); - - if (decompressionAlgorithm == NULL - || decompressionAlgorithm->algorithm == NULL - || decompressionAlgorithm->parameters == NULL) { - return B_NO_MEMORY; + switch (compression) { + case B_HPKG_COMPRESSION_NONE: + break; + case B_HPKG_COMPRESSION_ZLIB: + decompressionAlgorithm = DecompressionAlgorithmOwner::Create( + new(std::nothrow) BZlibCompressionAlgorithm, + new(std::nothrow) BZlibDecompressionParameters); + decompressionAlgorithmReference.SetTo(decompressionAlgorithm, true); + if (decompressionAlgorithm == NULL + || decompressionAlgorithm->algorithm == NULL + || decompressionAlgorithm->parameters == NULL) { + return B_NO_MEMORY; + } + break; + default: + fErrorOutput->PrintError("Error: Invalid heap compression\n"); + return B_BAD_DATA; } fRawHeapReader = new(std::nothrow) PackageFileHeapReader(fErrorOutput, diff --git a/src/kits/package/hpkg/RepositoryWriterImpl.cpp b/src/kits/package/hpkg/RepositoryWriterImpl.cpp index d5be01ce5b..cda6ff04d7 100644 --- a/src/kits/package/hpkg/RepositoryWriterImpl.cpp +++ b/src/kits/package/hpkg/RepositoryWriterImpl.cpp @@ -264,8 +264,11 @@ RepositoryWriterImpl::Finish() status_t RepositoryWriterImpl::_Init(const char* fileName) { - return inherited::Init(fileName, sizeof(hpkg_repo_header), - BPackageWriterParameters()); + status_t error = inherited::Init(fileName, BPackageWriterParameters()); + if (error != B_OK) + return error; + + return InitHeapReader(sizeof(hpkg_repo_header)); } @@ -291,7 +294,8 @@ RepositoryWriterImpl::_Finish() uint64 compressedHeapSize = fHeapWriter->CompressedHeapSize(); uint64 totalSize = fHeapWriter->HeapOffset() + compressedHeapSize; - header.heap_compression = B_HOST_TO_BENDIAN_INT16(B_HPKG_COMPRESSION_ZLIB); + header.heap_compression = B_HOST_TO_BENDIAN_INT16( + Parameters().Compression()); header.heap_chunk_size = B_HOST_TO_BENDIAN_INT32(fHeapWriter->ChunkSize()); header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(compressedHeapSize); header.heap_size_uncompressed = B_HOST_TO_BENDIAN_INT64( diff --git a/src/kits/package/hpkg/WriterImplBase.cpp b/src/kits/package/hpkg/WriterImplBase.cpp index 51fdabb4a4..65d3016c73 100644 --- a/src/kits/package/hpkg/WriterImplBase.cpp +++ b/src/kits/package/hpkg/WriterImplBase.cpp @@ -251,7 +251,7 @@ WriterImplBase::~WriterImplBase() status_t -WriterImplBase::Init(const char* fileName, size_t headerSize, +WriterImplBase::Init(const char* fileName, const BPackageWriterParameters& parameters) { fParameters = parameters; @@ -276,37 +276,49 @@ WriterImplBase::Init(const char* fileName, size_t headerSize, fFile = file; fFileName = fileName; - DecompressionAlgorithmOwner* decompressionAlgorithm - = DecompressionAlgorithmOwner::Create( - new(std::nothrow) BZlibCompressionAlgorithm, - new(std::nothrow) BZlibDecompressionParameters); - BReference decompressionAlgorithmReference( - decompressionAlgorithm, true); + return B_OK; +} - if (decompressionAlgorithm == NULL - || decompressionAlgorithm->algorithm == NULL - || decompressionAlgorithm->parameters == NULL) { - throw std::bad_alloc(); - } +status_t +WriterImplBase::InitHeapReader(size_t headerSize) +{ + // allocate the compression/decompression algorithm CompressionAlgorithmOwner* compressionAlgorithm = NULL; + BReference compressionAlgorithmReference; - if (fParameters.CompressionLevel() != B_HPKG_COMPRESSION_LEVEL_NONE) { - compressionAlgorithm = CompressionAlgorithmOwner::Create( - new(std::nothrow) BZlibCompressionAlgorithm, - new(std::nothrow) BZlibCompressionParameters( - fParameters.CompressionLevel())); + DecompressionAlgorithmOwner* decompressionAlgorithm = NULL; + BReference decompressionAlgorithmReference; - if (compressionAlgorithm == NULL - || compressionAlgorithm->algorithm == NULL - || compressionAlgorithm->parameters == NULL) { - throw std::bad_alloc(); - } + switch (fParameters.Compression()) { + case B_HPKG_COMPRESSION_NONE: + break; + case B_HPKG_COMPRESSION_ZLIB: + compressionAlgorithm = CompressionAlgorithmOwner::Create( + new(std::nothrow) BZlibCompressionAlgorithm, + new(std::nothrow) BZlibCompressionParameters( + fParameters.CompressionLevel())); + compressionAlgorithmReference.SetTo(compressionAlgorithm, true); + + decompressionAlgorithm = DecompressionAlgorithmOwner::Create( + new(std::nothrow) BZlibCompressionAlgorithm, + new(std::nothrow) BZlibDecompressionParameters); + decompressionAlgorithmReference.SetTo(decompressionAlgorithm, true); + + if (compressionAlgorithm == NULL + || compressionAlgorithm->algorithm == NULL + || compressionAlgorithm->parameters == NULL + || decompressionAlgorithm == NULL + || decompressionAlgorithm->algorithm == NULL + || decompressionAlgorithm->parameters == NULL) { + throw std::bad_alloc(); + } + break; + default: + fErrorOutput->PrintError("Error: Invalid heap compression\n"); + return B_BAD_VALUE; } - BReference compressionAlgorithmReference( - compressionAlgorithm, true); - // create heap writer fHeapWriter = new PackageFileHeapWriter(fErrorOutput, fFile, headerSize, compressionAlgorithm, decompressionAlgorithm); @@ -316,6 +328,13 @@ WriterImplBase::Init(const char* fileName, size_t headerSize, } +void +WriterImplBase::SetCompression(uint32 compression) +{ + fParameters.SetCompression(compression); +} + + void WriterImplBase::RegisterPackageInfo(PackageAttributeList& attributeList, const BPackageInfo& packageInfo)