diff --git a/headers/os/package/hpkg/PackageWriter.h b/headers/os/package/hpkg/PackageWriter.h index 996f83f418..32b6ee9508 100644 --- a/headers/os/package/hpkg/PackageWriter.h +++ b/headers/os/package/hpkg/PackageWriter.h @@ -52,11 +52,15 @@ public: uint32 Flags() const; void SetFlags(uint32 flags); + uint32 Compression() const; + void SetCompression(uint32 compression); + int32 CompressionLevel() const; void SetCompressionLevel(int32 compressionLevel); private: uint32 fFlags; + uint32 fCompression; int32 fCompressionLevel; }; diff --git a/headers/private/package/hpkg/PackageFileHeapAccessorBase.h b/headers/private/package/hpkg/PackageFileHeapAccessorBase.h index b302188344..5af32a092c 100644 --- a/headers/private/package/hpkg/PackageFileHeapAccessorBase.h +++ b/headers/private/package/hpkg/PackageFileHeapAccessorBase.h @@ -92,10 +92,6 @@ public: void SetFile(BPositionIO* file) { fFile = file; } - uint64 HeapOverhead(uint64 uncompressedSize) const; - // additional bytes needed when storing - // the given amount of data - // BAbstractBufferedDataReader virtual status_t ReadDataToOutput(off_t offset, size_t size, BDataIO* output); @@ -152,6 +148,8 @@ public: OffsetArray(); ~OffsetArray(); + bool InitUncompressedChunksOffsets( + size_t totalChunkCount); bool InitChunksOffsets(size_t totalChunkCount, size_t baseIndex, const uint16* chunkSizes, size_t chunkCount); @@ -162,6 +160,10 @@ public: uint64 operator[](size_t index) const; +private: + static uint32* _AllocateOffsetArray(size_t totalChunkCount, + size_t offset32BitChunkCount); + private: uint32* fOffsets; // - NULL, if chunkCount <= 1 diff --git a/headers/private/package/hpkg/PackageReaderImpl.h b/headers/private/package/hpkg/PackageReaderImpl.h index 51a3999ae3..37cdb344b5 100644 --- a/headers/private/package/hpkg/PackageReaderImpl.h +++ b/headers/private/package/hpkg/PackageReaderImpl.h @@ -22,6 +22,7 @@ class BPackageEntryAttribute; namespace BPrivate { +struct hpkg_header; class PackageWriterImpl; @@ -34,7 +35,7 @@ public: status_t Init(const char* fileName, uint32 flags); status_t Init(int fd, bool keepFD, uint32 flags); status_t Init(BPositionIO* file, bool keepFile, - uint32 flags); + uint32 flags, hpkg_header* _header = NULL); status_t ParseContent( BPackageContentHandler* contentHandler); status_t ParseContent(BLowLevelPackageContentHandler* diff --git a/headers/private/package/hpkg/WriterImplBase.h b/headers/private/package/hpkg/WriterImplBase.h index 2f2e3266b4..c4d975f979 100644 --- a/headers/private/package/hpkg/WriterImplBase.h +++ b/headers/private/package/hpkg/WriterImplBase.h @@ -101,8 +101,11 @@ protected: typedef DoublyLinkedList PackageAttributeList; protected: - status_t Init(const char* fileName, size_t headerSize, + status_t Init(const char* fileName, const BPackageWriterParameters& parameters); + status_t InitHeapReader(size_t headerSize); + + void SetCompression(uint32 compression); void RegisterPackageInfo( PackageAttributeList& attributeList, diff --git a/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp b/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp index f8bd2da482..a93663690e 100644 --- a/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp +++ b/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp @@ -43,6 +43,45 @@ PackageFileHeapAccessorBase::OffsetArray::~OffsetArray() } +bool +PackageFileHeapAccessorBase::OffsetArray::InitUncompressedChunksOffsets( + size_t totalChunkCount) +{ + if (totalChunkCount <= 1) + return true; + + const size_t max32BitChunks = (uint64(1) << 32) / kChunkSize; + size_t actual32BitChunks = totalChunkCount; + if (totalChunkCount - 1 > max32BitChunks) { + actual32BitChunks = max32BitChunks; + fOffsets = _AllocateOffsetArray(totalChunkCount, max32BitChunks); + } else + fOffsets = _AllocateOffsetArray(totalChunkCount, 0); + + if (fOffsets == NULL) + return false; + + { + uint32 offset = kChunkSize; + for (size_t i = 1; i < actual32BitChunks; i++, offset += kChunkSize) + fOffsets[i] = offset; + + } + + if (actual32BitChunks < totalChunkCount) { + uint64 offset = actual32BitChunks * kChunkSize; + uint32* offsets = fOffsets + actual32BitChunks; + for (size_t i = actual32BitChunks; i < totalChunkCount; + i++, offset += kChunkSize) { + *offsets++ = (uint32)offset; + *offsets++ = uint32(offset >> 32); + } + } + + return true; +} + + bool PackageFileHeapAccessorBase::OffsetArray::InitChunksOffsets( size_t totalChunkCount, size_t baseIndex, const uint16* chunkSizes, @@ -52,12 +91,9 @@ PackageFileHeapAccessorBase::OffsetArray::InitChunksOffsets( return true; if (fOffsets == NULL) { - fOffsets = new(std::nothrow) uint32[totalChunkCount]; + fOffsets = _AllocateOffsetArray(totalChunkCount, totalChunkCount); if (fOffsets == NULL) return false; - fOffsets[0] = 0; - // Value that serves as a marker that all offsets are 32 bit. We'll - // replace it, when necessary. } uint64 offset = (*this)[baseIndex]; @@ -74,14 +110,13 @@ PackageFileHeapAccessorBase::OffsetArray::InitChunksOffsets( } else { if (fOffsets[0] == 0) { // Not scaled to allow for 64 bit offsets yet. Do that. - fOffsets[0] = index; - uint32* newOffsets = new(std::nothrow) uint32[ - 2 * totalChunkCount - fOffsets[0]]; + uint32* newOffsets = _AllocateOffsetArray(totalChunkCount, + index); if (newOffsets == NULL) return false; - memcpy(newOffsets, fOffsets, - sizeof(newOffsets[0]) * fOffsets[0]); + fOffsets[0] = index; + memcpy(newOffsets, fOffsets, sizeof(newOffsets[0]) * index); delete[] fOffsets; fOffsets = newOffsets; @@ -117,6 +152,24 @@ PackageFileHeapAccessorBase::OffsetArray::Init(size_t totalChunkCount, } +/*static*/ uint32* +PackageFileHeapAccessorBase::OffsetArray::_AllocateOffsetArray( + size_t totalChunkCount, size_t offset32BitChunkCount) +{ + uint32* offsets = new(std::nothrow) uint32[ + 2 * totalChunkCount - offset32BitChunkCount]; + + if (offsets != NULL) { + offsets[0] = offset32BitChunkCount == totalChunkCount + ? 0 : offset32BitChunkCount; + // 0 means that all offsets are 32 bit. Otherwise it's the index of + // the first 64 bit offset. + } + + return offsets; +} + + // #pragma mark - PackageFileHeapAccessorBase @@ -143,16 +196,6 @@ PackageFileHeapAccessorBase::~PackageFileHeapAccessorBase() } -uint64 -PackageFileHeapAccessorBase::HeapOverhead(uint64 uncompressedSize) const -{ - // Determine number of chunks and the size of the chunk size table. Note - // that the size of the last chunk is not saved, since its size is implied. - size_t chunkCount = (uncompressedSize + kChunkSize - 1) / kChunkSize; - return chunkCount > 1 ? (chunkCount - 1) * 2 : 0; -} - - status_t PackageFileHeapAccessorBase::ReadDataToOutput(off_t offset, size_t size, BDataIO* output) diff --git a/src/kits/package/hpkg/PackageFileHeapReader.cpp b/src/kits/package/hpkg/PackageFileHeapReader.cpp index 9af47d08db..fb1449b63f 100644 --- a/src/kits/package/hpkg/PackageFileHeapReader.cpp +++ b/src/kits/package/hpkg/PackageFileHeapReader.cpp @@ -61,6 +61,23 @@ PackageFileHeapReader::Init() if (chunkCount == 0) return B_OK; + // If no compression is used at all, the chunk size table is omitted. Handle + // this case. + if (fDecompressionAlgorithm == NULL) { + if (fUncompressedHeapSize != fCompressedHeapSize) { + fErrorOutput->PrintError( + "Compressed and uncompressed heap sizes (%" B_PRIu64 " vs. " + "%" B_PRIu64 ") don't match for uncompressed heap.\n", + fCompressedHeapSize, fUncompressedHeapSize); + return B_BAD_DATA; + } + + if (!fOffsets.InitUncompressedChunksOffsets(chunkCount)) + return B_NO_MEMORY; + + return B_OK; + } + size_t chunkSizeTableSize = (chunkCount - 1) * 2; if (fCompressedHeapSize <= chunkSizeTableSize) { fErrorOutput->PrintError( diff --git a/src/kits/package/hpkg/PackageFileHeapWriter.cpp b/src/kits/package/hpkg/PackageFileHeapWriter.cpp index 1a17ef1647..3a5800ccd8 100644 --- a/src/kits/package/hpkg/PackageFileHeapWriter.cpp +++ b/src/kits/package/hpkg/PackageFileHeapWriter.cpp @@ -443,6 +443,10 @@ PackageFileHeapWriter::Finish() // write chunk sizes table + // We don't need to do that, if we don't use any compression. + if (fCompressionAlgorithm == NULL) + return B_OK; + // We don't need to write the last chunk size, since it is implied by the // total size minus the sum of all other chunk sizes. ssize_t offsetCount = fOffsets.Count(); diff --git a/src/kits/package/hpkg/PackageReaderImpl.cpp b/src/kits/package/hpkg/PackageReaderImpl.cpp index 4bc00739dc..1ff2bf2d29 100644 --- a/src/kits/package/hpkg/PackageReaderImpl.cpp +++ b/src/kits/package/hpkg/PackageReaderImpl.cpp @@ -346,7 +346,8 @@ PackageReaderImpl::Init(int fd, bool keepFD, uint32 flags) status_t -PackageReaderImpl::Init(BPositionIO* file, bool keepFile, uint32 flags) +PackageReaderImpl::Init(BPositionIO* file, bool keepFile, uint32 flags, + hpkg_header* _header) { hpkg_header header; status_t error = inherited::InitReinit(packageReader.RawHeapReader()); // Remove the old packages attributes and TOC section from the heap. @@ -673,6 +681,10 @@ PackageWriterImpl::_Init(const char* fileName, tocSection.uncompressedLength)) { throw std::bad_alloc(); } + } else { + result = InitHeapReader(fHeapOffset); + if (result != B_OK) + return result; } return B_OK; @@ -706,7 +718,8 @@ PackageWriterImpl::_Finish() uint64 compressedHeapSize = fHeapWriter->CompressedHeapSize(); - header.heap_compression = B_HOST_TO_BENDIAN_INT16(B_HPKG_COMPRESSION_ZLIB); + header.heap_compression = B_HOST_TO_BENDIAN_INT16( + Parameters().Compression()); header.heap_chunk_size = B_HOST_TO_BENDIAN_INT32(fHeapWriter->ChunkSize()); header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(compressedHeapSize); header.heap_size_uncompressed = B_HOST_TO_BENDIAN_INT64( @@ -763,20 +776,24 @@ PackageWriterImpl::_Recompress(PackageReaderImpl* reader) // for streaming an uncompressed package. uint64 uncompressedHeapSize = reader->RawHeapReader()->UncompressedHeapSize(); - uint64 compressedHeapSize = uncompressedHeapSize - + fHeapWriter->HeapOverhead(uncompressedHeapSize); + uint64 compressedHeapSize = uncompressedHeapSize; off_t totalSize = fHeapWriter->HeapOffset() + (off_t)compressedHeapSize; - header.heap_compression = B_HOST_TO_BENDIAN_INT16(B_HPKG_COMPRESSION_ZLIB); + header.heap_compression = B_HOST_TO_BENDIAN_INT16( + Parameters().Compression()); header.heap_chunk_size = B_HOST_TO_BENDIAN_INT32(fHeapWriter->ChunkSize()); - header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(compressedHeapSize); header.heap_size_uncompressed = B_HOST_TO_BENDIAN_INT64(uncompressedHeapSize); - header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize); - if (Parameters().CompressionLevel() == 0) + if (Parameters().Compression() == B_HPKG_COMPRESSION_NONE) { + header.heap_size_compressed + = B_HOST_TO_BENDIAN_INT64(compressedHeapSize); + header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize); + + // write the header RawWriteBuffer(&header, sizeof(hpkg_header), 0); + } // copy the heap data uint64 bytesCompressed; @@ -791,7 +808,7 @@ PackageWriterImpl::_Recompress(PackageReaderImpl* reader) return error; // If compression is enabled, update and write the header. - if (Parameters().CompressionLevel() != 0) { + if (Parameters().Compression() != B_HPKG_COMPRESSION_NONE) { compressedHeapSize = fHeapWriter->CompressedHeapSize(); totalSize = fHeapWriter->HeapOffset() + (off_t)compressedHeapSize; header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(compressedHeapSize); diff --git a/src/kits/package/hpkg/ReaderImplBase.cpp b/src/kits/package/hpkg/ReaderImplBase.cpp index eb301f7602..bb3bb1b05e 100644 --- a/src/kits/package/hpkg/ReaderImplBase.cpp +++ b/src/kits/package/hpkg/ReaderImplBase.cpp @@ -807,22 +807,26 @@ status_t ReaderImplBase::InitHeapReader(uint32 compression, uint32 chunkSize, off_t offset, uint64 compressedSize, uint64 uncompressedSize) { - if (compression != B_HPKG_COMPRESSION_ZLIB) { - fErrorOutput->PrintError("Error: Invalid heap compression\n"); - return B_BAD_DATA; - } + DecompressionAlgorithmOwner* decompressionAlgorithm = NULL; + BReference decompressionAlgorithmReference; - DecompressionAlgorithmOwner* decompressionAlgorithm - = DecompressionAlgorithmOwner::Create( - new(std::nothrow) BZlibCompressionAlgorithm, - new(std::nothrow) BZlibDecompressionParameters); - BReference decompressionAlgorithmReference( - decompressionAlgorithm, true); - - if (decompressionAlgorithm == NULL - || decompressionAlgorithm->algorithm == NULL - || decompressionAlgorithm->parameters == NULL) { - return B_NO_MEMORY; + switch (compression) { + case B_HPKG_COMPRESSION_NONE: + break; + case B_HPKG_COMPRESSION_ZLIB: + decompressionAlgorithm = DecompressionAlgorithmOwner::Create( + new(std::nothrow) BZlibCompressionAlgorithm, + new(std::nothrow) BZlibDecompressionParameters); + decompressionAlgorithmReference.SetTo(decompressionAlgorithm, true); + if (decompressionAlgorithm == NULL + || decompressionAlgorithm->algorithm == NULL + || decompressionAlgorithm->parameters == NULL) { + return B_NO_MEMORY; + } + break; + default: + fErrorOutput->PrintError("Error: Invalid heap compression\n"); + return B_BAD_DATA; } fRawHeapReader = new(std::nothrow) PackageFileHeapReader(fErrorOutput, diff --git a/src/kits/package/hpkg/RepositoryWriterImpl.cpp b/src/kits/package/hpkg/RepositoryWriterImpl.cpp index d5be01ce5b..cda6ff04d7 100644 --- a/src/kits/package/hpkg/RepositoryWriterImpl.cpp +++ b/src/kits/package/hpkg/RepositoryWriterImpl.cpp @@ -264,8 +264,11 @@ RepositoryWriterImpl::Finish() status_t RepositoryWriterImpl::_Init(const char* fileName) { - return inherited::Init(fileName, sizeof(hpkg_repo_header), - BPackageWriterParameters()); + status_t error = inherited::Init(fileName, BPackageWriterParameters()); + if (error != B_OK) + return error; + + return InitHeapReader(sizeof(hpkg_repo_header)); } @@ -291,7 +294,8 @@ RepositoryWriterImpl::_Finish() uint64 compressedHeapSize = fHeapWriter->CompressedHeapSize(); uint64 totalSize = fHeapWriter->HeapOffset() + compressedHeapSize; - header.heap_compression = B_HOST_TO_BENDIAN_INT16(B_HPKG_COMPRESSION_ZLIB); + header.heap_compression = B_HOST_TO_BENDIAN_INT16( + Parameters().Compression()); header.heap_chunk_size = B_HOST_TO_BENDIAN_INT32(fHeapWriter->ChunkSize()); header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(compressedHeapSize); header.heap_size_uncompressed = B_HOST_TO_BENDIAN_INT64( diff --git a/src/kits/package/hpkg/WriterImplBase.cpp b/src/kits/package/hpkg/WriterImplBase.cpp index 51fdabb4a4..65d3016c73 100644 --- a/src/kits/package/hpkg/WriterImplBase.cpp +++ b/src/kits/package/hpkg/WriterImplBase.cpp @@ -251,7 +251,7 @@ WriterImplBase::~WriterImplBase() status_t -WriterImplBase::Init(const char* fileName, size_t headerSize, +WriterImplBase::Init(const char* fileName, const BPackageWriterParameters& parameters) { fParameters = parameters; @@ -276,37 +276,49 @@ WriterImplBase::Init(const char* fileName, size_t headerSize, fFile = file; fFileName = fileName; - DecompressionAlgorithmOwner* decompressionAlgorithm - = DecompressionAlgorithmOwner::Create( - new(std::nothrow) BZlibCompressionAlgorithm, - new(std::nothrow) BZlibDecompressionParameters); - BReference decompressionAlgorithmReference( - decompressionAlgorithm, true); + return B_OK; +} - if (decompressionAlgorithm == NULL - || decompressionAlgorithm->algorithm == NULL - || decompressionAlgorithm->parameters == NULL) { - throw std::bad_alloc(); - } +status_t +WriterImplBase::InitHeapReader(size_t headerSize) +{ + // allocate the compression/decompression algorithm CompressionAlgorithmOwner* compressionAlgorithm = NULL; + BReference compressionAlgorithmReference; - if (fParameters.CompressionLevel() != B_HPKG_COMPRESSION_LEVEL_NONE) { - compressionAlgorithm = CompressionAlgorithmOwner::Create( - new(std::nothrow) BZlibCompressionAlgorithm, - new(std::nothrow) BZlibCompressionParameters( - fParameters.CompressionLevel())); + DecompressionAlgorithmOwner* decompressionAlgorithm = NULL; + BReference decompressionAlgorithmReference; - if (compressionAlgorithm == NULL - || compressionAlgorithm->algorithm == NULL - || compressionAlgorithm->parameters == NULL) { - throw std::bad_alloc(); - } + switch (fParameters.Compression()) { + case B_HPKG_COMPRESSION_NONE: + break; + case B_HPKG_COMPRESSION_ZLIB: + compressionAlgorithm = CompressionAlgorithmOwner::Create( + new(std::nothrow) BZlibCompressionAlgorithm, + new(std::nothrow) BZlibCompressionParameters( + fParameters.CompressionLevel())); + compressionAlgorithmReference.SetTo(compressionAlgorithm, true); + + decompressionAlgorithm = DecompressionAlgorithmOwner::Create( + new(std::nothrow) BZlibCompressionAlgorithm, + new(std::nothrow) BZlibDecompressionParameters); + decompressionAlgorithmReference.SetTo(decompressionAlgorithm, true); + + if (compressionAlgorithm == NULL + || compressionAlgorithm->algorithm == NULL + || compressionAlgorithm->parameters == NULL + || decompressionAlgorithm == NULL + || decompressionAlgorithm->algorithm == NULL + || decompressionAlgorithm->parameters == NULL) { + throw std::bad_alloc(); + } + break; + default: + fErrorOutput->PrintError("Error: Invalid heap compression\n"); + return B_BAD_VALUE; } - BReference compressionAlgorithmReference( - compressionAlgorithm, true); - // create heap writer fHeapWriter = new PackageFileHeapWriter(fErrorOutput, fFile, headerSize, compressionAlgorithm, decompressionAlgorithm); @@ -316,6 +328,13 @@ WriterImplBase::Init(const char* fileName, size_t headerSize, } +void +WriterImplBase::SetCompression(uint32 compression) +{ + fParameters.SetCompression(compression); +} + + void WriterImplBase::RegisterPackageInfo(PackageAttributeList& attributeList, const BPackageInfo& packageInfo)