From 1f633814fa347b0bd822f683af731615df600347 Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Sat, 18 May 2013 04:11:01 +0200 Subject: [PATCH] hpkg format: compress the whole heap Instead of handling compression for individual file/attribute data we do now compress the whole heap where they are stored. This significantly improves compression ratios. We still divide the uncompressed data into 64 KiB chunks and use a chunk offset array for the compressed chunks to allow for quick random access without too much overhead. The tradeoff is a limited possible compression ratio -- i.e. we won't be as good as tar.gz (though surprisingly with my test archives we did better than zip). The other package file sections (package attributes and TOC) are no longer compressed individually. Their uncompressed data are simply pushed onto the heap where the usual compression strategy applies. To simplify things the repository format has been changed in the same manner although it doesn't otherwise use the heap, since it only stores meta data. Due to the data compression having been exposed in public and private API, this change touches a lot of package kit using code, including packagefs and the boot loader packagefs support. The latter two haven't been tested yet. Moreover packagefs needs a new kind of cache so we avoid re-reading the same heap chunk for two different data items it contains. --- .../build/private/package/hpkg/DataWriters.h | 1 + .../hpkg/PackageFileHeapAccessorBase.h | 1 + .../package/hpkg/PackageFileHeapReader.h | 1 + .../package/hpkg/PackageFileHeapWriter.h | 1 + headers/os/package/hpkg/HPKGDefs.h | 65 +- headers/os/package/hpkg/PackageData.h | 22 +- headers/os/package/hpkg/PackageDataReader.h | 9 +- headers/os/package/hpkg/PackageReader.h | 9 +- headers/os/package/hpkg/RepositoryReader.h | 1 + headers/private/package/hpkg/DataWriters.h | 101 +++ .../private/package/hpkg/HPKGDefsPrivate.h | 31 +- .../hpkg/PackageFileHeapAccessorBase.h | 143 ++++ .../package/hpkg/PackageFileHeapReader.h | 57 ++ .../package/hpkg/PackageFileHeapWriter.h | 99 +++ .../private/package/hpkg/PackageReaderImpl.h | 10 +- .../private/package/hpkg/PackageWriterImpl.h | 31 +- headers/private/package/hpkg/ReaderImplBase.h | 457 ++++++++----- .../package/hpkg/RepositoryReaderImpl.h | 4 +- .../package/hpkg/RepositoryWriterImpl.h | 7 +- headers/private/package/hpkg/WriterImplBase.h | 127 +--- .../kernel/file_systems/packagefs/Jamfile | 2 + .../nodes/UnpackingAttributeCookie.cpp | 29 +- .../packagefs/package/GlobalFactory.cpp | 23 +- .../packagefs/package/GlobalFactory.h | 14 +- .../packagefs/package/Package.cpp | 190 +++++- .../file_systems/packagefs/package/Package.h | 9 + .../packagefs/package/PackageData.h | 4 +- .../packagefs/package/PackageFile.cpp | 26 +- src/bin/package/command_dump.cpp | 2 +- src/bin/package/command_extract.cpp | 157 +++-- src/bin/package/command_list.cpp | 22 +- src/build/libpackage/Jamfile | 4 + src/kits/package/Jamfile | 3 + src/kits/package/hpkg/DataWriters.cpp | 125 ++++ .../package/hpkg/PackageContentHandler.cpp | 3 - src/kits/package/hpkg/PackageData.cpp | 9 +- src/kits/package/hpkg/PackageDataReader.cpp | 373 +--------- .../hpkg/PackageFileHeapAccessorBase.cpp | 247 +++++++ .../package/hpkg/PackageFileHeapReader.cpp | 131 ++++ .../package/hpkg/PackageFileHeapWriter.cpp | 645 ++++++++++++++++++ src/kits/package/hpkg/PackageReader.cpp | 12 +- src/kits/package/hpkg/PackageReaderImpl.cpp | 285 +------- src/kits/package/hpkg/PackageWriterImpl.cpp | 613 +++-------------- src/kits/package/hpkg/ReaderImplBase.cpp | 169 ++--- .../package/hpkg/RepositoryReaderImpl.cpp | 174 +---- .../package/hpkg/RepositoryWriterImpl.cpp | 143 ++-- src/kits/package/hpkg/WriterImplBase.cpp | 132 +--- .../package/hpkg/v1/PackageDataReaderV1.cpp | 27 + .../loader/file_systems/packagefs/Jamfile | 2 + .../file_systems/packagefs/packagefs.cpp | 112 +-- 50 files changed, 2732 insertions(+), 2132 deletions(-) create mode 100644 headers/build/private/package/hpkg/DataWriters.h create mode 100644 headers/build/private/package/hpkg/PackageFileHeapAccessorBase.h create mode 100644 headers/build/private/package/hpkg/PackageFileHeapReader.h create mode 100644 headers/build/private/package/hpkg/PackageFileHeapWriter.h create mode 100644 headers/private/package/hpkg/DataWriters.h create mode 100644 headers/private/package/hpkg/PackageFileHeapAccessorBase.h create mode 100644 headers/private/package/hpkg/PackageFileHeapReader.h create mode 100644 headers/private/package/hpkg/PackageFileHeapWriter.h create mode 100644 src/kits/package/hpkg/DataWriters.cpp create mode 100644 src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp create mode 100644 src/kits/package/hpkg/PackageFileHeapReader.cpp create mode 100644 src/kits/package/hpkg/PackageFileHeapWriter.cpp diff --git a/headers/build/private/package/hpkg/DataWriters.h b/headers/build/private/package/hpkg/DataWriters.h new file mode 100644 index 0000000000..1f9f8962ca --- /dev/null +++ b/headers/build/private/package/hpkg/DataWriters.h @@ -0,0 +1 @@ +#include <../private/package/hpkg/DataWriters.h> diff --git a/headers/build/private/package/hpkg/PackageFileHeapAccessorBase.h b/headers/build/private/package/hpkg/PackageFileHeapAccessorBase.h new file mode 100644 index 0000000000..f9c224e97f --- /dev/null +++ b/headers/build/private/package/hpkg/PackageFileHeapAccessorBase.h @@ -0,0 +1 @@ +#include <../private/package/hpkg/PackageFileHeapAccessorBase.h> diff --git a/headers/build/private/package/hpkg/PackageFileHeapReader.h b/headers/build/private/package/hpkg/PackageFileHeapReader.h new file mode 100644 index 0000000000..e051dd2d6f --- /dev/null +++ b/headers/build/private/package/hpkg/PackageFileHeapReader.h @@ -0,0 +1 @@ +#include <../private/package/hpkg/PackageFileHeapReader.h> diff --git a/headers/build/private/package/hpkg/PackageFileHeapWriter.h b/headers/build/private/package/hpkg/PackageFileHeapWriter.h new file mode 100644 index 0000000000..88fd3c8057 --- /dev/null +++ b/headers/build/private/package/hpkg/PackageFileHeapWriter.h @@ -0,0 +1 @@ +#include <../private/package/hpkg/PackageFileHeapWriter.h> diff --git a/headers/os/package/hpkg/HPKGDefs.h b/headers/os/package/hpkg/HPKGDefs.h index d4f90156c1..7cd931b52a 100644 --- a/headers/os/package/hpkg/HPKGDefs.h +++ b/headers/os/package/hpkg/HPKGDefs.h @@ -20,7 +20,7 @@ enum { B_HPKG_VERSION = 2, // B_HPKG_REPO_MAGIC = 'hpkr', - B_HPKG_REPO_VERSION = 1 + B_HPKG_REPO_VERSION = 2 }; @@ -98,37 +98,34 @@ enum BHPKGAttributeID { B_HPKG_ATTRIBUTE_ID_FILE_ATTRIBUTE = 11, B_HPKG_ATTRIBUTE_ID_FILE_ATTRIBUTE_TYPE = 12, B_HPKG_ATTRIBUTE_ID_DATA = 13, - B_HPKG_ATTRIBUTE_ID_DATA_SIZE = 14, - B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION = 15, - B_HPKG_ATTRIBUTE_ID_DATA_CHUNK_SIZE = 16, - B_HPKG_ATTRIBUTE_ID_SYMLINK_PATH = 17, - B_HPKG_ATTRIBUTE_ID_PACKAGE_NAME = 18, - B_HPKG_ATTRIBUTE_ID_PACKAGE_SUMMARY = 19, - B_HPKG_ATTRIBUTE_ID_PACKAGE_DESCRIPTION = 20, - B_HPKG_ATTRIBUTE_ID_PACKAGE_VENDOR = 21, - B_HPKG_ATTRIBUTE_ID_PACKAGE_PACKAGER = 22, - B_HPKG_ATTRIBUTE_ID_PACKAGE_FLAGS = 23, - B_HPKG_ATTRIBUTE_ID_PACKAGE_ARCHITECTURE = 24, - B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MAJOR = 25, - B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MINOR = 26, - B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MICRO = 27, - B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_REVISION = 28, - B_HPKG_ATTRIBUTE_ID_PACKAGE_COPYRIGHT = 29, - B_HPKG_ATTRIBUTE_ID_PACKAGE_LICENSE = 30, - B_HPKG_ATTRIBUTE_ID_PACKAGE_PROVIDES = 31, - B_HPKG_ATTRIBUTE_ID_PACKAGE_REQUIRES = 32, - B_HPKG_ATTRIBUTE_ID_PACKAGE_SUPPLEMENTS = 33, - B_HPKG_ATTRIBUTE_ID_PACKAGE_CONFLICTS = 34, - B_HPKG_ATTRIBUTE_ID_PACKAGE_FRESHENS = 35, - B_HPKG_ATTRIBUTE_ID_PACKAGE_REPLACES = 36, - B_HPKG_ATTRIBUTE_ID_PACKAGE_RESOLVABLE_OPERATOR = 37, - B_HPKG_ATTRIBUTE_ID_PACKAGE_CHECKSUM = 38, - B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_PRE_RELEASE = 39, - B_HPKG_ATTRIBUTE_ID_PACKAGE_PROVIDES_COMPATIBLE = 40, - B_HPKG_ATTRIBUTE_ID_PACKAGE_URL = 41, - B_HPKG_ATTRIBUTE_ID_PACKAGE_SOURCE_URL = 42, - B_HPKG_ATTRIBUTE_ID_PACKAGE_INSTALL_PATH = 43, - B_HPKG_ATTRIBUTE_ID_PACKAGE_BASE_PACKAGE = 44, + B_HPKG_ATTRIBUTE_ID_SYMLINK_PATH = 14, + B_HPKG_ATTRIBUTE_ID_PACKAGE_NAME = 15, + B_HPKG_ATTRIBUTE_ID_PACKAGE_SUMMARY = 16, + B_HPKG_ATTRIBUTE_ID_PACKAGE_DESCRIPTION = 17, + B_HPKG_ATTRIBUTE_ID_PACKAGE_VENDOR = 18, + B_HPKG_ATTRIBUTE_ID_PACKAGE_PACKAGER = 19, + B_HPKG_ATTRIBUTE_ID_PACKAGE_FLAGS = 20, + B_HPKG_ATTRIBUTE_ID_PACKAGE_ARCHITECTURE = 21, + B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MAJOR = 22, + B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MINOR = 23, + B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MICRO = 24, + B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_REVISION = 25, + B_HPKG_ATTRIBUTE_ID_PACKAGE_COPYRIGHT = 26, + B_HPKG_ATTRIBUTE_ID_PACKAGE_LICENSE = 27, + B_HPKG_ATTRIBUTE_ID_PACKAGE_PROVIDES = 28, + B_HPKG_ATTRIBUTE_ID_PACKAGE_REQUIRES = 29, + B_HPKG_ATTRIBUTE_ID_PACKAGE_SUPPLEMENTS = 30, + B_HPKG_ATTRIBUTE_ID_PACKAGE_CONFLICTS = 31, + B_HPKG_ATTRIBUTE_ID_PACKAGE_FRESHENS = 32, + B_HPKG_ATTRIBUTE_ID_PACKAGE_REPLACES = 33, + B_HPKG_ATTRIBUTE_ID_PACKAGE_RESOLVABLE_OPERATOR = 34, + B_HPKG_ATTRIBUTE_ID_PACKAGE_CHECKSUM = 35, + B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_PRE_RELEASE = 36, + B_HPKG_ATTRIBUTE_ID_PACKAGE_PROVIDES_COMPATIBLE = 37, + B_HPKG_ATTRIBUTE_ID_PACKAGE_URL = 38, + B_HPKG_ATTRIBUTE_ID_PACKAGE_SOURCE_URL = 39, + B_HPKG_ATTRIBUTE_ID_PACKAGE_INSTALL_PATH = 40, + B_HPKG_ATTRIBUTE_ID_PACKAGE_BASE_PACKAGE = 41, // B_HPKG_ATTRIBUTE_ID_ENUM_COUNT, }; @@ -154,9 +151,7 @@ enum { B_HPKG_DEFAULT_FILE_TYPE = B_HPKG_FILE_TYPE_FILE, B_HPKG_DEFAULT_FILE_PERMISSIONS = 0644, B_HPKG_DEFAULT_DIRECTORY_PERMISSIONS = 0755, - B_HPKG_DEFAULT_SYMLINK_PERMISSIONS = 0777, - B_HPKG_DEFAULT_DATA_COMPRESSION = B_HPKG_COMPRESSION_NONE, - B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB = 64 * 1024 + B_HPKG_DEFAULT_SYMLINK_PERMISSIONS = 0777 }; diff --git a/headers/os/package/hpkg/PackageData.h b/headers/os/package/hpkg/PackageData.h index d0ef1c0799..fa0fa4811a 100644 --- a/headers/os/package/hpkg/PackageData.h +++ b/headers/os/package/hpkg/PackageData.h @@ -18,14 +18,10 @@ class BPackageData { public: BPackageData(); - uint64 CompressedSize() const - { return fCompressedSize; } - uint64 UncompressedSize() const - { return fUncompressedSize; } + uint64 Size() const + { return fSize; } uint64 Offset() const { return fEncodedInline ? 0 : fOffset; } - uint32 Compression() const { return fCompression; } - uint32 ChunkSize() const { return fChunkSize; } bool IsEncodedInline() const { return fEncodedInline; } @@ -34,23 +30,13 @@ public: void SetData(uint64 size, uint64 offset); void SetData(uint8 size, const void* data); - void SetCompression(uint32 compression) - { fCompression = compression; } - void SetUncompressedSize(uint64 size) - { fUncompressedSize = size; } - void SetChunkSize(uint32 size) - { fChunkSize = size; } - private: - uint64 fCompressedSize; - uint64 fUncompressedSize; + uint64 fSize : 63; + bool fEncodedInline : 1; union { uint64 fOffset; uint8 fInlineData[B_HPKG_MAX_INLINE_DATA_SIZE]; }; - uint32 fChunkSize; - uint32 fCompression; - bool fEncodedInline; }; diff --git a/headers/os/package/hpkg/PackageDataReader.h b/headers/os/package/hpkg/PackageDataReader.h index 1152fbaf49..0dcebf9397 100644 --- a/headers/os/package/hpkg/PackageDataReader.h +++ b/headers/os/package/hpkg/PackageDataReader.h @@ -20,15 +20,12 @@ class BPackageData; class BPackageDataReaderFactory { public: - BPackageDataReaderFactory( - BBufferPool* bufferPool); + BPackageDataReaderFactory(); - status_t CreatePackageDataReader(BDataReader* dataReader, + status_t CreatePackageDataReader( + BAbstractBufferedDataReader* dataReader, const BPackageData& data, BAbstractBufferedDataReader*& _reader); - -private: - BBufferPool* fBufferPool; }; diff --git a/headers/os/package/hpkg/PackageReader.h b/headers/os/package/hpkg/PackageReader.h index e559c35e10..3828a4b410 100644 --- a/headers/os/package/hpkg/PackageReader.h +++ b/headers/os/package/hpkg/PackageReader.h @@ -19,6 +19,8 @@ namespace BPrivate { } using BPrivate::PackageReaderImpl; + +class BAbstractBufferedDataReader; class BErrorOutput; class BLowLevelPackageContentHandler; class BPackageContentHandler; @@ -26,8 +28,7 @@ class BPackageContentHandler; class BPackageReader { public: - BPackageReader( - BErrorOutput* errorOutput); + BPackageReader(BErrorOutput* errorOutput); ~BPackageReader(); status_t Init(const char* fileName); @@ -38,6 +39,10 @@ public: contentHandler); int PackageFileFD(); + + BAbstractBufferedDataReader* HeapReader() const; + // Only valid as long as the reader lives. + private: PackageReaderImpl* fImpl; }; diff --git a/headers/os/package/hpkg/RepositoryReader.h b/headers/os/package/hpkg/RepositoryReader.h index f42c077d7d..d7e3539bbd 100644 --- a/headers/os/package/hpkg/RepositoryReader.h +++ b/headers/os/package/hpkg/RepositoryReader.h @@ -19,6 +19,7 @@ namespace BPrivate { } using BPrivate::RepositoryReaderImpl; + class BErrorOutput; class BRepositoryContentHandler; diff --git a/headers/private/package/hpkg/DataWriters.h b/headers/private/package/hpkg/DataWriters.h new file mode 100644 index 0000000000..26f67cf411 --- /dev/null +++ b/headers/private/package/hpkg/DataWriters.h @@ -0,0 +1,101 @@ +/* + * Copyright 2011, Oliver Tappe + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef _PACKAGE__HPKG__PRIVATE__DATA_WRITERS_H_ +#define _PACKAGE__HPKG__PRIVATE__DATA_WRITERS_H_ + + +#include +#include + + +namespace BPackageKit { + +namespace BHPKG { + + +class BErrorOutput; + + +namespace BPrivate { + + +class AbstractDataWriter { +public: + AbstractDataWriter(); + virtual ~AbstractDataWriter(); + + uint64 BytesWritten() const + { return fBytesWritten; } + + virtual status_t WriteDataNoThrow(const void* buffer, + size_t size) = 0; + + void WriteDataThrows(const void* buffer, + size_t size); + +protected: + uint64 fBytesWritten; +}; + + +class FDDataWriter : public AbstractDataWriter { +public: + FDDataWriter(int fd, off_t offset, + BErrorOutput* errorOutput); + + virtual status_t WriteDataNoThrow(const void* buffer, + size_t size); + + off_t Offset() const + { return fOffset; } + +private: + int fFD; + off_t fOffset; + BErrorOutput* fErrorOutput; +}; + + +class ZlibDataWriter : public AbstractDataWriter, private BDataOutput { +public: + ZlibDataWriter(AbstractDataWriter* dataWriter); + + void Init(); + void Finish(); + + virtual status_t WriteDataNoThrow(const void* buffer, + size_t size); + +private: + // BDataOutput + virtual status_t WriteData(const void* buffer, size_t size); + +private: + AbstractDataWriter* fDataWriter; + ZlibCompressor fCompressor; +}; + + +// inline implementations + + +inline void +AbstractDataWriter::WriteDataThrows(const void* buffer, size_t size) +{ + status_t error = WriteDataNoThrow(buffer, size); + if (error != B_OK) + throw status_t(error); +} + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit + + +#endif // _PACKAGE__HPKG__PRIVATE__DATA_WRITERS_H_ diff --git a/headers/private/package/hpkg/HPKGDefsPrivate.h b/headers/private/package/hpkg/HPKGDefsPrivate.h index 7b29f00a2d..dd2b82062c 100644 --- a/headers/private/package/hpkg/HPKGDefsPrivate.h +++ b/headers/private/package/hpkg/HPKGDefsPrivate.h @@ -18,7 +18,7 @@ namespace BHPKG { namespace BPrivate { -// header +// package file header struct hpkg_header { uint32 magic; // "hpkg" uint16 header_size; @@ -26,40 +26,41 @@ struct hpkg_header { uint64 total_size; // heap -// uint64 heap_compression; + uint32 heap_compression; + uint32 heap_chunk_size; + uint64 heap_size_compressed; + uint64 heap_size_uncompressed; // package attributes section - uint32 attributes_compression; - uint32 attributes_length_compressed; - uint32 attributes_length_uncompressed; + uint32 attributes_length; uint32 attributes_strings_length; uint32 attributes_strings_count; // TOC section - uint32 toc_compression; - uint64 toc_length_compressed; - uint64 toc_length_uncompressed; + uint64 toc_length; uint64 toc_strings_length; uint64 toc_strings_count; }; -// header +// repository file header struct hpkg_repo_header { uint32 magic; // "hpkr" uint16 header_size; uint16 version; uint64 total_size; + // heap + uint32 heap_compression; + uint32 heap_chunk_size; + uint64 heap_size_compressed; + uint64 heap_size_uncompressed; + // repository info section - uint32 info_compression; - uint32 info_length_compressed; - uint32 info_length_uncompressed; + uint32 info_length; // package attributes section - uint32 packages_compression; - uint64 packages_length_compressed; - uint64 packages_length_uncompressed; + uint64 packages_length; uint64 packages_strings_length; uint64 packages_strings_count; }; diff --git a/headers/private/package/hpkg/PackageFileHeapAccessorBase.h b/headers/private/package/hpkg/PackageFileHeapAccessorBase.h new file mode 100644 index 0000000000..1cf9bd25c2 --- /dev/null +++ b/headers/private/package/hpkg/PackageFileHeapAccessorBase.h @@ -0,0 +1,143 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_ACCESSOR_BASE_H_ +#define _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_ACCESSOR_BASE_H_ + + +#include + +#include + + +namespace BPackageKit { + +namespace BHPKG { + + +class BErrorOutput; + + +namespace BPrivate { + + +class PackageFileHeapAccessorBase : public BAbstractBufferedDataReader { +public: + class OffsetArray; + +public: + PackageFileHeapAccessorBase( + BErrorOutput* errorOutput, int fd, + off_t heapOffset); + virtual ~PackageFileHeapAccessorBase(); + + off_t HeapOffset() const + { return fHeapOffset; } + off_t CompressedHeapSize() const + { return fCompressedHeapSize; } + uint64 UncompressedHeapSize() const + { return fUncompressedHeapSize; } + size_t ChunkSize() const + { return kChunkSize; } + + // normally used after cloning a PackageFileHeapReader only + void SetErrorOutput(BErrorOutput* errorOutput) + { fErrorOutput = errorOutput; } + void SetFD(int fd) + { fFD = fd; } + + // BAbstractBufferedDataReader + virtual status_t ReadDataToOutput(off_t offset, size_t size, + BDataOutput* output); + +public: + static const size_t kChunkSize = 64 * 1024; + +protected: + virtual status_t ReadAndDecompressChunk(size_t chunkIndex, + void* compressedDataBuffer, + void* uncompressedDataBuffer) = 0; + status_t ReadAndDecompressChunkData(uint64 offset, + size_t compressedSize, + size_t uncompressedSize, + void* compressedDataBuffer, + void* uncompressedDataBuffer); + status_t ReadFileData(uint64 offset, void* buffer, + size_t size); + +protected: + BErrorOutput* fErrorOutput; + int fFD; + off_t fHeapOffset; + uint64 fCompressedHeapSize; + uint64 fUncompressedHeapSize; +}; + + +/*! Stores the chunk offsets in a compact way, while still providing quick + access. + - The object doesn't store the number of chunks/offsets it contains. During + initialization the chunk count is provided. Later, when getting an offset, + the caller is responsible for ensuring a valid index. + - The first (index 0) chunk offset is omitted, since it is always 0. + - The chunk offsets that fit in a 32 bit number use only one 32 bit element + in the offsets array. + - The chunk offsets that don't fit in a 32 bit number use two elements in + the offsets array. + Memory use is one pointer, if the chunk count is <= 1 (uncompressed heap size + <= 64 KiB). Afterwards it's one pointer plus 32 bit per chunk as long as the + last offset still fits 32 bit (compressed heap size < 4GiB). For any further + chunks it is 64 bit per chunk. So, for the common case we use sizeof(void*) + plus 1 KiB per 16 MiB of uncompressed heap, or about 64 KiB per 1 GiB. Which + seems reasonable for packagefs to keep in memory. + */ +class PackageFileHeapAccessorBase::OffsetArray { +public: + OffsetArray(); + ~OffsetArray(); + + bool InitChunksOffsets(size_t totalChunkCount, + size_t baseIndex, const uint16* chunkSizes, + size_t chunkCount); + + bool Init(size_t totalChunkCount, + const OffsetArray& other); + // "copy" init + + uint64 operator[](size_t index) const; + +private: + uint32* fOffsets; + // - NULL, if chunkCount <= 1 + // - element 0 contains the number of 32 bit + // offsets that follow, or is 0, when all + // offsets are 32 bit only + // - the following offsets use two elements + // each (lower followed by upper 32 bit) + // to represent the 64 bit value +}; + + +inline uint64 +PackageFileHeapAccessorBase::OffsetArray::operator[](size_t index) const +{ + if (index == 0) + return 0; + + if (fOffsets[0] == 0 || index < fOffsets[0]) + return fOffsets[index]; + + index += index - fOffsets[0]; + return fOffsets[index] | ((uint64)fOffsets[index + 1] << 32); +} + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit + + +#endif // _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_ACCESSOR_BASE_H_ diff --git a/headers/private/package/hpkg/PackageFileHeapReader.h b/headers/private/package/hpkg/PackageFileHeapReader.h new file mode 100644 index 0000000000..de1c36e2ad --- /dev/null +++ b/headers/private/package/hpkg/PackageFileHeapReader.h @@ -0,0 +1,57 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_READER_H_ +#define _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_READER_H_ + + +#include +#include + + +namespace BPackageKit { + +namespace BHPKG { + + +class BDataReader; +class BErrorOutput; + + +namespace BPrivate { + + +class PackageFileHeapReader : public PackageFileHeapAccessorBase { +public: + PackageFileHeapReader(BErrorOutput* errorOutput, + int fd, off_t heapOffset, + off_t compressedHeapSize, + uint64 uncompressedHeapSize); + ~PackageFileHeapReader(); + + status_t Init(); + + PackageFileHeapReader* Clone() const; + + const OffsetArray& Offsets() const + { return fOffsets; } + +protected: + virtual status_t ReadAndDecompressChunk(size_t chunkIndex, + void* compressedDataBuffer, + void* uncompressedDataBuffer); + +private: + OffsetArray fOffsets; +}; + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit + + +#endif // _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_READER_H_ diff --git a/headers/private/package/hpkg/PackageFileHeapWriter.h b/headers/private/package/hpkg/PackageFileHeapWriter.h new file mode 100644 index 0000000000..3b4d24e590 --- /dev/null +++ b/headers/private/package/hpkg/PackageFileHeapWriter.h @@ -0,0 +1,99 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_WRITER_H_ +#define _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_WRITER_H_ + + +#include +#include +#include + + +namespace BPrivate { + template class RangeArray; +} + + +namespace BPackageKit { + +namespace BHPKG { + + +class BDataReader; +class BErrorOutput; + + +namespace BPrivate { + + +class PackageFileHeapReader; + + +class PackageFileHeapWriter : public PackageFileHeapAccessorBase, + private AbstractDataWriter { +public: + PackageFileHeapWriter(BErrorOutput* errorOutput, + int fd, off_t heapOffset); + ~PackageFileHeapWriter(); + + void Init(); + void Reinit(PackageFileHeapReader* heapReader); + + AbstractDataWriter* DataWriter() + { return this; } + + status_t AddData(BDataReader& dataReader, off_t size, + uint64& _offset); + void RemoveDataRanges( + const ::BPrivate::RangeArray& + ranges); + // doesn't truncate the file + status_t Finish(); + +protected: + virtual status_t ReadAndDecompressChunk(size_t chunkIndex, + void* compressedDataBuffer, + void* uncompressedDataBuffer); + +private: + // AbstractDataWriter + virtual status_t WriteDataNoThrow(const void* buffer, + size_t size); + +private: + struct Chunk; + struct ChunkSegment; + struct ChunkBuffer; + +private: + void _Uninit(); + + status_t _FlushPendingData(); + status_t _WriteChunk(const void* data, size_t size, + bool mayCompress); + status_t _WriteDataCompressed(const void* data, + size_t size); + status_t _WriteDataUncompressed(const void* data, + size_t size); + + void _PushChunks(ChunkBuffer& chunkBuffer, + uint64 startOffset, uint64 endOffset); + +private: + void* fPendingDataBuffer; + void* fCompressedDataBuffer; + size_t fPendingDataSize; + Array fOffsets; +}; + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit + + +#endif // _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_WRITER_H_ diff --git a/headers/private/package/hpkg/PackageReaderImpl.h b/headers/private/package/hpkg/PackageReaderImpl.h index b35ac5ec08..7dd0b82cf1 100644 --- a/headers/private/package/hpkg/PackageReaderImpl.h +++ b/headers/private/package/hpkg/PackageReaderImpl.h @@ -25,8 +25,7 @@ namespace BPrivate { class PackageReaderImpl : public ReaderImplBase { typedef ReaderImplBase inherited; public: - PackageReaderImpl( - BErrorOutput* errorOutput); + PackageReaderImpl(BErrorOutput* errorOutput); ~PackageReaderImpl(); status_t Init(const char* fileName); @@ -41,13 +40,15 @@ public: uint64 HeapOffset() const; uint64 HeapSize() const; + PackageFileHeapReader* HeapReader() const + { return inherited::HeapReader(); } + protected: // from ReaderImplBase virtual status_t ReadAttributeValue(uint8 type, uint8 encoding, AttributeValue& _value); private: - struct DataAttributeHandler; struct AttributeAttributeHandler; struct EntryAttributeHandler; struct RootAttributeHandler; @@ -59,11 +60,10 @@ private: status_t _GetTOCBuffer(size_t size, const void*& _buffer); private: - uint64 fTotalSize; uint64 fHeapOffset; uint64 fHeapSize; - SectionInfo fTOCSection; + PackageFileSection fTOCSection; }; diff --git a/headers/private/package/hpkg/PackageWriterImpl.h b/headers/private/package/hpkg/PackageWriterImpl.h index 88febb119b..306f5ccd26 100644 --- a/headers/private/package/hpkg/PackageWriterImpl.h +++ b/headers/private/package/hpkg/PackageWriterImpl.h @@ -35,6 +35,8 @@ namespace BPrivate { struct hpkg_header; +class PackageFileHeapWriter; + class PackageWriterImpl : public WriterImplBase { typedef WriterImplBase inherited; @@ -78,30 +80,16 @@ private: Entry* entry, const char* fileName, char* pathBuffer); void _CompactHeap(); - void _MoveHeapChunk(off_t fromOffset, off_t toOffset, - off_t size); void _AttributeRemoved(Attribute* attribute); - void _WriteTOC(hpkg_header& header); - int32 _WriteTOCCompressed( - uint64& _uncompressedStringsSize, - uint64& _uncompressedMainSize, - uint64& _tocUncompressedSize); - int32 _WriteTOCUncompressed( - uint64& _uncompressedStringsSize, - uint64& _uncompressedMainSize, - uint64& _tocUncompressedSize); - int32 _WriteTOCSections(uint64& _stringsSize, - uint64& _mainSize); + void _WriteTOC(hpkg_header& header, uint64& _length); void _WriteAttributeChildren(Attribute* attribute); - void _WritePackageAttributes(hpkg_header& header); + void _WritePackageAttributes(hpkg_header& header, + uint64& _length); uint32 _WritePackageAttributesCompressed( uint32& _stringsLengthUncompressed, uint32& _attributesLengthUncompressed); - uint32 _WritePackageAttributesUncompressed( - uint32& _stringsLengthUncompressed, - uint32& _attributesLengthUncompressed); void _AddEntry(int dirFD, Entry* entry, const char* fileName, char* pathBuffer); @@ -125,8 +113,6 @@ private: status_t _AddData(BDataReader& dataReader, off_t size); - status_t _WriteUncompressedData(BDataReader& dataReader, - off_t size, uint64 writeOffset); status_t _WriteZlibCompressedData( BDataReader& dataReader, off_t size, uint64 writeOffset, @@ -136,12 +122,9 @@ private: BPackageWriterListener* fListener; off_t fHeapOffset; - off_t fHeapEnd; + uint16 fHeaderSize; - ::BPrivate::RangeArray* fHeapRangesToRemove; - - void* fDataBuffer; - const size_t fDataBufferSize; + ::BPrivate::RangeArray* fHeapRangesToRemove; Entry* fRootEntry; diff --git a/headers/private/package/hpkg/ReaderImplBase.h b/headers/private/package/hpkg/ReaderImplBase.h index 75273e92c2..0ed343eb82 100644 --- a/headers/private/package/hpkg/ReaderImplBase.h +++ b/headers/private/package/hpkg/ReaderImplBase.h @@ -7,6 +7,10 @@ #define _PACKAGE__HPKG__PRIVATE__READER_IMPL_BASE_H_ +#include +#include + +#include #include #include @@ -27,176 +31,73 @@ class BErrorOutput; namespace BPrivate { +class PackageFileHeapReader; + + +struct PackageFileSection { + uint32 uncompressedLength; + uint8* data; + uint64 offset; + uint64 currentOffset; + uint64 stringsLength; + uint64 stringsCount; + char** strings; + const char* name; + + PackageFileSection(const char* _name) + : + data(NULL), + strings(NULL), + name(_name) + { + } + + ~PackageFileSection() + { + delete[] strings; + delete[] data; + } +}; + + class ReaderImplBase { protected: - ReaderImplBase( + ReaderImplBase(const char* fileType, BErrorOutput* errorOutput); virtual ~ReaderImplBase(); - virtual status_t Init(int fd, bool keepFD); - int FD() const; BErrorOutput* ErrorOutput() const; + PackageFileHeapReader* HeapReader() const + { return fHeapReader; } + protected: - struct AttributeHandlerContext { - BErrorOutput* errorOutput; - union { - BPackageContentHandler* packageContentHandler; - BLowLevelPackageContentHandler* lowLevelHandler; - }; - bool hasLowLevelHandler; - - uint64 heapOffset; - uint64 heapSize; - - BHPKGPackageSectionID section; - - AttributeHandlerContext(BErrorOutput* errorOutput, - BPackageContentHandler* packageContentHandler, - BHPKGPackageSectionID section); - - AttributeHandlerContext(BErrorOutput* errorOutput, - BLowLevelPackageContentHandler* lowLevelHandler, - BHPKGPackageSectionID section); - - void ErrorOccurred(); - }; - + class AttributeHandlerContext; + class AttributeHandler; + class IgnoreAttributeHandler; + class PackageVersionAttributeHandler; + class PackageResolvableAttributeHandler; + class PackageResolvableExpressionAttributeHandler; + class PackageAttributeHandler; + class LowLevelAttributeHandler; typedef BPackageAttributeValue AttributeValue; - - struct AttributeHandler - : SinglyLinkedListLinkImpl { - virtual ~AttributeHandler(); - - void SetLevel(int level); - virtual status_t HandleAttribute( - AttributeHandlerContext* context, uint8 id, - const AttributeValue& value, AttributeHandler** _handler); - - virtual status_t Delete(AttributeHandlerContext* context); - - protected: - int fLevel; - }; - - - struct IgnoreAttributeHandler : AttributeHandler { - }; - - - struct PackageVersionAttributeHandler : AttributeHandler { - PackageVersionAttributeHandler( - BPackageInfoAttributeValue& packageInfoValue, - BPackageVersionData& versionData, bool notify); - - virtual status_t HandleAttribute( - AttributeHandlerContext* context, uint8 id, - const AttributeValue& value, AttributeHandler** _handler); - - virtual status_t Delete(AttributeHandlerContext* context); - - private: - BPackageInfoAttributeValue& fPackageInfoValue; - BPackageVersionData& fPackageVersionData; - bool fNotify; - }; - - - struct PackageResolvableAttributeHandler : AttributeHandler { - PackageResolvableAttributeHandler( - BPackageInfoAttributeValue& packageInfoValue); - - virtual status_t HandleAttribute( - AttributeHandlerContext* context, uint8 id, - const AttributeValue& value, AttributeHandler** _handler); - - virtual status_t Delete(AttributeHandlerContext* context); - - private: - BPackageInfoAttributeValue& fPackageInfoValue; - }; - - - struct PackageResolvableExpressionAttributeHandler - : AttributeHandler { - PackageResolvableExpressionAttributeHandler( - BPackageInfoAttributeValue& packageInfoValue); - - virtual status_t HandleAttribute( - AttributeHandlerContext* context, uint8 id, - const AttributeValue& value, AttributeHandler** _handler); - - virtual status_t Delete(AttributeHandlerContext* context); - - private: - BPackageInfoAttributeValue& fPackageInfoValue; - }; - - - struct PackageAttributeHandler : AttributeHandler { - virtual status_t HandleAttribute( - AttributeHandlerContext* context, uint8 id, - const AttributeValue& value, AttributeHandler** _handler); - - private: - BPackageInfoAttributeValue fPackageInfoValue; - }; - - - struct LowLevelAttributeHandler : AttributeHandler { - LowLevelAttributeHandler(); - LowLevelAttributeHandler(uint8 id, - const BPackageAttributeValue& value, void* parentToken, - void* token); - - virtual status_t HandleAttribute( - AttributeHandlerContext* context, uint8 id, - const AttributeValue& value, AttributeHandler** _handler); - virtual status_t Delete(AttributeHandlerContext* context); - - private: - void* fParentToken; - void* fToken; - uint8 fID; - AttributeValue fValue; - }; - - - struct SectionInfo { - uint32 compression; - uint32 compressedLength; - uint32 uncompressedLength; - uint8* data; - uint64 offset; - uint64 currentOffset; - uint64 stringsLength; - uint64 stringsCount; - char** strings; - const char* name; - - SectionInfo(const char* _name) - : - data(NULL), - strings(NULL), - name(_name) - { - } - - ~SectionInfo() - { - delete[] strings; - delete[] data; - } - }; - typedef SinglyLinkedList AttributeHandlerList; protected: - const char* CheckCompression( - const SectionInfo& section) const; + template + status_t Init(int fd, bool keepFD, Header& header); + status_t InitHeapReader(uint32 compression, + uint32 chunkSize, off_t offset, + uint64 compressedSize, + uint64 uncompressedSize); + status_t InitSection(PackageFileSection& section, + uint64 endOffset, uint64 length, + uint64 maxSaneLength, uint64 stringsLength, + uint64 stringsCount); + status_t PrepareSection(PackageFileSection& section); status_t ParseStrings(); @@ -214,8 +115,7 @@ protected: status_t ReadBuffer(off_t offset, void* buffer, size_t size); - status_t ReadCompressedBuffer( - const SectionInfo& section); + status_t ReadSection(const PackageFileSection& section); inline AttributeHandler* CurrentAttributeHandler() const; inline void PushAttributeHandler( @@ -223,13 +123,15 @@ protected: inline AttributeHandler* PopAttributeHandler(); inline void ClearAttributeHandlerStack(); - inline SectionInfo* CurrentSection(); - inline void SetCurrentSection(SectionInfo* section); + inline PackageFileSection* CurrentSection(); + inline void SetCurrentSection(PackageFileSection* section); protected: - SectionInfo fPackageAttributesSection; + PackageFileSection fPackageAttributesSection; private: + status_t _Init(int fd, bool keepFD); + status_t _ParseAttributeTree( AttributeHandlerContext* context); @@ -247,11 +149,14 @@ private: size_t* _stringLength = NULL); private: + const char* fFileType; BErrorOutput* fErrorOutput; int fFD; bool fOwnsFD; - SectionInfo* fCurrentSection; + PackageFileHeapReader* fHeapReader; + + PackageFileSection* fCurrentSection; AttributeHandlerList fAttributeHandlerStack; @@ -260,6 +165,232 @@ private: }; +// #pragma mark - attribute handlers + + +class ReaderImplBase::AttributeHandlerContext { +public: + BErrorOutput* errorOutput; + union { + BPackageContentHandler* packageContentHandler; + BLowLevelPackageContentHandler* lowLevelHandler; + }; + bool hasLowLevelHandler; + + BHPKGPackageSectionID section; + +public: + AttributeHandlerContext( + BErrorOutput* errorOutput, + BPackageContentHandler* + packageContentHandler, + BHPKGPackageSectionID section); + AttributeHandlerContext( + BErrorOutput* errorOutput, + BLowLevelPackageContentHandler* + lowLevelHandler, + BHPKGPackageSectionID section); + + void ErrorOccurred(); +}; + + +class ReaderImplBase::AttributeHandler + : public SinglyLinkedListLinkImpl { +public: + virtual ~AttributeHandler(); + + void SetLevel(int level); + virtual status_t HandleAttribute( + AttributeHandlerContext* context, uint8 id, + const AttributeValue& value, + AttributeHandler** _handler); + + virtual status_t Delete(AttributeHandlerContext* context); + +protected: + int fLevel; +}; + + +class ReaderImplBase::IgnoreAttributeHandler : public AttributeHandler { +}; + + +class ReaderImplBase::PackageVersionAttributeHandler : public AttributeHandler { +public: + PackageVersionAttributeHandler( + BPackageInfoAttributeValue& + packageInfoValue, + BPackageVersionData& versionData, + bool notify); + + virtual status_t HandleAttribute( + AttributeHandlerContext* context, uint8 id, + const AttributeValue& value, + AttributeHandler** _handler); + + virtual status_t Delete(AttributeHandlerContext* context); + +private: + BPackageInfoAttributeValue& fPackageInfoValue; + BPackageVersionData& fPackageVersionData; + bool fNotify; +}; + + +class ReaderImplBase::PackageResolvableAttributeHandler + : public AttributeHandler { +public: + PackageResolvableAttributeHandler( + BPackageInfoAttributeValue& + packageInfoValue); + + virtual status_t HandleAttribute( + AttributeHandlerContext* context, uint8 id, + const AttributeValue& value, + AttributeHandler** _handler); + + virtual status_t Delete(AttributeHandlerContext* context); + +private: + BPackageInfoAttributeValue& fPackageInfoValue; +}; + + +class ReaderImplBase::PackageResolvableExpressionAttributeHandler + : public AttributeHandler { +public: + PackageResolvableExpressionAttributeHandler( + BPackageInfoAttributeValue& + packageInfoValue); + + virtual status_t HandleAttribute( + AttributeHandlerContext* context, uint8 id, + const AttributeValue& value, + AttributeHandler** _handler); + + virtual status_t Delete(AttributeHandlerContext* context); + +private: + BPackageInfoAttributeValue& fPackageInfoValue; +}; + + +class ReaderImplBase::PackageAttributeHandler : public AttributeHandler { +public: + virtual status_t HandleAttribute( + AttributeHandlerContext* context, uint8 id, + const AttributeValue& value, + AttributeHandler** _handler); + +private: + BPackageInfoAttributeValue fPackageInfoValue; +}; + + +class ReaderImplBase::LowLevelAttributeHandler : public AttributeHandler { +public: + LowLevelAttributeHandler(); + LowLevelAttributeHandler(uint8 id, + const BPackageAttributeValue& value, + void* parentToken, void* token); + + virtual status_t HandleAttribute( + AttributeHandlerContext* context, uint8 id, + const AttributeValue& value, + AttributeHandler** _handler); + virtual status_t Delete(AttributeHandlerContext* context); + +private: + void* fParentToken; + void* fToken; + uint8 fID; + AttributeValue fValue; +}; + + +// #pragma mark - template and inline methods + + +template +status_t +ReaderImplBase::Init(int fd, bool keepFD, Header& header) +{ + status_t error = _Init(fd, keepFD); + if (error != B_OK) + return error; + + // stat the file + struct stat st; + if (fstat(FD(), &st) < 0) { + ErrorOutput()->PrintError("Error: Failed to access %s file: %s\n", + fFileType, strerror(errno)); + return errno; + } + + // read the header + if ((error = ReadBuffer(0, &header, sizeof(header))) != B_OK) + return error; + + // check the header + + // magic + if (B_BENDIAN_TO_HOST_INT32(header.magic) != kMagic) { + ErrorOutput()->PrintError("Error: Invalid %s file: Invalid " + "magic\n", fFileType); + return B_BAD_DATA; + } + + // version + if (B_BENDIAN_TO_HOST_INT16(header.version) != kVersion) { + ErrorOutput()->PrintError("Error: Invalid/unsupported %s file " + "version (%d)\n", fFileType, + B_BENDIAN_TO_HOST_INT16(header.version)); + return B_MISMATCHED_VALUES; + } + + // header size + uint64 heapOffset = B_BENDIAN_TO_HOST_INT16(header.header_size); + if (heapOffset < (off_t)sizeof(header)) { + ErrorOutput()->PrintError("Error: Invalid %s file: Invalid header " + "size (%" B_PRIu64 ")\n", fFileType, heapOffset); + return B_BAD_DATA; + } + + // total size + uint64 totalSize = B_BENDIAN_TO_HOST_INT64(header.total_size); + if (totalSize != (uint64)st.st_size) { + ErrorOutput()->PrintError("Error: Invalid %s file: Total size in " + "header (%" B_PRIu64 ") doesn't agree with total file size (%" + B_PRIdOFF ")\n", fFileType, totalSize, st.st_size); + return B_BAD_DATA; + } + + // heap size + uint64 compressedHeapSize + = B_BENDIAN_TO_HOST_INT64(header.heap_size_compressed); + if (compressedHeapSize > totalSize + || heapOffset > totalSize - compressedHeapSize) { + ErrorOutput()->PrintError("Error: Invalid %s file: Heap size in " + "header (%" B_PRIu64 ") doesn't agree with total file size (%" + B_PRIu64 ") and heap offset (%" B_PRIu64 ")\n", fFileType, + compressedHeapSize, totalSize, heapOffset); + return B_BAD_DATA; + } + + error = InitHeapReader( + B_BENDIAN_TO_HOST_INT32(header.heap_compression), + B_BENDIAN_TO_HOST_INT32(header.heap_chunk_size), heapOffset, + compressedHeapSize, + B_BENDIAN_TO_HOST_INT64(header.heap_size_uncompressed)); + if (error != B_OK) + return error; + + return B_OK; +} + + inline int ReaderImplBase::FD() const { @@ -274,7 +405,7 @@ ReaderImplBase::ErrorOutput() const } -ReaderImplBase::SectionInfo* +PackageFileSection* ReaderImplBase::CurrentSection() { return fCurrentSection; @@ -282,7 +413,7 @@ ReaderImplBase::CurrentSection() void -ReaderImplBase::SetCurrentSection(SectionInfo* section) +ReaderImplBase::SetCurrentSection(PackageFileSection* section) { fCurrentSection = section; } diff --git a/headers/private/package/hpkg/RepositoryReaderImpl.h b/headers/private/package/hpkg/RepositoryReaderImpl.h index 5e17a009ab..4d3951282f 100644 --- a/headers/private/package/hpkg/RepositoryReaderImpl.h +++ b/headers/private/package/hpkg/RepositoryReaderImpl.h @@ -25,8 +25,7 @@ namespace BPrivate { class RepositoryReaderImpl : public ReaderImplBase { typedef ReaderImplBase inherited; public: - RepositoryReaderImpl( - BErrorOutput* errorOutput); + RepositoryReaderImpl(BErrorOutput* errorOutput); ~RepositoryReaderImpl(); status_t Init(const char* fileName); @@ -42,7 +41,6 @@ private: struct RootAttributeHandler; private: - SectionInfo fRepositoryInfoSection; BRepositoryInfo fRepositoryInfo; }; diff --git a/headers/private/package/hpkg/RepositoryWriterImpl.h b/headers/private/package/hpkg/RepositoryWriterImpl.h index 0d538448e0..3909202fd0 100644 --- a/headers/private/package/hpkg/RepositoryWriterImpl.h +++ b/headers/private/package/hpkg/RepositoryWriterImpl.h @@ -50,10 +50,9 @@ private: status_t _RegisterCurrentPackageInfo(); status_t _WriteRepositoryInfo(hpkg_repo_header& header, - ssize_t& _infoLengthCompressed); - off_t _WritePackageAttributes( - hpkg_repo_header& header, off_t startOffset, - ssize_t& _packagesLengthCompressed); + uint64& _length); + void _WritePackageAttributes( + hpkg_repo_header& header, uint64& _length); struct PackageNameSet; diff --git a/headers/private/package/hpkg/WriterImplBase.h b/headers/private/package/hpkg/WriterImplBase.h index 54858bf7b5..0965bcf731 100644 --- a/headers/private/package/hpkg/WriterImplBase.h +++ b/headers/private/package/hpkg/WriterImplBase.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -29,16 +30,19 @@ class BErrorOutput; namespace BPrivate { +class AbstractDataWriter; +class PackageFileHeapWriter; + struct hpkg_header; + class WriterImplBase { public: - WriterImplBase(BErrorOutput* errorOutput); + WriterImplBase(const char* fileType, + BErrorOutput* errorOutput); ~WriterImplBase(); protected: - - struct AttributeValue { union { int64 signedInt; @@ -92,63 +96,10 @@ protected: void _DeleteChildren(); }; - - struct AbstractDataWriter { - AbstractDataWriter(); - virtual ~AbstractDataWriter(); - - uint64 BytesWritten() const; - - virtual status_t WriteDataNoThrow(const void* buffer, - size_t size) = 0; - - void WriteDataThrows(const void* buffer, size_t size); - - protected: - uint64 fBytesWritten; - }; - - - struct FDDataWriter : AbstractDataWriter { - FDDataWriter(int fd, off_t offset, BErrorOutput* errorOutput); - - virtual status_t WriteDataNoThrow(const void* buffer, - size_t size); - - off_t Offset() const; - - private: - int fFD; - off_t fOffset; - BErrorOutput* fErrorOutput; - }; - - - struct ZlibDataWriter : AbstractDataWriter, private BDataOutput { - ZlibDataWriter(AbstractDataWriter* dataWriter); - - void Init(); - - void Finish(); - - virtual status_t WriteDataNoThrow(const void* buffer, - size_t size); - - private: - // BDataOutput - virtual status_t WriteData(const void* buffer, size_t size); - - private: - AbstractDataWriter* fDataWriter; - ZlibCompressor fCompressor; - }; - - typedef DoublyLinkedList PackageAttributeList; protected: - status_t Init(const char* fileName, const char* type, - uint32 flags); + status_t Init(const char* fileName, uint32 flags); void RegisterPackageInfo( PackageAttributeList& attributeList, @@ -180,13 +131,16 @@ protected: void WriteAttributeValue(const AttributeValue& value, uint8 encoding); void WriteUnsignedLEB128(uint64 value); - inline void WriteString(const char* string); template inline void Write(const Type& value); + inline void WriteString(const char* string); + inline void WriteBuffer(const void* data, size_t size); + // appends data to the heap - void WriteBuffer(const void* buffer, size_t size, + void RawWriteBuffer(const void* buffer, size_t size, off_t offset); + // writes to the file directly inline int FD() const; inline uint32 Flags() const; @@ -197,11 +151,11 @@ protected: inline const StringCache& PackageStringCache() const; inline StringCache& PackageStringCache(); - inline AbstractDataWriter* DataWriter() const; - inline void SetDataWriter(AbstractDataWriter* dataWriter); - inline void SetFinished(bool finished); +protected: + PackageFileHeapWriter* fHeapWriter; + private: static const BHPKGAttributeID kDefaultVersionAttributeID = B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MAJOR; @@ -211,6 +165,7 @@ private: const PackageAttributeList& attributes); private: + const char* fFileType; BErrorOutput* fErrorOutput; const char* fFileName; uint32 fFlags; @@ -224,41 +179,25 @@ private: }; -inline uint64 -WriterImplBase::AbstractDataWriter::BytesWritten() const -{ - return fBytesWritten; -} - - -inline void -WriterImplBase::AbstractDataWriter::WriteDataThrows(const void* buffer, - size_t size) -{ - status_t error = WriteDataNoThrow(buffer, size); - if (error != B_OK) - throw status_t(error); -} - -inline off_t -WriterImplBase::FDDataWriter::Offset() const -{ - return fOffset; -} - - template inline void WriterImplBase::Write(const Type& value) { - fDataWriter->WriteDataThrows(&value, sizeof(Type)); + WriteBuffer(&value, sizeof(Type)); } inline void WriterImplBase::WriteString(const char* string) { - fDataWriter->WriteDataThrows(string, strlen(string) + 1); + WriteBuffer(string, strlen(string) + 1); +} + + +inline void +WriterImplBase::WriteBuffer(const void* data, size_t size) +{ + fDataWriter->WriteDataThrows(data, size); } @@ -276,20 +215,6 @@ WriterImplBase::Flags() const } -inline WriterImplBase::AbstractDataWriter* -WriterImplBase::DataWriter() const -{ - return fDataWriter; -} - - -inline void -WriterImplBase::SetDataWriter(AbstractDataWriter* dataWriter) -{ - fDataWriter = dataWriter; -} - - inline const WriterImplBase::PackageAttributeList& WriterImplBase::PackageAttributes() const { diff --git a/src/add-ons/kernel/file_systems/packagefs/Jamfile b/src/add-ons/kernel/file_systems/packagefs/Jamfile index 28f9826890..e2061a8a75 100644 --- a/src/add-ons/kernel/file_systems/packagefs/Jamfile +++ b/src/add-ons/kernel/file_systems/packagefs/Jamfile @@ -83,6 +83,8 @@ HAIKU_PACKAGE_FS_PACKAGE_READER_SOURCES = PackageDataReader.cpp PackageEntry.cpp PackageEntryAttribute.cpp + PackageFileHeapAccessorBase.cpp + PackageFileHeapReader.cpp PackageReaderImpl.cpp ReaderImplBase.cpp diff --git a/src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp b/src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp index f8bb5b0e7d..81ec8cbc18 100644 --- a/src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp @@ -28,17 +28,9 @@ using BPackageKit::BHPKG::BFDDataReader; static status_t -read_package_data(const PackageData& data, BDataReader* dataReader, - off_t offset, void* buffer, size_t* bufferSize) +read_package_data(const PackageData& data, BDataReader* reader, off_t offset, + void* buffer, size_t* bufferSize) { - // create a PackageDataReader - BAbstractBufferedDataReader* reader; - status_t error = GlobalFactory::Default()->CreatePackageDataReader( - dataReader, data, reader); - if (error != B_OK) - RETURN_ERROR(error); - ObjectDeleter readerDeleter(reader); - // check the offset if (offset < 0 || (uint64)offset > data.UncompressedSize()) return B_BAD_VALUE; @@ -131,19 +123,20 @@ UnpackingAttributeCookie::ReadAttribute(PackageNode* packageNode, const PackageData& data = attribute->Data(); if (data.IsEncodedInline()) { // inline data - BBufferDataReader dataReader(data.InlineData(), data.CompressedSize()); + BBufferDataReader dataReader(data.InlineData(), + data.UncompressedSize()); return read_package_data(data, &dataReader, offset, buffer, bufferSize); } - // data not inline -- open the package + // data not inline -- let the package create a data reader for us Package* package = packageNode->GetPackage(); - int fd = package->Open(); - if (fd < 0) - RETURN_ERROR(fd); - PackageCloser packageCloser(package); + BAbstractBufferedDataReader* reader; + status_t error = package->CreateDataReader(data, reader); + if (error != B_OK) + return error; + ObjectDeleter readerDeleter(reader); - BFDDataReader dataReader(fd); - return read_package_data(data, &dataReader, offset, buffer, bufferSize); + return read_package_data(data, reader, offset, buffer, bufferSize); } diff --git a/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.cpp b/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.cpp index 9f9d2a3f1e..f7d44141f5 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.cpp @@ -20,13 +20,10 @@ static const uint32 kMaxCachedBuffers = 32; GlobalFactory::GlobalFactory() : - fBufferPool(BPackageKit::BHPKG::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, + fBufferPool(BPackageKit::BHPKG::V1::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, kMaxCachedBuffers), - fPackageDataReaderFactoryV1(&fBufferPool), - fPackageDataReaderFactoryV2(&fBufferPool) + fPackageDataReaderFactory(&fBufferPool) { - STATIC_ASSERT((int)BPackageKit::BHPKG::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB - == (int)BPackageKit::BHPKG::V1::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB); } @@ -73,20 +70,10 @@ GlobalFactory::Default() status_t GlobalFactory::CreatePackageDataReader(BDataReader* dataReader, - const PackageData& data, BAbstractBufferedDataReader*& _reader) + const PackageDataV1& data, BAbstractBufferedDataReader*& _reader) { - switch (data.Version()) { - case 1: - return fPackageDataReaderFactoryV1.CreatePackageDataReader( - dataReader, data.DataV1(), _reader); - - case 2: - return fPackageDataReaderFactoryV2.CreatePackageDataReader( - dataReader, data.DataV2(), _reader); - - default: - return B_NOT_SUPPORTED; - } + return fPackageDataReaderFactory.CreatePackageDataReader(dataReader, data, + _reader); } diff --git a/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.h b/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.h index ca06a9d4b9..46f75da820 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.h +++ b/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.h @@ -1,27 +1,24 @@ /* - * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2009-2013, Ingo Weinhold, ingo_weinhold@gmx.de. * Distributed under the terms of the MIT License. */ #ifndef GLOBAL_FACTORY_H #define GLOBAL_FACTORY_H -#include #include #include "BlockBufferPoolKernel.h" +#include "PackageData.h" +using BPackageKit::BHPKG::BBufferPool; using BPackageKit::BHPKG::BDataReader; using BPackageKit::BHPKG::BPackageData; using BPackageKit::BHPKG::BAbstractBufferedDataReader; typedef BPackageKit::BHPKG::V1::BPackageDataReaderFactory BPackageDataReaderFactoryV1; -typedef BPackageKit::BHPKG::BPackageDataReaderFactory - BPackageDataReaderFactoryV2; - -class PackageData; class GlobalFactory { @@ -35,7 +32,7 @@ public: static GlobalFactory* Default(); status_t CreatePackageDataReader(BDataReader* dataReader, - const PackageData& data, + const PackageDataV1& data, BAbstractBufferedDataReader*& _reader); private: @@ -45,8 +42,7 @@ private: static GlobalFactory* sDefaultInstance; BlockBufferPoolKernel fBufferPool; - BPackageDataReaderFactoryV1 fPackageDataReaderFactoryV1; - BPackageDataReaderFactoryV2 fPackageDataReaderFactoryV2; + BPackageDataReaderFactoryV1 fPackageDataReaderFactory; }; #endif // GLOBAL_FACTORY_H diff --git a/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp b/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp index ad73f22a02..094b231850 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp @@ -12,17 +12,21 @@ #include #include -#include #include +#include #include #include -#include #include #include + +#include +#include +#include #include #include #include "DebugSupport.h" +#include "GlobalFactory.h" #include "PackageDirectory.h" #include "PackageFile.h" #include "PackageSymlink.h" @@ -32,10 +36,12 @@ using namespace BPackageKit; -typedef BPackageKit::BHPKG::BErrorOutput BErrorOutput; -typedef BPackageKit::BHPKG::BPackageInfoAttributeValue - BPackageInfoAttributeValue; -typedef BPackageKit::BHPKG::BPackageVersionData BPackageVersionData; +using BPackageKit::BHPKG::BDataOutput; +using BPackageKit::BHPKG::BErrorOutput; +using BPackageKit::BHPKG::BFDDataReader; +using BPackageKit::BHPKG::BPackageInfoAttributeValue; +using BPackageKit::BHPKG::BPackageVersionData; +using BPackageKit::BHPKG::BPrivate::PackageFileHeapReader; // current format version types typedef BPackageKit::BHPKG::BPackageContentHandler BPackageContentHandler; @@ -581,6 +587,129 @@ private: }; +// #pragma mark - HeapReader + + +struct Package::HeapReader { + virtual ~HeapReader() + { + } + + virtual void UpdateFD(int fd) = 0; + + virtual status_t CreateDataReader(const PackageData& data, + BAbstractBufferedDataReader*& _reader) = 0; +}; + + +// #pragma mark - HeapReaderV1 + + +struct Package::HeapReaderV1 : public HeapReader, private BDataReader { +public: + HeapReaderV1(int fd) + : + fFileReader(fd) + { + } + + ~HeapReaderV1() + { + } + + virtual void UpdateFD(int fd) + { + fFileReader.SetFD(fd); + } + + virtual status_t CreateDataReader(const PackageData& data, + BAbstractBufferedDataReader*& _reader) + { + return GlobalFactory::Default()->CreatePackageDataReader(this, + data.DataV1(), _reader); + } + +private: + // BDataReader + + virtual status_t ReadData(off_t offset, void* buffer, size_t size) + { + return fFileReader.ReadData(offset, buffer, size); + } + +private: + BFDDataReader fFileReader; +}; + + +// #pragma mark - HeapReaderV2 + + +struct Package::HeapReaderV2 : public HeapReader, + private BAbstractBufferedDataReader, private BErrorOutput { +public: + HeapReaderV2() + : + fHeapReader(NULL) + { + } + + ~HeapReaderV2() + { + delete fHeapReader; + } + + status_t Init(const PackageFileHeapReader* heapReader, int fd) + { + fHeapReader = heapReader->Clone(); + if (fHeapReader == NULL) + return B_NO_MEMORY; + + fHeapReader->SetErrorOutput(this); + fHeapReader->SetFD(fd); + + return B_OK; + } + + virtual void UpdateFD(int fd) + { + fHeapReader->SetFD(fd); + } + + virtual status_t CreateDataReader(const PackageData& data, + BAbstractBufferedDataReader*& _reader) + { + return BPackageKit::BHPKG::BPackageDataReaderFactory() + .CreatePackageDataReader(this, data.DataV2(), _reader); + } + +private: + // BAbstractBufferedDataReader + + virtual status_t ReadData(off_t offset, void* buffer, size_t size) + { + return fHeapReader->ReadData(offset, buffer, size); + } + + virtual status_t ReadDataToOutput(off_t offset, size_t size, + BDataOutput* output) + { + return fHeapReader->ReadDataToOutput(offset, size, output); + } + +private: + // BErrorOutput + + virtual void PrintErrorVarArgs(const char* format, va_list args) + { +// TODO:... + } + +private: + PackageFileHeapReader* fHeapReader; +}; + + // #pragma mark - Package @@ -595,6 +724,7 @@ Package::Package(::Volume* volume, dev_t deviceID, ino_t nodeID) fLinkDirectory(NULL), fFD(-1), fOpenCount(0), + fHeapReader(NULL), fNodeID(nodeID), fDeviceID(deviceID) { @@ -604,6 +734,8 @@ Package::Package(::Volume* volume, dev_t deviceID, ino_t nodeID) Package::~Package() { + delete fHeapReader; + while (PackageNode* node = fNodes.RemoveHead()) node->ReleaseReference(); @@ -652,7 +784,22 @@ Package::Load() if (error != B_OK) RETURN_ERROR(error); - RETURN_ERROR(packageReader.ParseContent(&handler)); + error = packageReader.ParseContent(&handler); + if (error != B_OK) + RETURN_ERROR(error); + + // create a heap reader + HeapReaderV2* heapReader = new(std::nothrow) HeapReaderV2; + if (heapReader == NULL) + RETURN_ERROR(B_NO_MEMORY); + + error = heapReader->Init(packageReader.HeapReader(), fd); + if (error != B_OK) { + RETURN_ERROR(error); + } + + fHeapReader = heapReader; + return B_OK; } if (error != B_MISMATCHED_VALUES) @@ -671,7 +818,16 @@ Package::Load() if (error != B_OK) RETURN_ERROR(error); - RETURN_ERROR(packageReader.ParseContent(&handler)); + error = packageReader.ParseContent(&handler); + if (error != B_OK) + RETURN_ERROR(error); + + // create a heap reader + fHeapReader = new(std::nothrow) HeapReaderV1(fd); + if (fHeapReader == NULL) + RETURN_ERROR(B_NO_MEMORY); + + return B_OK; } @@ -765,6 +921,10 @@ Package::Open() } fOpenCount = 1; + + if (fHeapReader != NULL) + fHeapReader->UpdateFD(fFD); + return fFD; } @@ -781,5 +941,19 @@ Package::Close() if (--fOpenCount == 0) { close(fFD); fFD = -1; + + if (fHeapReader != NULL) + fHeapReader->UpdateFD(fFD); } } + + +status_t +Package::CreateDataReader(const PackageData& data, + BAbstractBufferedDataReader*& _reader) +{ + if (fHeapReader == NULL) + return B_BAD_VALUE; + + return fHeapReader->CreateDataReader(data, _reader); +} diff --git a/src/add-ons/kernel/file_systems/packagefs/package/Package.h b/src/add-ons/kernel/file_systems/packagefs/package/Package.h index bb239fad22..b75996bf04 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/Package.h +++ b/src/add-ons/kernel/file_systems/packagefs/package/Package.h @@ -6,6 +6,7 @@ #define PACKAGE_H +#include #include #include @@ -23,6 +24,7 @@ using BPackageKit::BPackageArchitecture; +using BPackageKit::BHPKG::BAbstractBufferedDataReader; class PackageLinkDirectory; @@ -82,6 +84,9 @@ public: int Open(); void Close(); + status_t CreateDataReader(const PackageData& data, + BAbstractBufferedDataReader*& _reader); + const PackageNodeList& Nodes() const { return fNodes; } const ResolvableList& Resolvables() const { return fResolvables; } @@ -92,6 +97,9 @@ private: struct LoaderErrorOutput; struct LoaderContentHandler; struct LoaderContentHandlerV1; + struct HeapReader; + struct HeapReaderV1; + struct HeapReaderV2; private: mutex fLock; @@ -104,6 +112,7 @@ private: PackageLinkDirectory* fLinkDirectory; int fFD; uint32 fOpenCount; + HeapReader* fHeapReader; Package* fFileNameHashTableNext; ino_t fNodeID; dev_t fDeviceID; diff --git a/src/add-ons/kernel/file_systems/packagefs/package/PackageData.h b/src/add-ons/kernel/file_systems/packagefs/package/PackageData.h index 313531e1d9..6597aa30ca 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/PackageData.h +++ b/src/add-ons/kernel/file_systems/packagefs/package/PackageData.h @@ -83,7 +83,7 @@ PackageData::CompressedSize() const { if (fVersion == 1) return DataV1().CompressedSize(); - return DataV2().CompressedSize(); + return DataV2().Size(); } @@ -92,7 +92,7 @@ PackageData::UncompressedSize() const { if (fVersion == 1) return DataV1().UncompressedSize(); - return DataV2().UncompressedSize(); + return DataV2().Size(); } diff --git a/src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp b/src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp index 297515468d..08106735b7 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp @@ -48,10 +48,10 @@ private: struct PackageFile::DataAccessor { - DataAccessor(PackageData* data) + DataAccessor(Package* package, PackageData* data) : + fPackage(package), fData(data), - fDataReader(NULL), fReader(NULL), fFileCache(NULL) { @@ -62,27 +62,15 @@ struct PackageFile::DataAccessor { { file_cache_delete(fFileCache); delete fReader; - delete fDataReader; mutex_destroy(&fLock); } status_t Init(dev_t deviceID, ino_t nodeID, int fd) { - // create a BDataReader for the compressed data - if (fData->IsEncodedInline()) { - fDataReader = new(std::nothrow) BBufferDataReader( - fData->InlineData(), fData->CompressedSize()); - } else - fDataReader = new(std::nothrow) BFDDataReader(fd); - - if (fDataReader == NULL) - RETURN_ERROR(B_NO_MEMORY); - - // create a PackageDataReader - status_t error = GlobalFactory::Default()->CreatePackageDataReader( - fDataReader, *fData, fReader); + // create a reader for the data + status_t error = fPackage->CreateDataReader(*fData, fReader); if (error != B_OK) - RETURN_ERROR(error); + return error; // create a file cache fFileCache = file_cache_create(deviceID, nodeID, @@ -128,8 +116,8 @@ struct PackageFile::DataAccessor { private: mutex fLock; + Package* fPackage; PackageData* fData; - BDataReader* fDataReader; BAbstractBufferedDataReader* fReader; void* fFileCache; }; @@ -168,7 +156,7 @@ PackageFile::VFSInit(dev_t deviceID, ino_t nodeID) PackageCloser packageCloser(fPackage); // create the data accessor - fDataAccessor = new(std::nothrow) DataAccessor(&fData); + fDataAccessor = new(std::nothrow) DataAccessor(GetPackage(), &fData); if (fDataAccessor == NULL) RETURN_ERROR(B_NO_MEMORY); diff --git a/src/bin/package/command_dump.cpp b/src/bin/package/command_dump.cpp index c25437e63b..93401cdf2e 100644 --- a/src/bin/package/command_dump.cpp +++ b/src/bin/package/command_dump.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2009-2013, Ingo Weinhold, ingo_weinhold@gmx.de. * Distributed under the terms of the MIT License. */ diff --git a/src/bin/package/command_extract.cpp b/src/bin/package/command_extract.cpp index 9d85a1fc73..ae5a6070bd 100644 --- a/src/bin/package/command_extract.cpp +++ b/src/bin/package/command_extract.cpp @@ -44,7 +44,9 @@ using BPackageKit::BHPKG::BAbstractBufferedDataReader; using BPackageKit::BHPKG::BBlockBufferPoolNoLock; using BPackageKit::BHPKG::BBufferDataReader; +using BPackageKit::BHPKG::BBufferPool; using BPackageKit::BHPKG::BDataReader; +using BPackageKit::BHPKG::BErrorOutput; using BPackageKit::BHPKG::BFDDataReader; using BPackageKit::BHPKG::BPackageInfoAttributeValue; using BPackageKit::BHPKG::BStandardErrorOutput; @@ -54,38 +56,93 @@ struct VersionPolicyV1 { typedef BPackageKit::BHPKG::V1::BPackageContentHandler PackageContentHandler; typedef BPackageKit::BHPKG::V1::BPackageData PackageData; - typedef BPackageKit::BHPKG::V1::BPackageDataReaderFactory - PackageDataReaderFactory; typedef BPackageKit::BHPKG::V1::BPackageEntry PackageEntry; typedef BPackageKit::BHPKG::V1::BPackageEntryAttribute PackageEntryAttribute; typedef BPackageKit::BHPKG::V1::BPackageReader PackageReader; + typedef BDataReader HeapReaderBase; - static const int kDefaultDataChunkSize - = BPackageKit::BHPKG::V1::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB; + static inline size_t BufferSize() + { + return BPackageKit::BHPKG::V1::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB; + } static inline const char* PackageInfoFileName() { return BPackageKit::BHPKG::V1::B_HPKG_PACKAGE_INFO_FILE_NAME; } + + static inline uint64 PackageDataCompressedSize(const PackageData& data) + { + return data.CompressedSize(); + } + + static inline uint64 PackageDataUncompressedSize(const PackageData& data) + { + return data.UncompressedSize(); + } + + static status_t GetHeapReader(PackageReader& packageReader, + HeapReaderBase*& _heapReader, bool& _mustDelete) + { + _heapReader = new(std::nothrow) BFDDataReader( + packageReader.PackageFileFD()); + _mustDelete = false; + return _heapReader != NULL ? B_OK : B_NO_MEMORY; + } + + static status_t CreatePackageDataReader(BBufferPool* bufferPool, + HeapReaderBase* heapReader, const PackageData& data, + BAbstractBufferedDataReader*& _reader) + { + return BPackageKit::BHPKG::V1::BPackageDataReaderFactory(bufferPool) + .CreatePackageDataReader(heapReader, data, _reader); + } }; struct VersionPolicyV2 { typedef BPackageKit::BHPKG::BPackageContentHandler PackageContentHandler; typedef BPackageKit::BHPKG::BPackageData PackageData; - typedef BPackageKit::BHPKG::BPackageDataReaderFactory - PackageDataReaderFactory; typedef BPackageKit::BHPKG::BPackageEntry PackageEntry; typedef BPackageKit::BHPKG::BPackageEntryAttribute PackageEntryAttribute; typedef BPackageKit::BHPKG::BPackageReader PackageReader; + typedef BAbstractBufferedDataReader HeapReaderBase; - static const int kDefaultDataChunkSize - = BPackageKit::BHPKG::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB; + static inline size_t BufferSize() + { + return 64 * 1024; + } static inline const char* PackageInfoFileName() { return BPackageKit::BHPKG::B_HPKG_PACKAGE_INFO_FILE_NAME; } + + static inline uint64 PackageDataCompressedSize(const PackageData& data) + { + return data.Size(); + } + + static inline uint64 PackageDataUncompressedSize(const PackageData& data) + { + return data.Size(); + } + + static status_t GetHeapReader(PackageReader& packageReader, + HeapReaderBase*& _heapReader, bool& _mustDelete) + { + _heapReader = packageReader.HeapReader(); + _mustDelete = false; + return B_OK; + } + + static status_t CreatePackageDataReader(BBufferPool* bufferPool, + HeapReaderBase* heapReader, const PackageData& data, + BAbstractBufferedDataReader*& _reader) + { + return BPackageKit::BHPKG::BPackageDataReaderFactory() + .CreatePackageDataReader(heapReader, data, _reader); + } }; @@ -227,10 +284,11 @@ private: template struct PackageContentExtractHandler : VersionPolicy::PackageContentHandler { - PackageContentExtractHandler(int packageFileFD) + PackageContentExtractHandler(BBufferPool* bufferPool, + typename VersionPolicy::HeapReaderBase* heapReader) : - fBufferPool(VersionPolicy::kDefaultDataChunkSize, 2), - fPackageFileReader(packageFileFD), + fBufferPool(bufferPool), + fPackageFileReader(heapReader), fDataBuffer(NULL), fDataBufferSize(0), fRootFilterEntry(NULL, NULL, true), @@ -247,11 +305,7 @@ struct PackageContentExtractHandler : VersionPolicy::PackageContentHandler { status_t Init() { - status_t error = fBufferPool.Init(); - if (error != B_OK) - return error; - - error = fRootFilterEntry.Init(); + status_t error = fRootFilterEntry.Init(); if (error != B_OK) return error; @@ -434,15 +488,8 @@ struct PackageContentExtractHandler : VersionPolicy::PackageContentHandler { } // write data - status_t error; - const typename VersionPolicy::PackageData& data = entry->Data(); - if (data.IsEncodedInline()) { - BBufferDataReader dataReader(data.InlineData(), - data.CompressedSize()); - error = _ExtractFileData(&dataReader, data, fd); - } else - error = _ExtractFileData(&fPackageFileReader, data, fd); - + status_t error = _ExtractFileData(fPackageFileReader, entry->Data(), + fd); if (error != B_OK) return error; } else if (S_ISLNK(entry->Mode())) { @@ -528,14 +575,8 @@ struct PackageContentExtractHandler : VersionPolicy::PackageContentHandler { } // write data - status_t error; - const typename VersionPolicy::PackageData& data = attribute->Data(); - if (data.IsEncodedInline()) { - BBufferDataReader dataReader(data.InlineData(), - data.CompressedSize()); - error = _ExtractFileData(&dataReader, data, fd); - } else - error = _ExtractFileData(&fPackageFileReader, data, fd); + status_t error = _ExtractFileData(fPackageFileReader, attribute->Data(), + fd); fs_close_attr(fd); @@ -642,20 +683,20 @@ private: return path; } - status_t _ExtractFileData(BDataReader* dataReader, + status_t _ExtractFileData( + typename VersionPolicy::HeapReaderBase* dataReader, const typename VersionPolicy::PackageData& data, int fd) { // create a PackageDataReader BAbstractBufferedDataReader* reader; - status_t error = typename VersionPolicy::PackageDataReaderFactory( - &fBufferPool) - .CreatePackageDataReader(dataReader, data, reader); + status_t error = VersionPolicy::CreatePackageDataReader(fBufferPool, + dataReader, data, reader); if (error != B_OK) return error; ObjectDeleter readerDeleter(reader); // write the data - off_t bytesRemaining = data.UncompressedSize(); + off_t bytesRemaining = VersionPolicy::PackageDataUncompressedSize(data); off_t offset = 0; while (bytesRemaining > 0) { // read @@ -688,14 +729,14 @@ private: } private: - BBlockBufferPoolNoLock fBufferPool; - BFDDataReader fPackageFileReader; - void* fDataBuffer; - size_t fDataBufferSize; - Entry fRootFilterEntry; - int fBaseDirectory; - const char* fInfoFileName; - bool fErrorOccurred; + BBufferPool* fBufferPool; + typename VersionPolicy::HeapReaderBase* fPackageFileReader; + void* fDataBuffer; + size_t fDataBufferSize; + Entry fRootFilterEntry; + int fBaseDirectory; + const char* fInfoFileName; + bool fErrorOccurred; }; @@ -707,6 +748,12 @@ do_extract(const char* packageFileName, const char* changeToDirectory, { // open package BStandardErrorOutput errorOutput; + BBlockBufferPoolNoLock bufferPool(VersionPolicy::BufferSize(), 2); + if (bufferPool.Init() != B_OK) { + errorOutput.PrintError("Error: Out of memory!\n"); + exit(1); + } + typename VersionPolicy::PackageReader packageReader(&errorOutput); status_t error = packageReader.Init(packageFileName); if (error != B_OK) { @@ -715,8 +762,20 @@ do_extract(const char* packageFileName, const char* changeToDirectory, exit(1); } - PackageContentExtractHandler handler( - packageReader.PackageFileFD()); + typename VersionPolicy::HeapReaderBase* heapReader; + bool mustDeleteHeapReader; + error = VersionPolicy::GetHeapReader(packageReader, heapReader, + mustDeleteHeapReader); + if (error != B_OK) { + fprintf(stderr, "Error: Failed to create heap reader: \"%s\"\n", + strerror(error)); + exit(1); + } + ObjectDeleter heapReaderDeleter( + mustDeleteHeapReader ? heapReader : NULL); + + PackageContentExtractHandler handler(&bufferPool, + heapReader); error = handler.Init(); if (error != B_OK) exit(1); @@ -727,7 +786,7 @@ do_extract(const char* packageFileName, const char* changeToDirectory, for (int i = 0; i < explicitEntryCount; i++) { const char* entryName = explicitEntries[i]; if (entryName[0] == '\0' || entryName[0] == '/') { - fprintf(stderr, "Error: Invalid entry name: \"%s\".", + fprintf(stderr, "Error: Invalid entry name: \"%s\"\n", entryName); exit(1); } diff --git a/src/bin/package/command_list.cpp b/src/bin/package/command_list.cpp index de9220c217..b6f0b4147a 100644 --- a/src/bin/package/command_list.cpp +++ b/src/bin/package/command_list.cpp @@ -29,7 +29,9 @@ using namespace BPackageKit; +using BPackageKit::BHPKG::BErrorOutput; using BPackageKit::BHPKG::BPackageInfoAttributeValue; +using BPackageKit::BHPKG::BStandardErrorOutput; struct VersionPolicyV1 { @@ -39,6 +41,12 @@ struct VersionPolicyV1 { typedef BPackageKit::BHPKG::V1::BPackageEntryAttribute PackageEntryAttribute; typedef BPackageKit::BHPKG::V1::BPackageReader PackageReader; + + static inline uint64 PackageDataSize( + const BPackageKit::BHPKG::V1::BPackageData& data) + { + return data.UncompressedSize(); + } }; struct VersionPolicyV2 { @@ -46,6 +54,12 @@ struct VersionPolicyV2 { typedef BPackageKit::BHPKG::BPackageEntry PackageEntry; typedef BPackageKit::BHPKG::BPackageEntryAttribute PackageEntryAttribute; typedef BPackageKit::BHPKG::BPackageReader PackageReader; + + static inline uint64 PackageDataSize( + const BPackageKit::BHPKG::BPackageData& data) + { + return data.Size(); + } }; @@ -67,7 +81,8 @@ struct PackageContentListHandler : VersionPolicy::PackageContentHandler { // name and size printf("%-*s", indentation < 32 ? 32 - indentation : 0, entry->Name()); - printf(" %8llu", (unsigned long long)entry->Data().UncompressedSize()); + printf(" %8llu", + (unsigned long long)VersionPolicy::PackageDataSize(entry->Data())); // time struct tm* time = localtime(&entry->ModifiedTime().tv_sec); @@ -113,7 +128,8 @@ struct PackageContentListHandler : VersionPolicy::PackageContentHandler { printf("%*s<", indentation, ""); printf("%-*s %8llu", indentation < 31 ? 31 - indentation : 0, attribute->Name(), - (unsigned long long)attribute->Data().UncompressedSize()); + (unsigned long long)VersionPolicy::PackageDataSize( + attribute->Data())); uint32 type = attribute->Type(); if (isprint(type & 0xff) && isprint((type >> 8) & 0xff) @@ -352,7 +368,7 @@ do_list(const char* packageFileName, bool listAttributes, bool filePathsOnly, bool ignoreVersionError) { // open package - BHPKG::BStandardErrorOutput errorOutput; + BStandardErrorOutput errorOutput; typename VersionPolicy::PackageReader packageReader(&errorOutput); status_t error = packageReader.Init(packageFileName); if (error != B_OK) { diff --git a/src/build/libpackage/Jamfile b/src/build/libpackage/Jamfile index e55836571e..0b911b3988 100644 --- a/src/build/libpackage/Jamfile +++ b/src/build/libpackage/Jamfile @@ -28,6 +28,7 @@ HPKG_SOURCES = PoolBuffer.cpp DataOutput.cpp DataReader.cpp + DataWriters.cpp ErrorOutput.cpp FDDataReader.cpp HPKGDefs.cpp @@ -36,6 +37,9 @@ HPKG_SOURCES = PackageDataReader.cpp PackageEntry.cpp PackageEntryAttribute.cpp + PackageFileHeapAccessorBase.cpp + PackageFileHeapReader.cpp + PackageFileHeapWriter.cpp PackageReader.cpp PackageReaderImpl.cpp PackageWriter.cpp diff --git a/src/kits/package/Jamfile b/src/kits/package/Jamfile index ac7bd2c0ab..8ea102622a 100644 --- a/src/kits/package/Jamfile +++ b/src/kits/package/Jamfile @@ -18,6 +18,7 @@ HPKG_SOURCES = BufferPool.cpp DataOutput.cpp DataReader.cpp + DataWriters.cpp ErrorOutput.cpp FDDataReader.cpp HPKGDefs.cpp @@ -26,6 +27,8 @@ HPKG_SOURCES = PackageDataReader.cpp PackageEntry.cpp PackageEntryAttribute.cpp + PackageFileHeapAccessorBase.cpp + PackageFileHeapReader.cpp PackageFileHeapWriter.cpp PackageReader.cpp PackageReaderImpl.cpp diff --git a/src/kits/package/hpkg/DataWriters.cpp b/src/kits/package/hpkg/DataWriters.cpp new file mode 100644 index 0000000000..29318c890d --- /dev/null +++ b/src/kits/package/hpkg/DataWriters.cpp @@ -0,0 +1,125 @@ +/* + * Copyright 2011, Oliver Tappe + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include +#include +#include + +#include + + +namespace BPackageKit { + +namespace BHPKG { + +namespace BPrivate { + + +// #pragma mark - AbstractDataWriter + + +AbstractDataWriter::AbstractDataWriter() + : + fBytesWritten(0) +{ +} + + +AbstractDataWriter::~AbstractDataWriter() +{ +} + + +// #pragma mark - FDDataWriter + + +FDDataWriter::FDDataWriter(int fd, off_t offset, BErrorOutput* errorOutput) + : + fFD(fd), + fOffset(offset), + fErrorOutput(errorOutput) +{ +} + + +status_t +FDDataWriter::WriteDataNoThrow(const void* buffer, size_t size) +{ + ssize_t bytesWritten = pwrite(fFD, buffer, size, fOffset); + if (bytesWritten < 0) { + fErrorOutput->PrintError( + "WriteDataNoThrow(%p, %lu) failed to write data: %s\n", buffer, + size, strerror(errno)); + return errno; + } + if ((size_t)bytesWritten != size) { + fErrorOutput->PrintError( + "WriteDataNoThrow(%p, %lu) failed to write all data\n", buffer, + size); + return B_ERROR; + } + + fOffset += size; + fBytesWritten += size; + return B_OK; +} + + +// #pragma mark - ZlibDataWriter + + +ZlibDataWriter::ZlibDataWriter(AbstractDataWriter* dataWriter) + : + fDataWriter(dataWriter), + fCompressor(this) +{ +} + + +void +ZlibDataWriter::Init() +{ + status_t error = fCompressor.Init(); + if (error != B_OK) + throw status_t(error); +} + + +void +ZlibDataWriter::Finish() +{ + status_t error = fCompressor.Finish(); + if (error != B_OK) + throw status_t(error); +} + + +status_t +ZlibDataWriter::WriteDataNoThrow(const void* buffer, + size_t size) +{ + status_t error = fCompressor.CompressNext(buffer, size); + if (error == B_OK) + fBytesWritten += size; + return error; +} + + +status_t +ZlibDataWriter::WriteData(const void* buffer, size_t size) +{ + return fDataWriter->WriteDataNoThrow(buffer, size); +} + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit diff --git a/src/kits/package/hpkg/PackageContentHandler.cpp b/src/kits/package/hpkg/PackageContentHandler.cpp index 70bfc7e015..584af3cb77 100644 --- a/src/kits/package/hpkg/PackageContentHandler.cpp +++ b/src/kits/package/hpkg/PackageContentHandler.cpp @@ -30,9 +30,6 @@ static const char* kAttributeNames[B_HPKG_ATTRIBUTE_ID_ENUM_COUNT + 1] = { "file:attribute", "file:attribute:type", "data", - "data:compression", - "data:size", - "data:chunk_size", "symlink:path", "package:name", "package:summary", diff --git a/src/kits/package/hpkg/PackageData.cpp b/src/kits/package/hpkg/PackageData.cpp index 31b288f3b0..2e008fca46 100644 --- a/src/kits/package/hpkg/PackageData.cpp +++ b/src/kits/package/hpkg/PackageData.cpp @@ -21,10 +21,7 @@ using namespace BPrivate; BPackageData::BPackageData() : - fCompressedSize(0), - fUncompressedSize(0), - fChunkSize(0), - fCompression(B_HPKG_COMPRESSION_NONE), + fSize(0), fEncodedInline(true) { } @@ -33,7 +30,7 @@ BPackageData::BPackageData() void BPackageData::SetData(uint64 size, uint64 offset) { - fUncompressedSize = fCompressedSize = size; + fSize = size; fOffset = offset; fEncodedInline = false; } @@ -42,7 +39,7 @@ BPackageData::SetData(uint64 size, uint64 offset) void BPackageData::SetData(uint8 size, const void* data) { - fUncompressedSize = fCompressedSize = size; + fSize = size; if (size > 0) memcpy(fInlineData, data, size); fEncodedInline = true; diff --git a/src/kits/package/hpkg/PackageDataReader.cpp b/src/kits/package/hpkg/PackageDataReader.cpp index 6cccfe8ed4..9c808fec44 100644 --- a/src/kits/package/hpkg/PackageDataReader.cpp +++ b/src/kits/package/hpkg/PackageDataReader.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2009-2013, Ingo Weinhold, ingo_weinhold@gmx.de. * Distributed under the terms of the MIT License. */ @@ -12,9 +12,6 @@ #include #include -#include -#include -#include #include #include @@ -27,59 +24,20 @@ namespace BHPKG { using namespace BPrivate; -// minimum/maximum zlib chunk size we consider sane -static const size_t kMinSaneZlibChunkSize = 1024; -static const size_t kMaxSaneZlibChunkSize = 10 * 1024 * 1024; - -// maximum number of entries in the zlib offset table buffer -static const uint32 kMaxZlibOffsetTableBufferSize = 512; - -static const size_t kUncompressedReaderBufferSize - = B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB; +// #pragma mark - PackageDataHeapReader -// #pragma mark - BPackageDataReader - - -class PackageDataReader : public BAbstractBufferedDataReader { +class PackageDataHeapReader : public BAbstractBufferedDataReader { public: - PackageDataReader(BDataReader* dataReader) + PackageDataHeapReader(BAbstractBufferedDataReader* dataReader, + const BPackageData& data) : - fDataReader(dataReader) + fDataReader(dataReader), + fOffset(data.Offset()), + fSize(data.Size()) { } - virtual ~PackageDataReader() - { - } - - virtual status_t Init(const BPackageData& data) = 0; - -protected: - BDataReader* fDataReader; -}; - - -// #pragma mark - UncompressedPackageDataReader - - -class UncompressedPackageDataReader : public PackageDataReader { -public: - UncompressedPackageDataReader(BDataReader* dataReader, - BBufferPool* bufferPool) - : - PackageDataReader(dataReader), - fBufferPool(bufferPool) - { - } - - status_t Init(const BPackageData& data) - { - fOffset = data.Offset(); - fSize = data.UncompressedSize(); - return B_OK; - } - virtual status_t ReadData(off_t offset, void* buffer, size_t size) { if (size == 0) @@ -106,332 +64,55 @@ public: if ((uint64)offset > fSize || size > fSize - offset) return B_BAD_VALUE; - // get a temporary buffer - PoolBuffer* buffer = fBufferPool->GetBuffer( - kUncompressedReaderBufferSize); - if (buffer == NULL) - return B_NO_MEMORY; - PoolBufferPutter bufferPutter(fBufferPool, &buffer); - - while (size > 0) { - // read into the buffer - size_t toRead = std::min(size, buffer->Size()); - status_t error = fDataReader->ReadData(fOffset + offset, - buffer->Buffer(), toRead); - if (error != B_OK) - return error; - - // write to the output - error = output->WriteData(buffer->Buffer(), toRead); - if (error != B_OK) - return error; - - offset += toRead; - size -= toRead; - } - - return B_OK; + return fDataReader->ReadDataToOutput(fOffset + offset, size, output); } private: - BBufferPool* fBufferPool; - uint64 fOffset; - uint64 fSize; + BAbstractBufferedDataReader* fDataReader; + uint64 fOffset; + uint64 fSize; }; -// #pragma mark - ZlibPackageDataReader +// #pragma mark - PackageDataHeapReader -class ZlibPackageDataReader : public PackageDataReader { +class PackageDataInlineReader : public BBufferDataReader { public: - ZlibPackageDataReader(BDataReader* dataReader, BBufferPool* bufferPool) + PackageDataInlineReader(const BPackageData& data) : - PackageDataReader(dataReader), - fBufferPool(bufferPool), - fUncompressBuffer(NULL), - fOffsetTable(NULL) + BBufferDataReader(fData.InlineData(), data.Size()), + fData(data) { } - ~ZlibPackageDataReader() - { - delete[] fOffsetTable; - - fBufferPool->PutBuffer(&fUncompressBuffer); - } - - status_t Init(const BPackageData& data) - { - fOffset = data.Offset(); - fCompressedSize = data.CompressedSize(); - fUncompressedSize = data.UncompressedSize(); - fChunkSize = data.ChunkSize(); - - // validate chunk size - if (fChunkSize == 0) - fChunkSize = B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB; - if (fChunkSize < kMinSaneZlibChunkSize - || fChunkSize > kMaxSaneZlibChunkSize) { - return B_BAD_DATA; - } - - fChunkCount = (fUncompressedSize + (fChunkSize - 1)) / fChunkSize; - fOffsetTableSize = (fChunkCount - 1) * sizeof(uint64); - if (fOffsetTableSize >= fCompressedSize) - return B_BAD_DATA; - - // allocate a buffer for the offset table - if (fChunkCount > 1) { - fOffsetTableBufferEntryCount = std::min(fChunkCount - 1, - (uint64)kMaxZlibOffsetTableBufferSize); - fOffsetTable = new(std::nothrow) uint64[ - fOffsetTableBufferEntryCount]; - if (fOffsetTable == NULL) - return B_NO_MEMORY; - - fOffsetTableIndex = -1; - // mark the table content invalid - } else - fChunkSize = fUncompressedSize; - - // mark uncompressed content invalid - fUncompressedChunk = -1; - - return B_OK; - } - - virtual status_t ReadDataToOutput(off_t offset, size_t size, - BDataOutput* output) - { - // check offset and size - if (size == 0) - return B_OK; - - if (offset < 0) - return B_BAD_VALUE; - - if ((uint64)offset > fUncompressedSize - || size > fUncompressedSize - offset) { - return B_BAD_VALUE; - } - - // get our uncompressed chunk buffer back, if possible - bool newBuffer; - if (fBufferPool->GetBuffer(fChunkSize, &fUncompressBuffer, &newBuffer) - == NULL) { - return B_NO_MEMORY; - } - PoolBufferPutter uncompressBufferPutter(fBufferPool, - &fUncompressBuffer); - - if (newBuffer) - fUncompressedChunk = -1; - - // uncompress - int64 chunkIndex = offset / fChunkSize; - off_t chunkOffset = chunkIndex * fChunkSize; - size_t inChunkOffset = offset - chunkOffset; - - while (size > 0) { - // read and uncompress the chunk - status_t error = _ReadChunk(chunkIndex); - if (error != B_OK) - return error; - - // write data to output - size_t toCopy = std::min(size, (size_t)fChunkSize - inChunkOffset); - error = output->WriteData( - (uint8*)fUncompressBuffer->Buffer() + inChunkOffset, toCopy); - if (error != B_OK) - return error; - - size -= toCopy; - - chunkIndex++; - chunkOffset += fChunkSize; - inChunkOffset = 0; - } - - return B_OK; - } - private: - status_t _ReadChunk(int64 chunkIndex) - { - if (chunkIndex == fUncompressedChunk) - return B_OK; - - // get the chunk offset and size - uint64 offset; - uint32 compressedSize; - status_t error = _GetCompressedChunkOffsetAndSize(chunkIndex, offset, - compressedSize); - if (error != B_OK) - return error; - - uint32 uncompressedSize = (uint64)chunkIndex + 1 < fChunkCount - ? fChunkSize : fUncompressedSize - chunkIndex * fChunkSize; - - // read the chunk - if (compressedSize == uncompressedSize) { - // the chunk is not compressed -- read it directly into the - // uncompressed buffer - error = fDataReader->ReadData(offset, fUncompressBuffer->Buffer(), - compressedSize); - } else { - // read to a read buffer and uncompress - PoolBuffer* readBuffer = fBufferPool->GetBuffer(fChunkSize); - if (readBuffer == NULL) - return B_NO_MEMORY; - PoolBufferPutter readBufferPutter(fBufferPool, readBuffer); - - error = fDataReader->ReadData(offset, readBuffer->Buffer(), - compressedSize); - if (error != B_OK) - return error; - - size_t actuallyUncompressedSize; - error = ZlibDecompressor::DecompressSingleBuffer( - readBuffer->Buffer(), compressedSize, - fUncompressBuffer->Buffer(), uncompressedSize, - actuallyUncompressedSize); - if (error == B_OK && actuallyUncompressedSize != uncompressedSize) - error = B_BAD_DATA; - } - - if (error != B_OK) { - // error reading/decompressing data -- mark the cached data invalid - fUncompressedChunk = -1; - return error; - } - - fUncompressedChunk = chunkIndex; - return B_OK; - } - - status_t _GetCompressedChunkOffsetAndSize(int64 chunkIndex, uint64& _offset, - uint32& _size) - { - // get the offset - uint64 offset; - if (chunkIndex == 0) { - // first chunk is at 0 - offset = 0; - } else { - status_t error = _GetCompressedChunkRelativeOffset(chunkIndex, - offset); - if (error != B_OK) - return error; - } - - // get the end offset - uint64 endOffset; - if ((uint64)chunkIndex + 1 == fChunkCount) { - // last chunk end with the end of the data - endOffset = fCompressedSize - fOffsetTableSize; - } else { - status_t error = _GetCompressedChunkRelativeOffset(chunkIndex + 1, - endOffset); - if (error != B_OK) - return error; - } - - // sanity check - if (endOffset < offset) - return B_BAD_DATA; - - _offset = fOffset + fOffsetTableSize + offset; - _size = endOffset - offset; - return B_OK; - } - - status_t _GetCompressedChunkRelativeOffset(int64 chunkIndex, - uint64& _offset) - { - if (fOffsetTableIndex < 0 || fOffsetTableIndex > chunkIndex - || fOffsetTableIndex + fOffsetTableBufferEntryCount <= chunkIndex) { - // read the table at the given index, or, if we can, the whole table - int64 readAtIndex = fChunkCount - 1 > fOffsetTableBufferEntryCount - ? chunkIndex : 1; - uint32 entriesToRead = std::min( - (uint64)fOffsetTableBufferEntryCount, - fChunkCount - readAtIndex); - - status_t error = fDataReader->ReadData( - fOffset + (readAtIndex - 1) * sizeof(uint64), - fOffsetTable, entriesToRead * sizeof(uint64)); - if (error != B_OK) { - fOffsetTableIndex = -1; - return error; - } - - fOffsetTableIndex = readAtIndex; - } - - // get and check the offset - _offset = fOffsetTable[chunkIndex - fOffsetTableIndex]; - if (_offset > fCompressedSize - fOffsetTableSize) - return B_BAD_DATA; - - return B_OK; - } - -private: - BBufferPool* fBufferPool; - PoolBuffer* fUncompressBuffer; - int64 fUncompressedChunk; - - uint64 fOffset; - uint64 fUncompressedSize; - uint64 fCompressedSize; - uint64 fOffsetTableSize; - uint64 fChunkCount; - uint32 fChunkSize; - uint32 fOffsetTableBufferEntryCount; - uint64* fOffsetTable; - int32 fOffsetTableIndex; + BPackageData fData; }; // #pragma mark - BPackageDataReaderFactory -BPackageDataReaderFactory::BPackageDataReaderFactory(BBufferPool* bufferPool) - : - fBufferPool(bufferPool) +BPackageDataReaderFactory::BPackageDataReaderFactory() { } status_t -BPackageDataReaderFactory::CreatePackageDataReader(BDataReader* dataReader, - const BPackageData& data, BAbstractBufferedDataReader*& _reader) +BPackageDataReaderFactory::CreatePackageDataReader( + BAbstractBufferedDataReader* dataReader, const BPackageData& data, + BAbstractBufferedDataReader*& _reader) { - PackageDataReader* reader; - - switch (data.Compression()) { - case B_HPKG_COMPRESSION_NONE: - reader = new(std::nothrow) UncompressedPackageDataReader( - dataReader, fBufferPool); - break; - case B_HPKG_COMPRESSION_ZLIB: - reader = new(std::nothrow) ZlibPackageDataReader(dataReader, - fBufferPool); - break; - default: - return B_BAD_VALUE; - } + BAbstractBufferedDataReader* reader; + if (data.IsEncodedInline()) + reader = new(std::nothrow) PackageDataInlineReader(data); + else + reader = new(std::nothrow) PackageDataHeapReader(dataReader, data); if (reader == NULL) return B_NO_MEMORY; - status_t error = reader->Init(data); - if (error != B_OK) { - delete reader; - return error; - } - _reader = reader; return B_OK; } diff --git a/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp b/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp new file mode 100644 index 0000000000..be7d453ea5 --- /dev/null +++ b/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp @@ -0,0 +1,247 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include + + +namespace BPackageKit { + +namespace BHPKG { + +namespace BPrivate { + + +// #pragma mark - OffsetArray + + +PackageFileHeapAccessorBase::OffsetArray::OffsetArray() + : + fOffsets(NULL) +{ +} + + +PackageFileHeapAccessorBase::OffsetArray::~OffsetArray() +{ + delete[] fOffsets; +} + + +bool +PackageFileHeapAccessorBase::OffsetArray::InitChunksOffsets( + size_t totalChunkCount, size_t baseIndex, const uint16* chunkSizes, + size_t chunkCount) +{ + if (totalChunkCount <= 1) + return true; + + if (fOffsets == NULL) { + fOffsets = new(std::nothrow) uint32[totalChunkCount]; + if (fOffsets == NULL) + return false; + fOffsets[0] = 0; + // Value that serves as a marker that all offsets are 32 bit. We'll + // replace it, when necessary. + } + + uint64 offset = (*this)[baseIndex]; + for (size_t i = 0; i < chunkCount; i++) { + offset += (uint64)chunkSizes[i] + 1; + // the stored value is chunkSize - 1 + size_t index = baseIndex + i + 1; + // (baseIndex + i) is the index of the chunk whose size is stored in + // chunkSizes[i]. We compute the offset of the following element + // which we store at index (baseIndex + i + 1). + + if (offset <= ~(uint32)0) { + fOffsets[index] = (uint32)offset; + } else { + if (fOffsets[0] == 0) { + // Not scaled to allow for 64 bit offsets yet. Do that. + fOffsets[0] = index; + uint32* newOffsets = new(std::nothrow) uint32[ + 2 * totalChunkCount - fOffsets[0]]; + if (newOffsets == NULL) + return false; + + memcpy(newOffsets, fOffsets, + sizeof(newOffsets[0]) * fOffsets[0]); + + delete[] fOffsets; + fOffsets = newOffsets; + } + + index += index - fOffsets[0]; + fOffsets[index] = (uint32)offset; + fOffsets[index + 1] = uint32(offset >> 32); + } + } + + return true; +} + + +bool +PackageFileHeapAccessorBase::OffsetArray::Init(size_t totalChunkCount, + const OffsetArray& other) +{ + if (other.fOffsets == NULL) + return true; + + size_t elementCount = other.fOffsets[0] == 0 + ? totalChunkCount + : 2 * totalChunkCount - other.fOffsets[0]; + + fOffsets = new(std::nothrow) uint32[elementCount]; + if (fOffsets == NULL) + return false; + + memcpy(fOffsets, other.fOffsets, elementCount * sizeof(fOffsets[0])); + return true; +} + + +// #pragma mark - PackageFileHeapAccessorBase + + +PackageFileHeapAccessorBase::PackageFileHeapAccessorBase( + BErrorOutput* errorOutput, int fd, off_t heapOffset) + : + fErrorOutput(errorOutput), + fFD(fd), + fHeapOffset(heapOffset), + fCompressedHeapSize(0), + fUncompressedHeapSize(0) +{ +} + + +PackageFileHeapAccessorBase::~PackageFileHeapAccessorBase() +{ +} + + +status_t +PackageFileHeapAccessorBase::ReadDataToOutput(off_t offset, size_t size, + BDataOutput* output) +{ + if (size == 0) + return B_OK; + + if (offset < 0 || (uint64)offset > fUncompressedHeapSize + || size > fUncompressedHeapSize - offset) { + return B_BAD_VALUE; + } + + // allocate buffers for compressed and uncompressed data + uint16* compressedDataBuffer = (uint16*)malloc(kChunkSize); + uint16* uncompressedDataBuffer = (uint16*)malloc(kChunkSize); + MemoryDeleter compressedDataBufferDeleter(compressedDataBuffer); + MemoryDeleter uncompressedDataBufferDeleter(uncompressedDataBuffer); + if (compressedDataBuffer == NULL || uncompressedDataBuffer == NULL) + return B_NO_MEMORY; + + // read the data + size_t chunkIndex = size_t(offset / kChunkSize); + size_t inChunkOffset = (uint64)offset - (uint64)chunkIndex * kChunkSize; + size_t remainingBytes = size; + + while (remainingBytes > 0) { + status_t error = ReadAndDecompressChunk(chunkIndex, + compressedDataBuffer, uncompressedDataBuffer); + if (error != B_OK) + return error; + + size_t toWrite = std::min((size_t)kChunkSize - inChunkOffset, + remainingBytes); + // The last chunk may be shorter than kChunkSize, but since + // size (and thus remainingSize) had been clamped, that doesn't + // harm. + error = output->WriteData( + (char*)uncompressedDataBuffer + inChunkOffset, toWrite); + if (error != B_OK) + return error; + + remainingBytes -= toWrite; + chunkIndex++; + inChunkOffset = 0; + } + + return B_OK; +} + + +status_t +PackageFileHeapAccessorBase::ReadAndDecompressChunkData(uint64 offset, + size_t compressedSize, size_t uncompressedSize, void* compressedDataBuffer, + void* uncompressedDataBuffer) +{ + // if uncompressed, read directly into the uncompressed data buffer + if (compressedSize == uncompressedSize) + return ReadFileData(offset, uncompressedDataBuffer, compressedSize); + + // otherwise read into the other buffer and decompress + status_t error = ReadFileData(offset, compressedDataBuffer, compressedSize); + if (error != B_OK) + return error; + + size_t actualSize; + error = ZlibDecompressor::DecompressSingleBuffer(compressedDataBuffer, + compressedSize, uncompressedDataBuffer, uncompressedSize, actualSize); + if (error != B_OK) { + fErrorOutput->PrintError("Failed to decompress data chunk: %s\n", + strerror(error)); + return error; + } + + if (actualSize != uncompressedSize) { + fErrorOutput->PrintError("Failed to decompress data chunk: chunk " + "size mismatch\n"); + return B_ERROR; + } + + return B_OK; +} + + +status_t +PackageFileHeapAccessorBase::ReadFileData(uint64 offset, void* buffer, + size_t size) +{ + ssize_t bytesRead = pread(fFD, buffer, size, fHeapOffset + (off_t)offset); + if (bytesRead < 0) { + fErrorOutput->PrintError("_ReadData(%" B_PRIu64 "%p, %zu) failed to " + "read data: %s\n", offset, buffer, size, strerror(errno)); + return errno; + } + if ((size_t)bytesRead != size) { + fErrorOutput->PrintError("_ReadData(%" B_PRIu64 "%p, %zu) failed to " + "read all data\n", offset, buffer, size); + return B_ERROR; + } + + return B_OK; +} + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit diff --git a/src/kits/package/hpkg/PackageFileHeapReader.cpp b/src/kits/package/hpkg/PackageFileHeapReader.cpp new file mode 100644 index 0000000000..e02313ba87 --- /dev/null +++ b/src/kits/package/hpkg/PackageFileHeapReader.cpp @@ -0,0 +1,131 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include +#include + +#include +#include + +#include +#include + + +// minimum length of data we require before trying to zlib compress them +static const size_t kZlibCompressionSizeThreshold = 64; + + +namespace BPackageKit { + +namespace BHPKG { + +namespace BPrivate { + + +PackageFileHeapReader::PackageFileHeapReader(BErrorOutput* errorOutput, int fd, + off_t heapOffset, off_t compressedHeapSize, uint64 uncompressedHeapSize) + : + PackageFileHeapAccessorBase(errorOutput, fd, heapOffset), + fOffsets() +{ + fCompressedHeapSize = compressedHeapSize; + fUncompressedHeapSize = uncompressedHeapSize; +} + + +PackageFileHeapReader::~PackageFileHeapReader() +{ +} + + +status_t +PackageFileHeapReader::Init() +{ + if (fUncompressedHeapSize == 0) + return B_OK; + + // Determine number of chunks and adjust the compressed heap size (subtract + // the size of the chunk size array at the end). Note that the size of the + // last chunk has not been saved, since it size is implied. + ssize_t chunkCount = (fUncompressedHeapSize + kChunkSize - 1) / kChunkSize; + if (chunkCount <= 0) + return B_OK; + + fCompressedHeapSize -= (chunkCount - 1) * 2; + + // allocate a buffer + uint16* buffer = (uint16*)malloc(kChunkSize); + if (buffer == NULL) + return B_NO_MEMORY; + MemoryDeleter bufferDeleter(buffer); + + // read the chunk size array + size_t remainingChunks = chunkCount - 1; + size_t index = 0; + uint64 offset = fCompressedHeapSize; + while (remainingChunks > 0) { + size_t toRead = std::min(remainingChunks, kChunkSize / 2); + status_t error = ReadFileData(offset, buffer, toRead * 2); + if (error != B_OK) + return error; + + if (!fOffsets.InitChunksOffsets(chunkCount, index, buffer, toRead)) + return B_NO_MEMORY; + + remainingChunks -= toRead; + index += toRead; + offset += toRead * 2; + } + + return B_OK; +} + + +PackageFileHeapReader* +PackageFileHeapReader::Clone() const +{ + PackageFileHeapReader* clone = new(std::nothrow) PackageFileHeapReader( + fErrorOutput, fFD, fHeapOffset, fCompressedHeapSize, + fUncompressedHeapSize); + if (clone == NULL) + return NULL; + + ssize_t chunkCount = (fUncompressedHeapSize + kChunkSize - 1) / kChunkSize; + if (!clone->fOffsets.Init(chunkCount, fOffsets)) { + delete clone; + return NULL; + } + + return clone; +} + + +status_t +PackageFileHeapReader::ReadAndDecompressChunk(size_t chunkIndex, + void* compressedDataBuffer, void* uncompressedDataBuffer) +{ + uint64 offset = fOffsets[chunkIndex]; + bool isLastChunk + = uint64(chunkIndex + 1) * kChunkSize >= fUncompressedHeapSize; + size_t compressedSize = isLastChunk + ? fCompressedHeapSize - offset + : fOffsets[chunkIndex + 1] - offset; + size_t uncompressedSize = isLastChunk + ? fUncompressedHeapSize - (uint64)chunkIndex * kChunkSize + : kChunkSize; + + return ReadAndDecompressChunkData(offset, compressedSize, uncompressedSize, + compressedDataBuffer, uncompressedDataBuffer); +} + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit diff --git a/src/kits/package/hpkg/PackageFileHeapWriter.cpp b/src/kits/package/hpkg/PackageFileHeapWriter.cpp new file mode 100644 index 0000000000..d7fc523c97 --- /dev/null +++ b/src/kits/package/hpkg/PackageFileHeapWriter.cpp @@ -0,0 +1,645 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + + +// minimum length of data we require before trying to zlib compress them +static const size_t kZlibCompressionSizeThreshold = 64; + + +namespace BPackageKit { + +namespace BHPKG { + +namespace BPrivate { + + +struct PackageFileHeapWriter::Chunk { + uint64 offset; + uint32 compressedSize; + uint32 uncompressedSize; + void* buffer; +}; + + +struct PackageFileHeapWriter::ChunkSegment { + ssize_t chunkIndex; + uint32 toKeepOffset; + uint32 toKeepSize; +}; + + +struct PackageFileHeapWriter::ChunkBuffer { + ChunkBuffer(BErrorOutput* errorOutput, int fd, off_t heapOffset, + size_t bufferSize) + : + fErrorOutput(errorOutput), + fFD(fd), + fHeapOffset(heapOffset), + fChunks(), + fCurrentChunkIndex(0), + fNextReadIndex(0), + fSegments(), + fCurrentSegmentIndex(0), + fBuffers(), + fUnusedBuffers(), + fBufferSize(bufferSize) + { + } + + ~ChunkBuffer() + { + for (int32 i = 0; void* buffer = fBuffers.ItemAt(i); i++) + free(buffer); + } + + bool PushChunkSegment(uint64 chunkOffset, uint32 compressedSize, + uint32 uncompressedSize, uint32 toKeepOffset, uint32 toKeepSize) + { + ChunkSegment segment; + segment.toKeepOffset = toKeepOffset; + segment.toKeepSize = toKeepSize; + + // might refer to the last chunk + segment.chunkIndex = fChunks.Count() - 1; + + if (segment.chunkIndex < 0 + || fChunks.ElementAt(segment.chunkIndex).offset != chunkOffset) { + // no, need to push a new chunk + segment.chunkIndex++; + + Chunk chunk; + chunk.offset = chunkOffset; + chunk.compressedSize = compressedSize; + chunk.uncompressedSize = uncompressedSize; + chunk.buffer = NULL; + if (!fChunks.Add(chunk)) + return false; + } + + return fSegments.Add(segment); + } + + bool HasMoreSegments() const + { + return fCurrentSegmentIndex < fSegments.Count(); + } + + const ChunkSegment& CurrentSegment() const + { + return fSegments[fCurrentSegmentIndex]; + } + + const Chunk& ChunkAt(ssize_t index) const + { + return fChunks[index]; + } + + bool HasMoreChunksToRead() const + { + return fNextReadIndex < fChunks.Count(); + } + + bool HasBufferedChunk() const + { + return fCurrentChunkIndex < fNextReadIndex; + } + + uint64 NextReadOffset() const + { + return fChunks[fNextReadIndex].offset; + } + + void ReadNextChunk() + { + if (!HasMoreChunksToRead()) + throw status_t(B_BAD_VALUE); + + Chunk& chunk = fChunks[fNextReadIndex++]; + chunk.buffer = _GetBuffer(); + + ssize_t bytesRead = pread(fFD, chunk.buffer, chunk.compressedSize, + fHeapOffset + chunk.offset); + if (bytesRead < 0) { + fErrorOutput->PrintError("PackageFileHeapWriter::ChunkBuffer::" + "ReadNextChunk(offset: %" B_PRIu64 ", size: %" B_PRIu32 + ") failed to read data: %s\n", chunk.offset, + chunk.compressedSize, strerror(errno)); + throw status_t(errno); + } + + if ((size_t)bytesRead != chunk.compressedSize) { + fErrorOutput->PrintError("PackageFileHeapWriter::ChunkBuffer::" + "ReadNextChunk(offset: %" B_PRIu64 ", size: %" B_PRIu32 + ") failed to read all data\n", chunk.offset, + chunk.compressedSize); + throw status_t(B_ERROR); + } + } + + void CurrentSegmentDone() + { + // Unless the next segment refers to the same chunk, advance to the next + // chunk. + const ChunkSegment& segment = fSegments[fCurrentSegmentIndex++]; + if (!HasMoreSegments() + || segment.chunkIndex != CurrentSegment().chunkIndex) { + _PutBuffer(fChunks[fCurrentChunkIndex++].buffer); + } + } + +private: + void* _GetBuffer() + { + if (!fUnusedBuffers.IsEmpty()) + return fUnusedBuffers.RemoveItem(fUnusedBuffers.CountItems() - 1); + + void* buffer = malloc(fBufferSize); + if (buffer == NULL && !fBuffers.AddItem(buffer)) { + free(buffer); + throw std::bad_alloc(); + } + + return buffer; + } + + void _PutBuffer(void* buffer) + { + if (buffer != NULL && !fUnusedBuffers.AddItem(buffer)) { + fBuffers.RemoveItem(buffer); + free(buffer); + } + } + +private: + BErrorOutput* fErrorOutput; + int fFD; + off_t fHeapOffset; + + Array fChunks; + ssize_t fCurrentChunkIndex; + ssize_t fNextReadIndex; + + Array fSegments; + ssize_t fCurrentSegmentIndex; + + BList fBuffers; + BList fUnusedBuffers; + size_t fBufferSize; +}; + + +PackageFileHeapWriter::PackageFileHeapWriter(BErrorOutput* errorOutput, int fd, + off_t heapOffset) + : + PackageFileHeapAccessorBase(errorOutput, fd, heapOffset), + fPendingDataBuffer(NULL), + fCompressedDataBuffer(NULL), + fPendingDataSize(0), + fOffsets() +{ +} + + +PackageFileHeapWriter::~PackageFileHeapWriter() +{ + _Uninit(); +} + + +void +PackageFileHeapWriter::Init() +{ + // allocate data buffers + fPendingDataBuffer = malloc(kChunkSize); + fCompressedDataBuffer = malloc(kChunkSize); + if (fPendingDataBuffer == NULL || fCompressedDataBuffer == NULL) + throw std::bad_alloc(); +} + + +void +PackageFileHeapWriter::Reinit(PackageFileHeapReader* heapReader) +{ + fHeapOffset = heapReader->HeapOffset(); + fCompressedHeapSize = heapReader->CompressedHeapSize(); + fUncompressedHeapSize = heapReader->UncompressedHeapSize(); + fPendingDataSize = 0; + + // copy the offsets array + size_t chunkCount = (fUncompressedHeapSize + kChunkSize - 1) / kChunkSize; + if (chunkCount > 0) { + if (!fOffsets.AddUninitialized(chunkCount)) + throw std::bad_alloc(); + + for (size_t i = 0; i < chunkCount; i++) + fOffsets[i] = heapReader->Offsets()[i]; + } + + // If the last chunk is partial, read it in and remove it from the offsets. + size_t lastChunkSize = fUncompressedHeapSize % kChunkSize; + if (lastChunkSize != 0) { + status_t error = heapReader->ReadData( + fUncompressedHeapSize - lastChunkSize, fPendingDataBuffer, + lastChunkSize); + if (error != B_OK) + throw error; + + fCompressedHeapSize = fOffsets[fOffsets.Count() - 1]; + fOffsets.Remove(fOffsets.Count() - 1); + } +} + + +status_t +PackageFileHeapWriter::AddData(BDataReader& dataReader, off_t size, + uint64& _offset) +{ + _offset = fUncompressedHeapSize; + + // copy the data to the heap + off_t readOffset = 0; + off_t remainingSize = size; + while (remainingSize > 0) { + // read data into pending data buffer + size_t toCopy = std::min(remainingSize, + off_t(kChunkSize - fPendingDataSize)); + status_t error = dataReader.ReadData(readOffset, + (uint8*)fPendingDataBuffer + fPendingDataSize, toCopy); + if (error != B_OK) { + fErrorOutput->PrintError("Failed to read data: %s\n", + strerror(error)); + return error; + } + + fPendingDataSize += toCopy; + fUncompressedHeapSize += toCopy; + remainingSize -= toCopy; + readOffset += toCopy; + + if (fPendingDataSize == kChunkSize) { + error = _FlushPendingData(); + if (error != B_OK) + return error; + } + } + + return B_OK; +} + + +void +PackageFileHeapWriter::RemoveDataRanges( + const ::BPrivate::RangeArray& ranges) +{ + ssize_t rangeCount = ranges.CountRanges(); + if (rangeCount == 0) + return; + + if (fOffsets.IsEmpty()) { + fErrorOutput->PrintError("Can't remove ranges from empty heap\n"); + throw status_t(B_BAD_VALUE); + } + + // We potentially have to recompress all data from the first affected chunk + // to the end (minus the removed ranges, of course). As a basic algorithm we + // can use our usual data writing strategy, i.e. read a chunk, decompress it + // to a temporary buffer, and write the data to keep via AddData(). There + // are a few complications/optimizations, though: + // * As data moves to other chunks, it may actually compress worse than + // before. While unlikely, we still have to take care of this case by + // making sure our reading end is at least a complete uncompressed chunk + // ahead of the writing end. + // * When we run into the situation that we have to move complete aligned + // chunks, we want to avoid uncompressing and recompressing them + // needlessly. + + // Build a list of (possibly partial) chunks we want to keep. + + // the first partial chunk (if any) and all chunks between ranges + ChunkBuffer chunkBuffer(fErrorOutput, fFD, fHeapOffset, kChunkSize); + uint64 writeOffset = ranges[0].offset - ranges[0].offset % kChunkSize; + uint64 readOffset = writeOffset; + for (ssize_t i = 0; i < rangeCount; i++) { + const Range& range = ranges[i]; + if (range.size > 0) { + _PushChunks(chunkBuffer, readOffset, range.offset); + readOffset = range.offset + range.size; + } + } + + if (readOffset == writeOffset) { + fErrorOutput->PrintError("Only empty ranges to remove from heap\n"); + throw status_t(B_BAD_VALUE); + } + + // all chunks after the last range + _PushChunks(chunkBuffer, readOffset, fUncompressedHeapSize); + + // Reset our state to look like all chunks from the first affected one have + // been removed and re-add all data we want to keep. + + // truncate the offsets array and reset the heap end offset + ssize_t firstChunkIndex = ssize_t(writeOffset / kChunkSize); + fCompressedHeapSize = fOffsets[firstChunkIndex]; + fOffsets.Remove(firstChunkIndex, fOffsets.Count() - firstChunkIndex); + + // we need a decompression buffer + void* decompressionBuffer = malloc(kChunkSize); + if (decompressionBuffer == NULL) + throw std::bad_alloc(); + MemoryDeleter decompressionBufferDeleter(decompressionBuffer); + + const Chunk* decompressedChunk = NULL; + + while (chunkBuffer.HasMoreSegments()) { + const ChunkSegment& segment = chunkBuffer.CurrentSegment(); + + // If we have an aligned, complete chunk, copy its compressed data. + bool copyCompressed = fPendingDataSize == 0 && segment.toKeepOffset == 0 + && segment.toKeepSize == kChunkSize; + + // Read more chunks. We need at least one buffered one to do anything + // and we want to buffer as many as necessary to ensure we don't + // overwrite one we haven't buffered yet. + while (chunkBuffer.HasMoreChunksToRead() + && (!chunkBuffer.HasBufferedChunk() + || (!copyCompressed + && chunkBuffer.NextReadOffset() + < fCompressedHeapSize + kChunkSize))) { + // read chunk + chunkBuffer.ReadNextChunk(); + } + + // copy compressed chunk data, if possible + const Chunk& chunk = chunkBuffer.ChunkAt(segment.chunkIndex); + if (copyCompressed) { + status_t error = _WriteChunk(chunk.buffer, chunk.compressedSize, + false); + if (error != B_OK) + throw error; + continue; + } + + // decompress chunk, if compressed + void* uncompressedData; + if (chunk.uncompressedSize == chunk.compressedSize) { + uncompressedData = chunk.buffer; + } else if (decompressedChunk == &chunk) { + uncompressedData = decompressionBuffer; + } else { + size_t uncompressedSize; + status_t error = ZlibDecompressor::DecompressSingleBuffer( + chunk.buffer, chunk.compressedSize, + decompressionBuffer, chunk.uncompressedSize, uncompressedSize); + if (error != B_OK) { + fErrorOutput->PrintError("Failed to decompress data chunk: %s\n", + strerror(error)); + throw error; + } + + if (uncompressedSize != chunk.uncompressedSize) { + fErrorOutput->PrintError("Failed to decompress data chunk: chunk " + "size mismatch\n"); + throw status_t(B_ERROR); + } + + decompressedChunk = &chunk; + uncompressedData = decompressionBuffer; + } + + // add chunk data + WriteDataThrows((uint8*)uncompressedData + segment.toKeepOffset, + segment.toKeepSize); + + chunkBuffer.CurrentSegmentDone(); + } +} + + +status_t +PackageFileHeapWriter::Finish() +{ + // flush pending data, if any + status_t error = _FlushPendingData(); + if (error != B_OK) + return error; + + // write chunk sizes table + + // We don't need to write the last chunk size, since it is implied by the + // total size minus the sum of all other chunk sizes. + ssize_t offsetCount = fOffsets.Count(); + if (offsetCount < 2) + return B_OK; + + // Convert the offsets to 16 bit sizes and write them. We use the (no longer + // used) pending data buffer for the conversion. + uint16* buffer = (uint16*)fPendingDataBuffer; + for (ssize_t offsetIndex = 1; offsetIndex < offsetCount;) { + ssize_t toWrite = std::min(offsetCount - offsetIndex, + ssize_t(kChunkSize / 2)); + + for (ssize_t i = 0; i < toWrite; i++, offsetIndex++) { + // store chunkSize - 1, so it fits 16 bit (chunks cannot be empty) + buffer[i] = uint16(fOffsets[offsetIndex] - fOffsets[offsetIndex - 1] + - 1); + } + + error = _WriteDataUncompressed(buffer, toWrite * 2); + if (error != B_OK) + return error; + } + + return B_OK; +} + + +status_t +PackageFileHeapWriter::ReadAndDecompressChunk(size_t chunkIndex, + void* compressedDataBuffer, void* uncompressedDataBuffer) +{ + if (uint64(chunkIndex + 1) * kChunkSize > fUncompressedHeapSize) { + // The chunk has not been written to disk yet. Its data are still in the + // pending data buffer. + memcpy(uncompressedDataBuffer, fPendingDataBuffer, fPendingDataSize); + // TODO: This can be optimized. Since we write to a BDataOutput anyway, + // there's no need to copy the data. + return B_OK; + } + + uint64 offset = fOffsets[chunkIndex]; + size_t compressedSize = chunkIndex + 1 == (size_t)fOffsets.Count() + ? fCompressedHeapSize - offset + : fOffsets[chunkIndex + 1] - offset; + + return ReadAndDecompressChunkData(offset, compressedSize, kChunkSize, + compressedDataBuffer, uncompressedDataBuffer); +} + + +status_t +PackageFileHeapWriter::WriteDataNoThrow(const void* buffer, size_t size) +{ + BBufferDataReader reader(buffer, size); + uint64 dummyOffset; + return AddData(reader, size, dummyOffset); +} + + +void +PackageFileHeapWriter::_Uninit() +{ + free(fPendingDataBuffer); + free(fCompressedDataBuffer); + fPendingDataBuffer = NULL; + fCompressedDataBuffer = NULL; +} + + +status_t +PackageFileHeapWriter::_FlushPendingData() +{ + if (fPendingDataSize == 0) + return B_OK; + + status_t error = _WriteChunk(fPendingDataBuffer, fPendingDataSize, true); + if (error == B_OK) + fPendingDataSize = 0; + + return error; +} + + +status_t +PackageFileHeapWriter::_WriteChunk(const void* data, size_t size, + bool mayCompress) +{ + // add offset + if (!fOffsets.Add(fCompressedHeapSize)) { + fErrorOutput->PrintError("Out of memory!\n"); + return B_NO_MEMORY; + } + + // Try to use zlib compression only for data large enough. + bool compress = mayCompress && size >= (off_t)kZlibCompressionSizeThreshold; + if (compress) { + status_t error = _WriteDataCompressed(data, size); + if (error != B_OK) { + if (error != B_BUFFER_OVERFLOW) + return error; + compress = false; + } + } + + // Write uncompressed, if necessary. + if (!compress) { + status_t error = _WriteDataUncompressed(data, size); + if (error != B_OK) + return error; + } + + return B_OK; +} + + +status_t +PackageFileHeapWriter::_WriteDataCompressed(const void* data, size_t size) +{ + size_t compressedSize; + status_t error = ZlibCompressor::CompressSingleBuffer(data, size, + fCompressedDataBuffer, size, compressedSize); + if (error != B_OK) + return error; + + // only use compressed data when we've actually saved space + if (compressedSize == size) + return B_BUFFER_OVERFLOW; + + return _WriteDataUncompressed(fCompressedDataBuffer, compressedSize); +} + + +status_t +PackageFileHeapWriter::_WriteDataUncompressed(const void* data, size_t size) +{ + ssize_t bytesWritten = pwrite(fFD, data, size, + fHeapOffset + (off_t)fCompressedHeapSize); + if (bytesWritten < 0) { + fErrorOutput->PrintError("Failed to write data: %s\n", strerror(errno)); + return errno; + } + if ((size_t)bytesWritten != size) { + fErrorOutput->PrintError("Failed to write all data\n"); + return B_ERROR; + } + + fCompressedHeapSize += bytesWritten; + + return B_OK; +} + + +void +PackageFileHeapWriter::_PushChunks(ChunkBuffer& chunkBuffer, uint64 startOffset, + uint64 endOffset) +{ + if (endOffset > fUncompressedHeapSize) { + fErrorOutput->PrintError("Invalid range to remove from heap\n"); + throw status_t(B_BAD_VALUE); + } + + ssize_t chunkIndex = startOffset / kChunkSize; + uint64 uncompressedChunkOffset = (uint64)chunkIndex * kChunkSize; + + while (startOffset < endOffset) { + uint32 inChunkOffset = uint32(startOffset - uncompressedChunkOffset); + uint32 uncompressedChunkSize = chunkIndex + 1 < fOffsets.Count() + ? kChunkSize + : fUncompressedHeapSize - uncompressedChunkOffset; + uint64 compressedChunkOffset = fOffsets[chunkIndex]; + uint32 compressedChunkSize = chunkIndex + 1 < fOffsets.Count() + ? fOffsets[chunkIndex + 1] - compressedChunkOffset + : fCompressedHeapSize - compressedChunkOffset; + uint32 toKeepSize = uint32(std::min( + (uint64)uncompressedChunkSize - inChunkOffset, + endOffset - startOffset)); + + if (!chunkBuffer.PushChunkSegment(compressedChunkOffset, + compressedChunkSize, uncompressedChunkSize, inChunkOffset, + toKeepSize)) { + throw std::bad_alloc(); + } + + startOffset += toKeepSize; + chunkIndex++; + uncompressedChunkOffset += uncompressedChunkSize; + } +} + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit diff --git a/src/kits/package/hpkg/PackageReader.cpp b/src/kits/package/hpkg/PackageReader.cpp index 549bf2f9e6..8453d25b40 100644 --- a/src/kits/package/hpkg/PackageReader.cpp +++ b/src/kits/package/hpkg/PackageReader.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2009-2013, Ingo Weinhold, ingo_weinhold@gmx.de. * Distributed under the terms of the MIT License. */ @@ -9,6 +9,8 @@ #include #include + +#include #include @@ -80,6 +82,14 @@ BPackageReader::PackageFileFD() } + +BAbstractBufferedDataReader* +BPackageReader::HeapReader() const +{ + return fImpl != NULL ? fImpl->HeapReader() : NULL; +} + + } // namespace BHPKG } // namespace BPackageKit diff --git a/src/kits/package/hpkg/PackageReaderImpl.cpp b/src/kits/package/hpkg/PackageReaderImpl.cpp index 40551e7b8b..1446c31ae5 100644 --- a/src/kits/package/hpkg/PackageReaderImpl.cpp +++ b/src/kits/package/hpkg/PackageReaderImpl.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -48,80 +49,16 @@ static const size_t kMaxTOCSize = 64 * 1024 * 1024; static const size_t kMaxPackageAttributesSize = 1 * 1024 * 1024; -// #pragma mark - DataAttributeHandler - - -struct PackageReaderImpl::DataAttributeHandler : AttributeHandler { - DataAttributeHandler(BPackageData* data) - : - fData(data) - { - } - - static status_t InitData(AttributeHandlerContext* context, - BPackageData* data, const AttributeValue& value) - { - if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) - data->SetData(value.data.size, value.data.raw); - else - data->SetData(value.data.size, value.data.offset); - - data->SetUncompressedSize(value.data.size); - - return B_OK; - } - - static status_t Create(AttributeHandlerContext* context, - BPackageData* data, const AttributeValue& value, - AttributeHandler*& _handler) - { - DataAttributeHandler* handler = new(std::nothrow) DataAttributeHandler( - data); - if (handler == NULL) - return B_NO_MEMORY; - - InitData(context, data, value); - - _handler = handler; - return B_OK; - } - - virtual status_t HandleAttribute(AttributeHandlerContext* context, - uint8 id, const AttributeValue& value, AttributeHandler** _handler) - { - switch (id) { - case B_HPKG_ATTRIBUTE_ID_DATA_SIZE: - fData->SetUncompressedSize(value.unsignedInt); - return B_OK; - - case B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION: - { - switch (value.unsignedInt) { - case B_HPKG_COMPRESSION_NONE: - case B_HPKG_COMPRESSION_ZLIB: - break; - default: - context->errorOutput->PrintError("Error: Invalid " - "compression type for data (%llu)\n", - value.unsignedInt); - return B_BAD_DATA; - } - - fData->SetCompression(value.unsignedInt); - return B_OK; - } - - case B_HPKG_ATTRIBUTE_ID_DATA_CHUNK_SIZE: - fData->SetChunkSize(value.unsignedInt); - return B_OK; - } - - return AttributeHandler::HandleAttribute(context, id, value, _handler); - } - -private: - BPackageData* fData; -}; +static status_t +set_package_data_from_attribute_value(const BPackageAttributeValue& value, + BPackageData& data) +{ + if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) + data.SetData(value.data.size, value.data.raw); + else + data.SetData(value.data.size, value.data.offset); + return B_OK; +} // #pragma mark - AttributeAttributeHandler @@ -140,12 +77,8 @@ struct PackageReaderImpl::AttributeAttributeHandler : AttributeHandler { { switch (id) { case B_HPKG_ATTRIBUTE_ID_DATA: - if (_handler != NULL) { - return DataAttributeHandler::Create(context, - &fAttribute.Data(), value, *_handler); - } - return DataAttributeHandler::InitData(context, - &fAttribute.Data(), value); + return set_package_data_from_attribute_value(value, + fAttribute.Data()); case B_HPKG_ATTRIBUTE_ID_FILE_ATTRIBUTE_TYPE: fAttribute.SetType(value.unsignedInt); @@ -279,12 +212,8 @@ struct PackageReaderImpl::EntryAttributeHandler : AttributeHandler { } case B_HPKG_ATTRIBUTE_ID_DATA: - if (_handler != NULL) { - return DataAttributeHandler::Create(context, &fEntry.Data(), - value, *_handler); - } - return DataAttributeHandler::InitData(context, &fEntry.Data(), - value); + return set_package_data_from_attribute_value(value, + fEntry.Data()); case B_HPKG_ATTRIBUTE_ID_SYMLINK_PATH: fEntry.SetSymlinkPath(value.string); @@ -378,7 +307,7 @@ struct PackageReaderImpl::RootAttributeHandler : PackageAttributeHandler { PackageReaderImpl::PackageReaderImpl(BErrorOutput* errorOutput) : - inherited(errorOutput), + inherited("package", errorOutput), fTOCSection("TOC") { } @@ -407,179 +336,39 @@ PackageReaderImpl::Init(const char* fileName) status_t PackageReaderImpl::Init(int fd, bool keepFD) { - status_t error = inherited::Init(fd, keepFD); - if (error != B_OK) - return error; - - // stat it - struct stat st; - if (fstat(FD(), &st) < 0) { - ErrorOutput()->PrintError("Error: Failed to access package file: %s\n", - strerror(errno)); - return errno; - } - - // read the header hpkg_header header; - if ((error = ReadBuffer(0, &header, sizeof(header))) != B_OK) + status_t error = inherited::Init( + fd, keepFD, header); + if (error != B_OK) return error; + fHeapSize = HeapReader()->UncompressedHeapSize(); - // check the header - - // magic - if (B_BENDIAN_TO_HOST_INT32(header.magic) != B_HPKG_MAGIC) { - ErrorOutput()->PrintError("Error: Invalid package file: Invalid " - "magic\n"); - return B_BAD_DATA; - } - - // version - if (B_BENDIAN_TO_HOST_INT16(header.version) != B_HPKG_VERSION) { - ErrorOutput()->PrintError("Error: Invalid/unsupported package file " - "version (%d)\n", B_BENDIAN_TO_HOST_INT16(header.version)); - return B_MISMATCHED_VALUES; - } - - // header size - fHeapOffset = B_BENDIAN_TO_HOST_INT16(header.header_size); - if ((size_t)fHeapOffset < sizeof(hpkg_header)) { - ErrorOutput()->PrintError("Error: Invalid package file: Invalid header " - "size (%llu)\n", fHeapOffset); - return B_BAD_DATA; - } - - // total size - fTotalSize = B_BENDIAN_TO_HOST_INT64(header.total_size); - if (fTotalSize != (uint64)st.st_size) { - ErrorOutput()->PrintError("Error: Invalid package file: Total size in " - "header (%llu) doesn't agree with total file size (%lld)\n", - fTotalSize, st.st_size); - return B_BAD_DATA; - } - - // package attributes length and compression - fPackageAttributesSection.compression - = B_BENDIAN_TO_HOST_INT32(header.attributes_compression); - fPackageAttributesSection.compressedLength - = B_BENDIAN_TO_HOST_INT32(header.attributes_length_compressed); - fPackageAttributesSection.uncompressedLength - = B_BENDIAN_TO_HOST_INT32(header.attributes_length_uncompressed); - fPackageAttributesSection.stringsLength - = B_BENDIAN_TO_HOST_INT32(header.attributes_strings_length); - fPackageAttributesSection.stringsCount - = B_BENDIAN_TO_HOST_INT32(header.attributes_strings_count); - - if (const char* errorString = CheckCompression( - fPackageAttributesSection)) { - ErrorOutput()->PrintError("Error: Invalid package file: package " - "attributes section: %s\n", errorString); - return B_BAD_DATA; - } - - // TOC length and compression - fTOCSection.compression = B_BENDIAN_TO_HOST_INT32(header.toc_compression); - fTOCSection.compressedLength - = B_BENDIAN_TO_HOST_INT64(header.toc_length_compressed); - fTOCSection.uncompressedLength - = B_BENDIAN_TO_HOST_INT64(header.toc_length_uncompressed); - - if (const char* errorString = CheckCompression(fTOCSection)) { - ErrorOutput()->PrintError("Error: Invalid package file: TOC section: " - "%s\n", errorString); - return B_BAD_DATA; - } - - // TOC subsections - fTOCSection.stringsLength - = B_BENDIAN_TO_HOST_INT64(header.toc_strings_length); - fTOCSection.stringsCount - = B_BENDIAN_TO_HOST_INT64(header.toc_strings_count); - - if (fTOCSection.stringsLength > fTOCSection.uncompressedLength - || fTOCSection.stringsCount > fTOCSection.stringsLength) { - ErrorOutput()->PrintError("Error: Invalid package file: Invalid TOC " - "subsections description\n"); - return B_BAD_DATA; - } - - // check whether the sections fit together - if (fPackageAttributesSection.compressedLength > fTotalSize - || fTOCSection.compressedLength - > fTotalSize - fPackageAttributesSection.compressedLength - || fHeapOffset - > fTotalSize - fPackageAttributesSection.compressedLength - - fTOCSection.compressedLength) { - ErrorOutput()->PrintError("Error: Invalid package file: The sum of the " - "sections sizes is greater than the package size\n"); - return B_BAD_DATA; - } - - fPackageAttributesSection.offset - = fTotalSize - fPackageAttributesSection.compressedLength; - fTOCSection.offset = fPackageAttributesSection.offset - - fTOCSection.compressedLength; - fHeapSize = fTOCSection.offset - fHeapOffset; - - // TOC size sanity check - if (fTOCSection.uncompressedLength > kMaxTOCSize) { - ErrorOutput()->PrintError("Error: Package file TOC section size " - "is %llu bytes. This is beyond the reader's sanity limit\n", - fTOCSection.uncompressedLength); - return B_UNSUPPORTED; - } - - // package attributes size sanity check - if (fPackageAttributesSection.uncompressedLength - > kMaxPackageAttributesSize) { - ErrorOutput()->PrintError( - "Error: Package file package attributes section size " - "is %llu bytes. This is beyond the reader's sanity limit\n", - fPackageAttributesSection.uncompressedLength); - return B_UNSUPPORTED; - } - - // read in the complete TOC - fTOCSection.data - = new(std::nothrow) uint8[fTOCSection.uncompressedLength]; - if (fTOCSection.data == NULL) { - ErrorOutput()->PrintError("Error: Out of memory!\n"); - return B_NO_MEMORY; - } - error = ReadCompressedBuffer(fTOCSection); + // init package attributes section + error = InitSection(fPackageAttributesSection, fHeapSize, + B_BENDIAN_TO_HOST_INT32(header.attributes_length), + kMaxPackageAttributesSize, + B_BENDIAN_TO_HOST_INT32(header.attributes_strings_length), + B_BENDIAN_TO_HOST_INT32(header.attributes_strings_count)); if (error != B_OK) return error; - // read in the complete package attributes section - fPackageAttributesSection.data - = new(std::nothrow) uint8[fPackageAttributesSection.uncompressedLength]; - if (fPackageAttributesSection.data == NULL) { - ErrorOutput()->PrintError("Error: Out of memory!\n"); - return B_NO_MEMORY; - } - error = ReadCompressedBuffer(fPackageAttributesSection); + // init TOC section + error = InitSection(fTOCSection, fPackageAttributesSection.offset, + B_BENDIAN_TO_HOST_INT64(header.toc_length), kMaxTOCSize, + B_BENDIAN_TO_HOST_INT64(header.toc_strings_length), + B_BENDIAN_TO_HOST_INT64(header.toc_strings_count)); if (error != B_OK) return error; - // start parsing the TOC - fTOCSection.currentOffset = 0; - SetCurrentSection(&fTOCSection); - - // strings - error = ParseStrings(); + // prepare the sections for use + error = PrepareSection(fTOCSection); if (error != B_OK) return error; - // parse strings from package attributes section - fPackageAttributesSection.currentOffset = 0; - SetCurrentSection(&fPackageAttributesSection); - - // strings - error = ParseStrings(); + error = PrepareSection(fPackageAttributesSection); if (error != B_OK) return error; - SetCurrentSection(NULL); - return B_OK; } @@ -630,10 +419,6 @@ PackageReaderImpl::_ParseTOC(AttributeHandlerContext* context, fTOCSection.currentOffset = fTOCSection.stringsLength; SetCurrentSection(&fTOCSection); - // prepare attribute handler context - context->heapOffset = fHeapOffset; - context->heapSize = fHeapSize; - // init the attribute handler stack rootAttributeHandler->SetLevel(0); ClearAttributeHandlerStack(); @@ -688,7 +473,7 @@ PackageReaderImpl::ReadAttributeValue(uint8 type, uint8 encoding, return B_BAD_DATA; } - _value.SetToData(size, fHeapOffset + offset); + _value.SetToData(size, offset); } else if (encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) { if (size > B_HPKG_MAX_INLINE_DATA_SIZE) { ErrorOutput()->PrintError("Error: Invalid %s section: " diff --git a/src/kits/package/hpkg/PackageWriterImpl.cpp b/src/kits/package/hpkg/PackageWriterImpl.cpp index 3778cd36e9..fc407ea7fc 100644 --- a/src/kits/package/hpkg/PackageWriterImpl.cpp +++ b/src/kits/package/hpkg/PackageWriterImpl.cpp @@ -39,6 +39,7 @@ #include #include +#include #include #include @@ -156,12 +157,11 @@ struct PackageWriterImpl::Attribute struct PackageWriterImpl::PackageContentHandler : BLowLevelPackageContentHandler { PackageContentHandler(Attribute* rootAttribute, BErrorOutput* errorOutput, - StringCache& stringCache, uint64 heapOffset) + StringCache& stringCache) : fErrorOutput(errorOutput), fStringCache(stringCache), fRootAttribute(rootAttribute), - fHeapOffset(heapOffset), fErrorOccurred(false) { } @@ -212,7 +212,7 @@ struct PackageWriterImpl::PackageContentHandler case B_HPKG_ATTRIBUTE_TYPE_RAW: if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_HEAP) { attribute->value.SetToData(value.data.size, - value.data.offset - fHeapOffset); + value.data.offset); } else if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) { attribute->value.SetToData(value.data.size, value.data.raw); @@ -249,7 +249,6 @@ private: BErrorOutput* fErrorOutput; StringCache& fStringCache; Attribute* fRootAttribute; - uint64 fHeapOffset; bool fErrorOccurred; }; @@ -394,8 +393,8 @@ private: struct PackageWriterImpl::HeapAttributeOffsetter { - HeapAttributeOffsetter(const RangeArray& ranges, - const Array& deltas) + HeapAttributeOffsetter(const RangeArray& ranges, + const Array& deltas) : fRanges(ranges), fDeltas(deltas) @@ -409,7 +408,7 @@ struct PackageWriterImpl::HeapAttributeOffsetter { if (value.type == B_HPKG_ATTRIBUTE_TYPE_RAW && value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_HEAP) { - off_t delta = fDeltas[fRanges.InsertionIndex(value.data.offset)]; + uint64 delta = fDeltas[fRanges.InsertionIndex(value.data.offset)]; value.data.offset -= delta; } @@ -422,8 +421,8 @@ struct PackageWriterImpl::HeapAttributeOffsetter { } private: - const RangeArray& fRanges; - const Array& fDeltas; + const RangeArray& fRanges; + const Array& fDeltas; }; @@ -445,11 +444,9 @@ PackageWriterImpl::_AddAttribute(BHPKGAttributeID attributeID, Type value) PackageWriterImpl::PackageWriterImpl(BPackageWriterListener* listener) : - inherited(listener), + inherited("package", listener), fListener(listener), fHeapRangesToRemove(NULL), - fDataBuffer(NULL), - fDataBufferSize(2 * B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB), fRootEntry(NULL), fRootAttribute(NULL), fTopAttribute(NULL), @@ -461,10 +458,7 @@ PackageWriterImpl::PackageWriterImpl(BPackageWriterListener* listener) PackageWriterImpl::~PackageWriterImpl() { delete fRootAttribute; - delete fRootEntry; - - free(fDataBuffer); } @@ -566,7 +560,7 @@ status_t PackageWriterImpl::Finish() { try { - RangeArray heapRangesToRemove; + RangeArray heapRangesToRemove; fHeapRangesToRemove = &heapRangesToRemove; if ((Flags() & B_HPKG_WRITER_UPDATE_PACKAGE) != 0) { @@ -610,15 +604,10 @@ PackageWriterImpl::Finish() status_t PackageWriterImpl::_Init(const char* fileName, uint32 flags) { - status_t result = inherited::Init(fileName, "package", flags); + status_t result = inherited::Init(fileName, flags); if (result != B_OK) return result; - // allocate data buffer - fDataBuffer = malloc(fDataBufferSize); - if (fDataBuffer == NULL) - throw std::bad_alloc(); - if (fStringCache.Init() != B_OK) throw std::bad_alloc(); @@ -627,7 +616,7 @@ PackageWriterImpl::_Init(const char* fileName, uint32 flags) fRootAttribute = new Attribute(); - fHeapOffset = fHeapEnd = sizeof(hpkg_header); + fHeapOffset = fHeaderSize = sizeof(hpkg_header); fTopAttribute = fRootAttribute; // in update mode, parse the TOC @@ -638,19 +627,14 @@ PackageWriterImpl::_Init(const char* fileName, uint32 flags) return result; fHeapOffset = packageReader.HeapOffset(); - fHeapEnd = fHeapOffset + packageReader.HeapSize(); - PackageContentHandler handler(fRootAttribute, fListener, fStringCache, - fHeapOffset); + PackageContentHandler handler(fRootAttribute, fListener, fStringCache); result = packageReader.ParseContent(&handler); if (result != B_OK) return result; - if ((uint64)fHeapOffset > packageReader.HeapOffset()) { - fListener->PrintError("Unexpected heap offset in package file.\n"); - return B_BAD_DATA; - } + fHeapWriter->Reinit(packageReader.HeapReader()); } return B_OK; @@ -815,99 +799,22 @@ PackageWriterImpl::_UpdateReadPackageInfo() if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) data.SetData(value.data.size, value.data.raw); else - data.SetData(value.data.size, value.data.offset + fHeapOffset); - - // get the compression - uint8 compression = B_HPKG_DEFAULT_DATA_COMPRESSION; - if (Attribute* compressionAttribute = dataAttribute->ChildWithID( - B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION)) { - if (compressionAttribute->value.type != B_HPKG_ATTRIBUTE_TYPE_UINT) { - fListener->PrintError("%s entry in package file has an invalid " - "data compression attribute (not of type uint).\n", - B_HPKG_PACKAGE_INFO_FILE_NAME); - throw status_t(B_BAD_DATA); - } - compression = compressionAttribute->value.unsignedInt; - } - - data.SetCompression(compression); - - // get the size - uint64 size; - Attribute* sizeAttribute = dataAttribute->ChildWithID( - B_HPKG_ATTRIBUTE_ID_DATA_SIZE); - if (sizeAttribute == NULL) { - size = value.data.size; - } else if (sizeAttribute->value.type != B_HPKG_ATTRIBUTE_TYPE_UINT) { - fListener->PrintError("%s entry in package file has an invalid data " - "size attribute (not of type uint).\n", - B_HPKG_PACKAGE_INFO_FILE_NAME); - throw status_t(B_BAD_DATA); - } else - size = sizeAttribute->value.unsignedInt; - - data.SetUncompressedSize(size); - - // get the chunk size - uint64 chunkSize = compression == B_HPKG_COMPRESSION_ZLIB - ? B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB : 0; - if (Attribute* chunkSizeAttribute = dataAttribute->ChildWithID( - B_HPKG_ATTRIBUTE_ID_DATA_CHUNK_SIZE)) { - if (chunkSizeAttribute->value.type != B_HPKG_ATTRIBUTE_TYPE_UINT) { - fListener->PrintError("%s entry in package file has an invalid " - "data chunk size attribute (not of type uint).\n", - B_HPKG_PACKAGE_INFO_FILE_NAME); - throw status_t(B_BAD_DATA); - } - chunkSize = chunkSizeAttribute->value.unsignedInt; - } - - data.SetChunkSize(chunkSize); + data.SetData(value.data.size, value.data.offset); // read the value into a string BString valueString; - char* valueBuffer = valueString.LockBuffer(size); + char* valueBuffer = valueString.LockBuffer(value.data.size); if (valueBuffer == NULL) throw std::bad_alloc(); if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) { // data encoded inline -- just copy to buffer - if (size != value.data.size) { - fListener->PrintError("%s entry in package file has an invalid " - "data attribute (mismatching size).\n", - B_HPKG_PACKAGE_INFO_FILE_NAME); - throw status_t(B_BAD_DATA); - } memcpy(valueBuffer, value.data.raw, value.data.size); } else { // data on heap -- read from there - BBlockBufferPoolNoLock bufferCache(16 * 1024, 1); - status_t error = bufferCache.Init(); - if (error != B_OK) { - fListener->PrintError("Failed to initialize buffer cache: %s\n", - strerror(error)); - throw status_t(error); - } - - // create a PackageDataReader - BFDDataReader packageFileReader(FD()); - BAbstractBufferedDataReader* reader; - error = BPackageDataReaderFactory(&bufferCache) - .CreatePackageDataReader(&packageFileReader, data, reader); - if (error != B_OK) { - fListener->PrintError("Failed to create package data reader: %s\n", - strerror(error)); - throw status_t(error); - } - ObjectDeleter readerDeleter(reader); - - // read the data - error = reader->ReadData(0, valueBuffer, size); - if (error != B_OK) { - fListener->PrintError("Failed to read data of %s entry in package " - "file: %s\n", B_HPKG_PACKAGE_INFO_FILE_NAME, strerror(error)); - throw status_t(error); - } + status_t error = fHeapWriter->ReadData(data.Offset(), valueBuffer, + data.Size()); + throw error; } valueString.UnlockBuffer(); @@ -1101,8 +1008,8 @@ PackageWriterImpl::_CompactHeap() return; // compute the move deltas for the ranges - Array deltas; - off_t delta = 0; + Array deltas; + uint64 delta = 0; for (int32 i = 0; i < count; i++) { if (!deltas.Add(delta)) throw std::bad_alloc(); @@ -1117,73 +1024,8 @@ PackageWriterImpl::_CompactHeap() HeapAttributeOffsetter(*fHeapRangesToRemove, deltas).ProcessAttribute( fRootAttribute); - // move the heap chunks in the file around - off_t chunkOffset = fHeapOffset; - delta = 0; - - for (int32 i = 0; i < count; i++) { - const Range& range = fHeapRangesToRemove->RangeAt(i); - - if (delta > 0 && chunkOffset < range.offset) { - // move chunk - _MoveHeapChunk(chunkOffset, chunkOffset - delta, - range.offset - chunkOffset); - } - - chunkOffset = range.EndOffset(); - delta += range.size; - } - - // move the final chunk - off_t heapSize = fHeapEnd - fHeapOffset; - if (delta > 0 && chunkOffset < heapSize) { - _MoveHeapChunk(chunkOffset, chunkOffset - delta, - heapSize - chunkOffset); - } - - fHeapEnd -= delta; -} - - -void -PackageWriterImpl::_MoveHeapChunk(off_t fromOffset, off_t toOffset, off_t size) -{ - // convert heap offsets to file offsets - fromOffset += fHeapOffset; - toOffset += fHeapOffset; - - while (size > 0) { - size_t toCopy = std::min(size, (off_t)fDataBufferSize); - - // read data into buffer - ssize_t bytesRead = read_pos(FD(), fromOffset, fDataBuffer, toCopy); - if (bytesRead < 0) { - fListener->PrintError("Failed to read from package file: %s\n", - strerror(errno)); - throw status_t(errno); - } - if ((size_t)bytesRead < toCopy) { - fListener->PrintError("Failed to read from package file (wanted " - "%zu bytes, got %zd).\n", toCopy, bytesRead); - throw status_t(B_IO_ERROR); - } - - // write data to target offset - ssize_t bytesWritten = write_pos(FD(), toOffset, fDataBuffer, toCopy); - if (bytesWritten < 0) { - fListener->PrintError("Failed to write to package file: %s\n", - strerror(errno)); - throw status_t(errno); - } - if ((size_t)bytesWritten < toCopy) { - fListener->PrintError("Failed to write to package file.\n"); - throw status_t(B_IO_ERROR); - } - - fromOffset += toCopy; - toOffset += toCopy; - size -= toCopy; - } + // remove the ranges from the heap + fHeapWriter->RemoveDataRanges(*fHeapRangesToRemove); } @@ -1216,41 +1058,52 @@ PackageWriterImpl::_Finish() _AddEntry(AT_FDCWD, entry, entry->Name(), pathBuffer); } - off_t heapSize = fHeapEnd - fHeapOffset; - hpkg_header header; // write the TOC and package attributes - _WriteTOC(header); - _WritePackageAttributes(header); + uint64 tocLength; + _WriteTOC(header, tocLength); - off_t totalSize = fHeapEnd; + uint64 attributesLength; + _WritePackageAttributes(header, attributesLength); + + // flush the heap + status_t error = fHeapWriter->Finish(); + if (error != B_OK) + return error; + + uint64 compressedHeapSize = fHeapWriter->CompressedHeapSize(); + + header.heap_compression = B_HOST_TO_BENDIAN_INT32(B_HPKG_COMPRESSION_ZLIB); + header.heap_chunk_size = B_HOST_TO_BENDIAN_INT32(fHeapWriter->ChunkSize()); + header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64( + fHeapWriter->CompressedHeapSize()); + header.heap_size_uncompressed = B_HOST_TO_BENDIAN_INT64( + fHeapWriter->UncompressedHeapSize()); // Truncate the file to the size it is supposed to have. In update mode, it - // can be greater when one or more files are shrunk. In creation mode, when - // writing compressed TOC or package attributes yields a larger size than - // uncompressed, the file size may also be greater than it should be. + // can be greater when one or more files are shrunk. In creation mode it + // should already have the correct size. + off_t totalSize = fHeapWriter->HeapOffset() + (off_t)compressedHeapSize; if (ftruncate(FD(), totalSize) != 0) { fListener->PrintError("Failed to truncate package file to new " "size: %s\n", strerror(errno)); return errno; } - fListener->OnPackageSizeInfo(fHeapOffset, heapSize, - B_BENDIAN_TO_HOST_INT64(header.toc_length_compressed), - B_BENDIAN_TO_HOST_INT32(header.attributes_length_compressed), - totalSize); + fListener->OnPackageSizeInfo(fHeaderSize, compressedHeapSize, tocLength, + attributesLength, totalSize); // prepare the header // general header.magic = B_HOST_TO_BENDIAN_INT32(B_HPKG_MAGIC); - header.header_size = B_HOST_TO_BENDIAN_INT16((uint16)fHeapOffset); + header.header_size = B_HOST_TO_BENDIAN_INT16(fHeaderSize); header.version = B_HOST_TO_BENDIAN_INT16(B_HPKG_VERSION); header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize); // write the header - WriteBuffer(&header, sizeof(hpkg_header), 0); + RawWriteBuffer(&header, sizeof(hpkg_header), 0); SetFinished(true); return B_OK; @@ -1332,107 +1185,32 @@ PackageWriterImpl::_RegisterEntry(Entry* parent, const char* name, void -PackageWriterImpl::_WriteTOC(hpkg_header& header) +PackageWriterImpl::_WriteTOC(hpkg_header& header, uint64& _length) { - // prepare the writer (zlib writer on top of a file writer) - off_t startOffset = fHeapEnd; + // write the subsections + uint64 startOffset = fHeapWriter->UncompressedHeapSize(); - // write the sections - uint32 compression = B_HPKG_COMPRESSION_ZLIB; - uint64 uncompressedStringsSize; - uint64 uncompressedMainSize; - uint64 tocUncompressedSize; - int32 cachedStringsWritten = _WriteTOCCompressed(uncompressedStringsSize, - uncompressedMainSize, tocUncompressedSize); - - off_t endOffset = fHeapEnd; - - if (endOffset - startOffset >= (off_t)tocUncompressedSize) { - // the compressed section isn't shorter -- write uncompressed - fHeapEnd = startOffset; - compression = B_HPKG_COMPRESSION_NONE; - cachedStringsWritten = _WriteTOCUncompressed(uncompressedStringsSize, - uncompressedMainSize, tocUncompressedSize); - - endOffset = fHeapEnd; - } - - fListener->OnTOCSizeInfo(uncompressedStringsSize, uncompressedMainSize, - tocUncompressedSize); - - // update the header - - // TOC - header.toc_compression = B_HOST_TO_BENDIAN_INT32(compression); - header.toc_length_compressed = B_HOST_TO_BENDIAN_INT64( - endOffset - startOffset); - header.toc_length_uncompressed = B_HOST_TO_BENDIAN_INT64( - tocUncompressedSize); - - // TOC subsections - header.toc_strings_length = B_HOST_TO_BENDIAN_INT64( - uncompressedStringsSize); - header.toc_strings_count = B_HOST_TO_BENDIAN_INT64(cachedStringsWritten); -} - - -int32 -PackageWriterImpl::_WriteTOCCompressed(uint64& _uncompressedStringsSize, - uint64& _uncompressedMainSize, uint64& _tocUncompressedSize) -{ - FDDataWriter realWriter(FD(), fHeapEnd, fListener); - ZlibDataWriter zlibWriter(&realWriter); - SetDataWriter(&zlibWriter); - zlibWriter.Init(); - - // write the sections - int32 cachedStringsWritten - = _WriteTOCSections(_uncompressedStringsSize, _uncompressedMainSize); - - // finish the writer - zlibWriter.Finish(); - fHeapEnd = realWriter.Offset(); - SetDataWriter(NULL); - - _tocUncompressedSize = zlibWriter.BytesWritten(); - return cachedStringsWritten; -} - - -int32 -PackageWriterImpl::_WriteTOCUncompressed(uint64& _uncompressedStringsSize, - uint64& _uncompressedMainSize, uint64& _tocUncompressedSize) -{ - FDDataWriter realWriter(FD(), fHeapEnd, fListener); - SetDataWriter(&realWriter); - - // write the sections - int32 cachedStringsWritten - = _WriteTOCSections(_uncompressedStringsSize, _uncompressedMainSize); - - fHeapEnd = realWriter.Offset(); - SetDataWriter(NULL); - - _tocUncompressedSize = realWriter.BytesWritten(); - return cachedStringsWritten; -} - - -int32 -PackageWriterImpl::_WriteTOCSections(uint64& _stringsSize, uint64& _mainSize) -{ - // write the cached strings - uint64 cachedStringsOffset = DataWriter()->BytesWritten(); + // cached strings + uint64 cachedStringsOffset = fHeapWriter->UncompressedHeapSize(); int32 cachedStringsWritten = WriteCachedStrings(fStringCache, 2); - // write the main TOC section - uint64 mainOffset = DataWriter()->BytesWritten(); + // main TOC section + uint64 mainOffset = fHeapWriter->UncompressedHeapSize(); _WriteAttributeChildren(fRootAttribute); - _stringsSize = mainOffset - cachedStringsOffset; - _mainSize = DataWriter()->BytesWritten() - mainOffset; + // notify the listener + uint64 endOffset = fHeapWriter->UncompressedHeapSize(); + uint64 stringsSize = mainOffset - cachedStringsOffset; + uint64 mainSize = endOffset - mainOffset; + uint64 tocSize = endOffset - startOffset; + fListener->OnTOCSizeInfo(stringsSize, mainSize, tocSize); - return cachedStringsWritten; + // update the header + header.toc_length = B_HOST_TO_BENDIAN_INT64(tocSize); + header.toc_strings_length = B_HOST_TO_BENDIAN_INT64(stringsSize); + header.toc_strings_count = B_HOST_TO_BENDIAN_INT64(cachedStringsWritten); + + _length = tocSize; } @@ -1459,85 +1237,25 @@ PackageWriterImpl::_WriteAttributeChildren(Attribute* attribute) void -PackageWriterImpl::_WritePackageAttributes(hpkg_header& header) +PackageWriterImpl::_WritePackageAttributes(hpkg_header& header, uint64& _length) { - // write the package attributes (zlib writer on top of a file writer) - off_t startOffset = fHeapEnd; + // write cached strings and package attributes tree + off_t startOffset = fHeapWriter->UncompressedHeapSize(); - uint32 compression = B_HPKG_COMPRESSION_ZLIB; - uint32 stringsLengthUncompressed; - uint32 attributesLengthUncompressed; - uint32 stringsCount = _WritePackageAttributesCompressed( - stringsLengthUncompressed, attributesLengthUncompressed); + uint32 stringsLength; + uint32 stringsCount = WritePackageAttributes(PackageAttributes(), + stringsLength); - off_t endOffset = fHeapEnd; - - if ((off_t)attributesLengthUncompressed <= endOffset - startOffset) { - // the compressed section isn't shorter -- write uncompressed - fHeapEnd = startOffset; - compression = B_HPKG_COMPRESSION_NONE; - stringsCount = _WritePackageAttributesUncompressed( - stringsLengthUncompressed, attributesLengthUncompressed); - - endOffset = fHeapEnd; - } - - fListener->OnPackageAttributesSizeInfo(stringsCount, - attributesLengthUncompressed); + // notify listener + uint32 attributesLength = fHeapWriter->UncompressedHeapSize() - startOffset; + fListener->OnPackageAttributesSizeInfo(stringsCount, attributesLength); // update the header - header.attributes_compression = B_HOST_TO_BENDIAN_INT32(compression); - header.attributes_length_compressed - = B_HOST_TO_BENDIAN_INT32(endOffset - startOffset); - header.attributes_length_uncompressed - = B_HOST_TO_BENDIAN_INT32(attributesLengthUncompressed); + header.attributes_length = B_HOST_TO_BENDIAN_INT32(attributesLength); header.attributes_strings_count = B_HOST_TO_BENDIAN_INT32(stringsCount); - header.attributes_strings_length - = B_HOST_TO_BENDIAN_INT32(stringsLengthUncompressed); -} + header.attributes_strings_length = B_HOST_TO_BENDIAN_INT32(stringsLength); - -uint32 -PackageWriterImpl::_WritePackageAttributesCompressed( - uint32& _stringsLengthUncompressed, uint32& _attributesLengthUncompressed) -{ - off_t startOffset = fHeapEnd; - FDDataWriter realWriter(FD(), startOffset, fListener); - ZlibDataWriter zlibWriter(&realWriter); - SetDataWriter(&zlibWriter); - zlibWriter.Init(); - - // write cached strings and package attributes tree - uint32 stringsCount = WritePackageAttributes(PackageAttributes(), - _stringsLengthUncompressed); - - zlibWriter.Finish(); - fHeapEnd = realWriter.Offset(); - SetDataWriter(NULL); - - _attributesLengthUncompressed = zlibWriter.BytesWritten(); - return stringsCount; -} - - -uint32 -PackageWriterImpl::_WritePackageAttributesUncompressed( - uint32& _stringsLengthUncompressed, uint32& _attributesLengthUncompressed) -{ - off_t startOffset = fHeapEnd; - FDDataWriter realWriter(FD(), startOffset, fListener); - - SetDataWriter(&realWriter); - - // write cached strings and package attributes tree - uint32 stringsCount = WritePackageAttributes(PackageAttributes(), - _stringsLengthUncompressed); - - fHeapEnd = realWriter.Offset(); - SetDataWriter(NULL); - - _attributesLengthUncompressed = realWriter.BytesWritten(); - return stringsCount; + _length = attributesLength; } @@ -1804,178 +1522,13 @@ PackageWriterImpl::_AddData(BDataReader& dataReader, off_t size) return B_OK; } - // longer data -- try to compress - uint64 dataOffset = fHeapEnd; - - uint64 compression = B_HPKG_COMPRESSION_NONE; - uint64 compressedSize; - - status_t error = _WriteZlibCompressedData(dataReader, size, dataOffset, - compressedSize); - if (error == B_OK) { - compression = B_HPKG_COMPRESSION_ZLIB; - } else { - error = _WriteUncompressedData(dataReader, size, dataOffset); - compressedSize = size; - } + // add data to heap + uint64 dataOffset; + status_t error = fHeapWriter->AddData(dataReader, size, dataOffset); if (error != B_OK) return error; - fHeapEnd = dataOffset + compressedSize; - - // add data attribute - Attribute* dataAttribute = _AddDataAttribute(B_HPKG_ATTRIBUTE_ID_DATA, - compressedSize, dataOffset - fHeapOffset); - Stacker attributeAttributeStacker(fTopAttribute, dataAttribute); - - // if compressed, add compression attributes - if (compression != B_HPKG_COMPRESSION_NONE) { - _AddAttribute(B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION, compression); - _AddAttribute(B_HPKG_ATTRIBUTE_ID_DATA_SIZE, (uint64)size); - // uncompressed size - } - - return B_OK; -} - - -status_t -PackageWriterImpl::_WriteUncompressedData(BDataReader& dataReader, off_t size, - uint64 writeOffset) -{ - // copy the data to the heap - off_t readOffset = 0; - off_t remainingSize = size; - while (remainingSize > 0) { - // read data - size_t toCopy = std::min(remainingSize, (off_t)fDataBufferSize); - status_t error = dataReader.ReadData(readOffset, fDataBuffer, toCopy); - if (error != B_OK) { - fListener->PrintError("Failed to read data: %s\n", strerror(error)); - return error; - } - - // write to heap - ssize_t bytesWritten = pwrite(FD(), fDataBuffer, toCopy, writeOffset); - if (bytesWritten < 0) { - fListener->PrintError("Failed to write data: %s\n", - strerror(errno)); - return errno; - } - if ((size_t)bytesWritten != toCopy) { - fListener->PrintError("Failed to write all data\n"); - return B_ERROR; - } - - remainingSize -= toCopy; - readOffset += toCopy; - writeOffset += toCopy; - } - - return B_OK; -} - - -status_t -PackageWriterImpl::_WriteZlibCompressedData(BDataReader& dataReader, off_t size, - uint64 writeOffset, uint64& _compressedSize) -{ - // Use zlib compression only for data large enough. - if (size < (off_t)kZlibCompressionSizeThreshold) - return B_BAD_VALUE; - - // fDataBuffer is 2 * B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, so split it into - // two halves we can use for reading and compressing - const size_t chunkSize = B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB; - uint8* inputBuffer = (uint8*)fDataBuffer; - uint8* outputBuffer = (uint8*)fDataBuffer + chunkSize; - - // account for the offset table - uint64 chunkCount = (size + (chunkSize - 1)) / chunkSize; - off_t offsetTableOffset = writeOffset; - uint64* offsetTable = NULL; - if (chunkCount > 1) { - offsetTable = new uint64[chunkCount - 1]; - writeOffset = offsetTableOffset + (chunkCount - 1) * sizeof(uint64); - } - ArrayDeleter offsetTableDeleter(offsetTable); - - const uint64 dataOffset = writeOffset; - const uint64 dataEndLimit = offsetTableOffset + size; - - // read the data, compress them and write them to the heap - off_t readOffset = 0; - off_t remainingSize = size; - uint64 chunkIndex = 0; - while (remainingSize > 0) { - // read data - size_t toCopy = std::min(remainingSize, (off_t)chunkSize); - status_t error = dataReader.ReadData(readOffset, inputBuffer, toCopy); - if (error != B_OK) { - fListener->PrintError("Failed to read data: %s\n", strerror(error)); - return error; - } - - // compress - size_t compressedSize; - error = ZlibCompressor::CompressSingleBuffer(inputBuffer, toCopy, - outputBuffer, toCopy, compressedSize); - - const void* writeBuffer; - size_t bytesToWrite; - if (error == B_OK) { - writeBuffer = outputBuffer; - bytesToWrite = compressedSize; - } else { - if (error != B_BUFFER_OVERFLOW) - return error; - writeBuffer = inputBuffer; - bytesToWrite = toCopy; - } - - // check the total compressed data size - if (writeOffset + bytesToWrite >= dataEndLimit) - return B_BUFFER_OVERFLOW; - - if (chunkIndex > 0) - offsetTable[chunkIndex - 1] = writeOffset - dataOffset; - - // write to heap - ssize_t bytesWritten = pwrite(FD(), writeBuffer, bytesToWrite, - writeOffset); - if (bytesWritten < 0) { - fListener->PrintError("Failed to write data: %s\n", - strerror(errno)); - return errno; - } - if ((size_t)bytesWritten != bytesToWrite) { - fListener->PrintError("Failed to write all data\n"); - return B_ERROR; - } - - remainingSize -= toCopy; - readOffset += toCopy; - writeOffset += bytesToWrite; - chunkIndex++; - } - - // write the offset table - if (chunkCount > 1) { - size_t bytesToWrite = (chunkCount - 1) * sizeof(uint64); - ssize_t bytesWritten = pwrite(FD(), offsetTable, bytesToWrite, - offsetTableOffset); - if (bytesWritten < 0) { - fListener->PrintError("Failed to write data: %s\n", - strerror(errno)); - return errno; - } - if ((size_t)bytesWritten != bytesToWrite) { - fListener->PrintError("Failed to write all data\n"); - return B_ERROR; - } - } - - _compressedSize = writeOffset - offsetTableOffset; + _AddDataAttribute(B_HPKG_ATTRIBUTE_ID_DATA, size, dataOffset); return B_OK; } diff --git a/src/kits/package/hpkg/ReaderImplBase.cpp b/src/kits/package/hpkg/ReaderImplBase.cpp index 5137d70213..434d6efc1a 100644 --- a/src/kits/package/hpkg/ReaderImplBase.cpp +++ b/src/kits/package/hpkg/ReaderImplBase.cpp @@ -21,6 +21,7 @@ #include #include +#include #include @@ -536,12 +537,14 @@ ReaderImplBase::LowLevelAttributeHandler::Delete( // #pragma mark - ReaderImplBase -ReaderImplBase::ReaderImplBase(BErrorOutput* errorOutput) +ReaderImplBase::ReaderImplBase(const char* fileType, BErrorOutput* errorOutput) : fPackageAttributesSection("package attributes"), + fFileType(fileType), fErrorOutput(errorOutput), fFD(-1), fOwnsFD(false), + fHeapReader(NULL), fCurrentSection(NULL), fScratchBuffer(NULL), fScratchBufferSize(0) @@ -551,6 +554,8 @@ ReaderImplBase::ReaderImplBase(BErrorOutput* errorOutput) ReaderImplBase::~ReaderImplBase() { + delete fHeapReader; + if (fOwnsFD && fFD >= 0) close(fFD); @@ -559,7 +564,7 @@ ReaderImplBase::~ReaderImplBase() status_t -ReaderImplBase::Init(int fd, bool keepFD) +ReaderImplBase::_Init(int fd, bool keepFD) { fFD = fd; fOwnsFD = keepFD; @@ -576,27 +581,84 @@ ReaderImplBase::Init(int fd, bool keepFD) } -const char* -ReaderImplBase::CheckCompression(const SectionInfo& section) const +status_t +ReaderImplBase::InitHeapReader(uint32 compression, uint32 chunkSize, + off_t offset, uint64 compressedSize, uint64 uncompressedSize) { - switch (section.compression) { - case B_HPKG_COMPRESSION_NONE: - if (section.compressedLength != section.uncompressedLength) { - return "Uncompressed, but compressed and uncompressed length " - "don't match"; - } - return NULL; - - case B_HPKG_COMPRESSION_ZLIB: - if (section.compressedLength >= section.uncompressedLength) { - return "Compressed, but compressed length is not less than " - "uncompressed length"; - } - return NULL; - - default: - return "Invalid compression algorithm ID"; + if (compression != B_HPKG_COMPRESSION_ZLIB) { + fErrorOutput->PrintError("Error: Invalid heap compression\n"); + return B_BAD_DATA; } + + fHeapReader = new(std::nothrow) PackageFileHeapReader(fErrorOutput, fFD, + offset, compressedSize, uncompressedSize); + return fHeapReader->Init(); +} + + +status_t +ReaderImplBase::InitSection(PackageFileSection& section, uint64 endOffset, + uint64 length, uint64 maxSaneLength, uint64 stringsLength, + uint64 stringsCount) +{ + // check length vs. endOffset + if (length > endOffset) { + ErrorOutput()->PrintError("Error: %s file %s section size is %" + B_PRIu64 " bytes. This is greater than the available space\n", + fFileType, section.name, length); + return B_BAD_DATA; + } + + // check sanity length + if (maxSaneLength > 0 && length > maxSaneLength) { + ErrorOutput()->PrintError("Error: %s file %s section size is %" + B_PRIu64 " bytes. This is beyond the reader's sanity limit\n", + fFileType, section.name, length); + return B_NOT_SUPPORTED; + } + + // check strings subsection size/count + if ((stringsLength == 0) != (stringsCount == 0) || stringsLength > length) { + ErrorOutput()->PrintError("Error: strings subsection description of %s " + "file %s section is invalid (%" B_PRIu64 " strings, length: %" + B_PRIu64 ")\n", + fFileType, section.name, length, stringsCount, stringsLength); + return B_BAD_DATA; + } + + section.uncompressedLength = length; + section.offset = endOffset - length; + section.currentOffset = 0; + section.stringsLength = stringsLength; + section.stringsCount = stringsCount; + + return B_OK; +} + + +status_t +ReaderImplBase::PrepareSection(PackageFileSection& section) +{ + // allocate memory for the section data and read it in + section.data = new(std::nothrow) uint8[section.uncompressedLength]; + if (section.data == NULL) { + ErrorOutput()->PrintError("Error: Out of memory!\n"); + return B_NO_MEMORY; + } + + status_t error = ReadSection(section); + if (error != B_OK) + return error; + + // parse the section strings + section.currentOffset = 0; + SetCurrentSection(§ion); + + error = ParseStrings(); + if (error != B_OK) + return error; + + return B_OK; } @@ -982,8 +1044,9 @@ ReaderImplBase::_ReadSectionBuffer(void* buffer, size_t size) { if (size > fCurrentSection->uncompressedLength - fCurrentSection->currentOffset) { - fErrorOutput->PrintError("_ReadBuffer(%lu): read beyond %s end\n", - size, fCurrentSection->name); + fErrorOutput->PrintError( + "_ReadSectionBuffer(%lu): read beyond %s end\n", size, + fCurrentSection->name); return B_BAD_DATA; } @@ -1014,63 +1077,11 @@ ReaderImplBase::ReadBuffer(off_t offset, void* buffer, size_t size) status_t -ReaderImplBase::ReadCompressedBuffer(const SectionInfo& section) +ReaderImplBase::ReadSection(const PackageFileSection& section) { - uint32 compressedSize = section.compressedLength; - uint64 offset = section.offset; - - switch (section.compression) { - case B_HPKG_COMPRESSION_NONE: - return ReadBuffer(offset, section.data, compressedSize); - - case B_HPKG_COMPRESSION_ZLIB: - { - // init the decompressor - BBufferDataOutput bufferOutput(section.data, - section.uncompressedLength); - ZlibDecompressor decompressor(&bufferOutput); - status_t error = decompressor.Init(); - if (error != B_OK) - return error; - - while (compressedSize > 0) { - // read compressed buffer - size_t toRead = std::min((size_t)compressedSize, - fScratchBufferSize); - error = ReadBuffer(offset, fScratchBuffer, toRead); - if (error != B_OK) - return error; - - // uncompress - error = decompressor.DecompressNext(fScratchBuffer, toRead); - if (error != B_OK) - return error; - - compressedSize -= toRead; - offset += toRead; - } - - error = decompressor.Finish(); - if (error != B_OK) - return error; - - // verify that all data have been read - if (bufferOutput.BytesWritten() != section.uncompressedLength) { - fErrorOutput->PrintError("Error: Missing bytes in uncompressed " - "buffer!\n"); - return B_BAD_DATA; - } - - return B_OK; - } - - default: - { - fErrorOutput->PrintError("Error: Invalid compression type: %u\n", - section.compression); - return B_BAD_DATA; - } - } + BBufferDataOutput output(section.data, section.uncompressedLength); + return fHeapReader->ReadDataToOutput(section.offset, + section.uncompressedLength, &output); } diff --git a/src/kits/package/hpkg/RepositoryReaderImpl.cpp b/src/kits/package/hpkg/RepositoryReaderImpl.cpp index 5f315a4000..13f78a1471 100644 --- a/src/kits/package/hpkg/RepositoryReaderImpl.cpp +++ b/src/kits/package/hpkg/RepositoryReaderImpl.cpp @@ -20,6 +20,7 @@ #include #include +#include #include @@ -43,8 +44,7 @@ static const size_t kMaxPackageAttributesSize = 64 * 1024 * 1024; RepositoryReaderImpl::RepositoryReaderImpl(BErrorOutput* errorOutput) : - inherited(errorOutput), - fRepositoryInfoSection("repository info") + inherited("repository", errorOutput) { } @@ -73,146 +73,43 @@ RepositoryReaderImpl::Init(const char* fileName) status_t RepositoryReaderImpl::Init(int fd, bool keepFD) { - status_t error = inherited::Init(fd, keepFD); - if (error != B_OK) - return error; - - // stat it - struct stat st; - if (fstat(FD(), &st) < 0) { - ErrorOutput()->PrintError( - "Error: Failed to access repository file: %s\n", strerror(errno)); - return errno; - } - - // read the header hpkg_repo_header header; - if ((error = ReadBuffer(0, &header, sizeof(header))) != B_OK) - return error; - - // check the header - - // magic - if (B_BENDIAN_TO_HOST_INT32(header.magic) != B_HPKG_REPO_MAGIC) { - ErrorOutput()->PrintError("Error: Invalid repository file: Invalid " - "magic\n"); - return B_BAD_DATA; - } - - // header size - size_t headerSize = B_BENDIAN_TO_HOST_INT16(header.header_size); - if (headerSize < sizeof(hpkg_repo_header)) { - ErrorOutput()->PrintError("Error: Invalid repository file: Invalid " - "header size (%llu)\n", headerSize); - return B_BAD_DATA; - } - - // version - if (B_BENDIAN_TO_HOST_INT16(header.version) != B_HPKG_REPO_VERSION) { - ErrorOutput()->PrintError("Error: Invalid/unsupported repository file " - "version (%d)\n", B_BENDIAN_TO_HOST_INT16(header.version)); - return B_BAD_DATA; - } - - // total size - uint64 totalSize = B_BENDIAN_TO_HOST_INT64(header.total_size); - if (totalSize != (uint64)st.st_size) { - ErrorOutput()->PrintError("Error: Invalid repository file: Total size " - "in header (%llu) doesn't agree with total file size (%lld)\n", - totalSize, st.st_size); - return B_BAD_DATA; - } - - // repository info length and compression - fRepositoryInfoSection.compression - = B_BENDIAN_TO_HOST_INT32(header.info_compression); - fRepositoryInfoSection.compressedLength - = B_BENDIAN_TO_HOST_INT32(header.info_length_compressed); - fRepositoryInfoSection.uncompressedLength - = B_BENDIAN_TO_HOST_INT32(header.info_length_uncompressed); - - if (const char* errorString = CheckCompression(fRepositoryInfoSection)) { - ErrorOutput()->PrintError( - "Error: Invalid repository file: info section: %s\n", errorString); - return B_BAD_DATA; - } - - // package attributes length and compression - fPackageAttributesSection.compression - = B_BENDIAN_TO_HOST_INT32(header.packages_compression); - fPackageAttributesSection.compressedLength - = B_BENDIAN_TO_HOST_INT64(header.packages_length_compressed); - fPackageAttributesSection.uncompressedLength - = B_BENDIAN_TO_HOST_INT64(header.packages_length_uncompressed); - fPackageAttributesSection.stringsLength - = B_BENDIAN_TO_HOST_INT64(header.packages_strings_length); - fPackageAttributesSection.stringsCount - = B_BENDIAN_TO_HOST_INT64(header.packages_strings_count); - - if (const char* errorString = CheckCompression( - fPackageAttributesSection)) { - ErrorOutput()->PrintError("Error: Invalid repository file: package " - "attributes section: %s\n", errorString); - return B_BAD_DATA; - } - - // check whether the sections fit together - if (fPackageAttributesSection.compressedLength > totalSize - || fRepositoryInfoSection.compressedLength - > totalSize - fPackageAttributesSection.compressedLength) { - ErrorOutput()->PrintError("Error: Invalid repository file: The sum of " - "the sections sizes is greater than the repository size\n"); - return B_BAD_DATA; - } - - fPackageAttributesSection.offset - = totalSize - fPackageAttributesSection.compressedLength; - fRepositoryInfoSection.offset = fPackageAttributesSection.offset - - fRepositoryInfoSection.compressedLength; - - // repository info size sanity check - if (fRepositoryInfoSection.uncompressedLength > kMaxRepositoryInfoSize) { - ErrorOutput()->PrintError("Error: Repository file info section size " - "is %llu bytes. This is beyond the reader's sanity limit\n", - fRepositoryInfoSection.uncompressedLength); - return B_UNSUPPORTED; - } - - // package attributes size sanity check - if (fPackageAttributesSection.uncompressedLength - > kMaxPackageAttributesSize) { - ErrorOutput()->PrintError( - "Error: Package file package attributes section size " - "is %llu bytes. This is beyond the reader's sanity limit\n", - fPackageAttributesSection.uncompressedLength); - return B_UNSUPPORTED; - } - - // read in the complete repository info section - fRepositoryInfoSection.data - = new(std::nothrow) uint8[fRepositoryInfoSection.uncompressedLength]; - if (fRepositoryInfoSection.data == NULL) { - ErrorOutput()->PrintError("Error: Out of memory!\n"); - return B_NO_MEMORY; - } - error = ReadCompressedBuffer(fRepositoryInfoSection); + status_t error = inherited::Init(fd, keepFD, header); if (error != B_OK) return error; - // read in the complete package attributes section - fPackageAttributesSection.data - = new(std::nothrow) uint8[fPackageAttributesSection.uncompressedLength]; - if (fPackageAttributesSection.data == NULL) { - ErrorOutput()->PrintError("Error: Out of memory!\n"); - return B_NO_MEMORY; - } - error = ReadCompressedBuffer(fPackageAttributesSection); + // init package attributes section + error = InitSection(fPackageAttributesSection, + HeapReader()->UncompressedHeapSize(), + B_BENDIAN_TO_HOST_INT64(header.packages_length), + kMaxPackageAttributesSize, + B_BENDIAN_TO_HOST_INT64(header.packages_strings_length), + B_BENDIAN_TO_HOST_INT64(header.packages_strings_count)); + if (error != B_OK) + return error; + + // init repository info section + PackageFileSection repositoryInfoSection("repository info"); + error = InitSection(repositoryInfoSection, + fPackageAttributesSection.offset, + B_BENDIAN_TO_HOST_INT32(header.info_length), kMaxRepositoryInfoSize, 0, + 0); + if (error != B_OK) + return error; + + // prepare the sections for use + error = PrepareSection(repositoryInfoSection); + if (error != B_OK) + return error; + + error = PrepareSection(fPackageAttributesSection); if (error != B_OK) return error; // unarchive repository info BMessage repositoryInfoArchive; - error = repositoryInfoArchive.Unflatten((char*)fRepositoryInfoSection.data); + error = repositoryInfoArchive.Unflatten((char*)repositoryInfoSection.data); if (error != B_OK) { ErrorOutput()->PrintError( "Error: Unable to unflatten repository info archive!\n"); @@ -225,17 +122,6 @@ RepositoryReaderImpl::Init(int fd, bool keepFD) return error; } - // parse strings from package attributes section - fPackageAttributesSection.currentOffset = 0; - SetCurrentSection(&fPackageAttributesSection); - - // strings - error = ParseStrings(); - if (error != B_OK) - return error; - - SetCurrentSection(NULL); - return B_OK; } diff --git a/src/kits/package/hpkg/RepositoryWriterImpl.cpp b/src/kits/package/hpkg/RepositoryWriterImpl.cpp index c7c87dda7f..c8ceb33a6b 100644 --- a/src/kits/package/hpkg/RepositoryWriterImpl.cpp +++ b/src/kits/package/hpkg/RepositoryWriterImpl.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -42,28 +43,31 @@ using BPackageKit::BPrivate::HashableString; namespace { +// #pragma mark - PackageEntryDataFetcher + + struct PackageEntryDataFetcher { PackageEntryDataFetcher(BErrorOutput* errorOutput, BPackageData& packageData) : fErrorOutput(errorOutput), - fBufferPool(B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, 2), fPackageData(packageData) { } - status_t ReadIntoString(BDataReader* dataReader, BString& _contents) + status_t ReadIntoString(BAbstractBufferedDataReader* heapReader, + BString& _contents) { // create a PackageDataReader BAbstractBufferedDataReader* reader; - status_t result = BPackageDataReaderFactory(&fBufferPool) - .CreatePackageDataReader(dataReader, fPackageData, reader); + status_t result = BPackageDataReaderFactory() + .CreatePackageDataReader(heapReader, fPackageData, reader); if (result != B_OK) return result; ObjectDeleter readerDeleter(reader); // copy data into the given string - int32 bufferSize = fPackageData.UncompressedSize(); + int32 bufferSize = fPackageData.Size(); char* buffer = _contents.LockBuffer(bufferSize); if (buffer == NULL) return B_NO_MEMORY; @@ -81,17 +85,20 @@ struct PackageEntryDataFetcher { private: BErrorOutput* fErrorOutput; - BBlockBufferPoolNoLock fBufferPool; BPackageData& fPackageData; }; +// #pragma mark - PackageContentHandler + + struct PackageContentHandler : public BPackageInfoContentHandler { PackageContentHandler(BErrorOutput* errorOutput, BPackageInfo* packageInfo, - int packageFileFD, BRepositoryInfo* repositoryInfo) + BAbstractBufferedDataReader* heapReader, + BRepositoryInfo* repositoryInfo) : BPackageInfoContentHandler(*packageInfo, errorOutput), - fPackageFileReader(packageFileFD), + fHeapReader(heapReader), fRepositoryInfo(repositoryInfo) { } @@ -132,16 +139,7 @@ struct PackageContentHandler : public BPackageInfoContentHandler { PackageEntryDataFetcher dataFetcher(fErrorOutput, packageData); BString licenseText; - status_t result; - if (packageData.IsEncodedInline()) { - BBufferDataReader dataReader(packageData.InlineData(), - packageData.CompressedSize()); - result = dataFetcher.ReadIntoString(&dataReader, licenseText); - } else { - result - = dataFetcher.ReadIntoString(&fPackageFileReader, licenseText); - } - + status_t result = dataFetcher.ReadIntoString(fHeapReader, licenseText); if (result != B_OK) return result; @@ -165,24 +163,30 @@ struct PackageContentHandler : public BPackageInfoContentHandler { } private: - BPackageReader* fPackageReader; - BFDDataReader fPackageFileReader; - BRepositoryInfo* fRepositoryInfo; + BPackageReader* fPackageReader; + BAbstractBufferedDataReader* fHeapReader; + BRepositoryInfo* fRepositoryInfo; }; } // anonymous namespace +// #pragma mark - PackageNameSet + + struct RepositoryWriterImpl::PackageNameSet : public ::BPrivate::HashSet { }; +// #pragma mark - RepositoryWriterImpl + + RepositoryWriterImpl::RepositoryWriterImpl(BRepositoryWriterListener* listener, BRepositoryInfo* repositoryInfo) : - inherited(listener), + inherited("repository", listener), fListener(listener), fRepositoryInfo(repositoryInfo), fPackageCount(0), @@ -246,7 +250,7 @@ RepositoryWriterImpl::Finish() status_t RepositoryWriterImpl::_Init(const char* fileName) { - return inherited::Init(fileName, "repository", 0); + return inherited::Init(fileName, 0); } @@ -255,29 +259,34 @@ RepositoryWriterImpl::_Finish() { hpkg_repo_header header; - // write repository header - ssize_t infoLengthCompressed; - status_t result = _WriteRepositoryInfo(header, infoLengthCompressed); + // write repository info + uint64 infoLength; + status_t result = _WriteRepositoryInfo(header, infoLength); if (result != B_OK) return result; // write package attributes - ssize_t packagesLengthCompressed; - off_t totalSize = _WritePackageAttributes(header, - sizeof(header) + infoLengthCompressed, packagesLengthCompressed); + uint64 packagesLength; + _WritePackageAttributes(header, packagesLength); - fListener->OnRepositoryDone(sizeof(header), infoLengthCompressed, + // flush the heap writer + result = fHeapWriter->Finish(); + if (result != B_OK) + return result; + uint64 totalSize = fHeapWriter->HeapOffset() + + fHeapWriter->CompressedHeapSize(); + + fListener->OnRepositoryDone(sizeof(header), infoLength, fRepositoryInfo->LicenseNames().CountStrings(), fPackageCount, - packagesLengthCompressed, totalSize); + packagesLength, totalSize); - // general + // update the general header info and write the header header.magic = B_HOST_TO_BENDIAN_INT32(B_HPKG_REPO_MAGIC); header.header_size = B_HOST_TO_BENDIAN_INT16((uint16)sizeof(header)); header.version = B_HOST_TO_BENDIAN_INT16(B_HPKG_REPO_VERSION); header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize); - // write the header - WriteBuffer(&header, sizeof(header), 0); + RawWriteBuffer(&header, sizeof(header), 0); SetFinished(true); return B_OK; @@ -309,7 +318,7 @@ RepositoryWriterImpl::_AddPackage(const BEntry& packageEntry) // parse package PackageContentHandler contentHandler(fListener, &fPackageInfo, - packageReader.PackageFileFD(), fRepositoryInfo); + packageReader.HeapReader(), fRepositoryInfo); if ((result = packageReader.ParseContent(&contentHandler)) != B_OK) return result; @@ -384,8 +393,9 @@ RepositoryWriterImpl::_RegisterCurrentPackageInfo() status_t RepositoryWriterImpl::_WriteRepositoryInfo(hpkg_repo_header& header, - ssize_t& _infoLengthCompressed) + uint64& _length) { + // archive and flatten the repository info and write it BMessage archive; status_t result = fRepositoryInfo->Archive(&archive); if (result != B_OK) { @@ -400,71 +410,40 @@ RepositoryWriterImpl::_WriteRepositoryInfo(hpkg_repo_header& header, return result; } - off_t startOffset = sizeof(hpkg_repo_header); + WriteBuffer(buffer, flattenedSize); - // write the package attributes (zlib writer on top of a file writer) - FDDataWriter realWriter(FD(), startOffset, fListener); - ZlibDataWriter zlibWriter(&realWriter); - SetDataWriter(&zlibWriter); - zlibWriter.Init(); - - DataWriter()->WriteDataThrows(buffer, flattenedSize); - - zlibWriter.Finish(); - SetDataWriter(NULL); - - fListener->OnRepositoryInfoSectionDone(zlibWriter.BytesWritten()); - - _infoLengthCompressed = realWriter.BytesWritten(); + // notify listener + fListener->OnRepositoryInfoSectionDone(flattenedSize); // update the header - header.info_compression - = B_HOST_TO_BENDIAN_INT32(B_HPKG_COMPRESSION_ZLIB); - header.info_length_compressed - = B_HOST_TO_BENDIAN_INT32(_infoLengthCompressed); - header.info_length_uncompressed - = B_HOST_TO_BENDIAN_INT32(flattenedSize); + header.info_length = B_HOST_TO_BENDIAN_INT32(flattenedSize); + _length = flattenedSize; return B_OK; } -off_t +void RepositoryWriterImpl::_WritePackageAttributes(hpkg_repo_header& header, - off_t startOffset, ssize_t& _packagesLengthCompressed) + uint64& _length) { // write the package attributes (zlib writer on top of a file writer) - FDDataWriter realWriter(FD(), startOffset, fListener); - ZlibDataWriter zlibWriter(&realWriter); - SetDataWriter(&zlibWriter); - zlibWriter.Init(); + uint64 startOffset = fHeapWriter->UncompressedHeapSize(); - // write cached strings and package attributes tree - uint32 stringsLengthUncompressed; + uint32 stringsLength; uint32 stringsCount = WritePackageAttributes(PackageAttributes(), - stringsLengthUncompressed); + stringsLength); - zlibWriter.Finish(); - off_t endOffset = realWriter.Offset(); - SetDataWriter(NULL); + uint64 sectionSize = fHeapWriter->UncompressedHeapSize() - startOffset; - fListener->OnPackageAttributesSectionDone(stringsCount, - zlibWriter.BytesWritten()); - - _packagesLengthCompressed = endOffset - startOffset; + fListener->OnPackageAttributesSectionDone(stringsCount, sectionSize); // update the header - header.packages_compression - = B_HOST_TO_BENDIAN_INT32(B_HPKG_COMPRESSION_ZLIB); - header.packages_length_compressed - = B_HOST_TO_BENDIAN_INT64(_packagesLengthCompressed); - header.packages_length_uncompressed - = B_HOST_TO_BENDIAN_INT64(zlibWriter.BytesWritten()); + header.packages_length = B_HOST_TO_BENDIAN_INT64(sectionSize); header.packages_strings_count = B_HOST_TO_BENDIAN_INT64(stringsCount); - header.packages_strings_length - = B_HOST_TO_BENDIAN_INT64(stringsLengthUncompressed); + header.packages_strings_length = B_HOST_TO_BENDIAN_INT64(stringsLength); - return endOffset; + _length = sectionSize; } diff --git a/src/kits/package/hpkg/WriterImplBase.cpp b/src/kits/package/hpkg/WriterImplBase.cpp index 7a539a3795..88d9d9be52 100644 --- a/src/kits/package/hpkg/WriterImplBase.cpp +++ b/src/kits/package/hpkg/WriterImplBase.cpp @@ -18,11 +18,12 @@ #include -#include - #include #include +#include +#include + namespace BPackageKit { @@ -176,106 +177,9 @@ WriterImplBase::AttributeValue::_ApplicableIntEncoding(uint64 value) } -// #pragma mark - AbstractDataWriter - - -WriterImplBase::AbstractDataWriter::AbstractDataWriter() - : - fBytesWritten(0) -{ -} - - -WriterImplBase::AbstractDataWriter::~AbstractDataWriter() -{ -} - - -// #pragma mark - FDDataWriter - - -WriterImplBase::FDDataWriter::FDDataWriter(int fd, off_t offset, - BErrorOutput* errorOutput) - : - fFD(fd), - fOffset(offset), - fErrorOutput(errorOutput) -{ -} - - -status_t -WriterImplBase::FDDataWriter::WriteDataNoThrow(const void* buffer, size_t size) -{ - ssize_t bytesWritten = pwrite(fFD, buffer, size, fOffset); - if (bytesWritten < 0) { - fErrorOutput->PrintError( - "WriteDataNoThrow(%p, %lu) failed to write data: %s\n", buffer, - size, strerror(errno)); - return errno; - } - if ((size_t)bytesWritten != size) { - fErrorOutput->PrintError( - "WriteDataNoThrow(%p, %lu) failed to write all data\n", buffer, - size); - return B_ERROR; - } - - fOffset += size; - fBytesWritten += size; - return B_OK; -} - - -// #pragma mark - ZlibDataWriter - - -WriterImplBase::ZlibDataWriter::ZlibDataWriter(AbstractDataWriter* dataWriter) - : - fDataWriter(dataWriter), - fCompressor(this) -{ -} - - -void -WriterImplBase::ZlibDataWriter::Init() -{ - status_t error = fCompressor.Init(); - if (error != B_OK) - throw status_t(error); -} - - -void -WriterImplBase::ZlibDataWriter::Finish() -{ - status_t error = fCompressor.Finish(); - if (error != B_OK) - throw status_t(error); -} - - -status_t -WriterImplBase::ZlibDataWriter::WriteDataNoThrow(const void* buffer, - size_t size) -{ - status_t error = fCompressor.CompressNext(buffer, size); - if (error == B_OK) - fBytesWritten += size; - return error; -} - - -status_t -WriterImplBase::ZlibDataWriter::WriteData(const void* buffer, size_t size) -{ - return fDataWriter->WriteDataNoThrow(buffer, size); -} - - // #pragma mark - PackageAttribute + WriterImplBase::PackageAttribute::PackageAttribute(BHPKGAttributeID id_, uint8 type_, uint8 encoding_) : @@ -310,8 +214,10 @@ WriterImplBase::PackageAttribute::_DeleteChildren() // #pragma mark - WriterImplBase -WriterImplBase::WriterImplBase(BErrorOutput* errorOutput) +WriterImplBase::WriterImplBase(const char* fileType, BErrorOutput* errorOutput) : + fHeapWriter(NULL), + fFileType(fileType), fErrorOutput(errorOutput), fFileName(NULL), fFlags(0), @@ -324,6 +230,8 @@ WriterImplBase::WriterImplBase(BErrorOutput* errorOutput) WriterImplBase::~WriterImplBase() { + delete fHeapWriter; + if (fFD >= 0) close(fFD); @@ -335,7 +243,7 @@ WriterImplBase::~WriterImplBase() status_t -WriterImplBase::Init(const char* fileName, const char* type, uint32 flags) +WriterImplBase::Init(const char* fileName, uint32 flags) { if (fPackageStringCache.Init() != B_OK) throw std::bad_alloc(); @@ -347,14 +255,20 @@ WriterImplBase::Init(const char* fileName, const char* type, uint32 flags) fFD = open(fileName, openMode, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (fFD < 0) { - fErrorOutput->PrintError("Failed to open %s file \"%s\": %s\n", type, - fileName, strerror(errno)); + fErrorOutput->PrintError("Failed to open %s file \"%s\": %s\n", + fFileType, fileName, strerror(errno)); return errno; } fFileName = fileName; fFlags = flags; + // create heap writer + fHeapWriter = new PackageFileHeapWriter(fErrorOutput, FD(), + sizeof(hpkg_header)); + fHeapWriter->Init(); + fDataWriter = fHeapWriter->DataWriter(); + return B_OK; } @@ -670,8 +584,10 @@ WriterImplBase::WritePackageAttributes( uint32& _stringsLengthUncompressed) { // write the cached strings + uint64 startOffset = fHeapWriter->UncompressedHeapSize(); uint32 stringsCount = WriteCachedStrings(fPackageStringCache, 2); - _stringsLengthUncompressed = DataWriter()->BytesWritten(); + _stringsLengthUncompressed + = fHeapWriter->UncompressedHeapSize() - startOffset; _WritePackageAttributes(packageAttributes); @@ -760,18 +676,18 @@ WriterImplBase::WriteUnsignedLEB128(uint64 value) void -WriterImplBase::WriteBuffer(const void* buffer, size_t size, off_t offset) +WriterImplBase::RawWriteBuffer(const void* buffer, size_t size, off_t offset) { ssize_t bytesWritten = pwrite(fFD, buffer, size, offset); if (bytesWritten < 0) { fErrorOutput->PrintError( - "WriteBuffer(%p, %lu) failed to write data: %s\n", buffer, size, + "RawWriteBuffer(%p, %lu) failed to write data: %s\n", buffer, size, strerror(errno)); throw status_t(errno); } if ((size_t)bytesWritten != size) { fErrorOutput->PrintError( - "WriteBuffer(%p, %lu) failed to write all data\n", buffer, size); + "RawWriteBuffer(%p, %lu) failed to write all data\n", buffer, size); throw status_t(B_ERROR); } } diff --git a/src/kits/package/hpkg/v1/PackageDataReaderV1.cpp b/src/kits/package/hpkg/v1/PackageDataReaderV1.cpp index a77e0b0877..9fc006319f 100644 --- a/src/kits/package/hpkg/v1/PackageDataReaderV1.cpp +++ b/src/kits/package/hpkg/v1/PackageDataReaderV1.cpp @@ -397,6 +397,23 @@ private: }; +// #pragma mark - PackageDataHeapReader + + +class PackageDataInlineReader : public BBufferDataReader { +public: + PackageDataInlineReader(const BPackageData& data) + : + BBufferDataReader(fData.InlineData(), data.UncompressedSize()), + fData(data) + { + } + +private: + BPackageData fData; +}; + + // #pragma mark - BPackageDataReaderFactory @@ -411,6 +428,16 @@ status_t BPackageDataReaderFactory::CreatePackageDataReader(BDataReader* dataReader, const BPackageData& data, BAbstractBufferedDataReader*& _reader) { + if (data.IsEncodedInline()) { + BAbstractBufferedDataReader* reader + = new(std::nothrow) PackageDataInlineReader(data); + if (reader == NULL) + return B_NO_MEMORY; + + _reader = reader; + return B_OK; + } + PackageDataReader* reader; switch (data.Compression()) { diff --git a/src/system/boot/loader/file_systems/packagefs/Jamfile b/src/system/boot/loader/file_systems/packagefs/Jamfile index bd42445609..2798472348 100644 --- a/src/system/boot/loader/file_systems/packagefs/Jamfile +++ b/src/system/boot/loader/file_systems/packagefs/Jamfile @@ -35,6 +35,8 @@ BootStaticLibrary boot_packagefs : PackageDataReader.cpp PackageEntry.cpp PackageEntryAttribute.cpp + PackageFileHeapAccessorBase.cpp + PackageFileHeapReader.cpp PackageReaderImpl.cpp ReaderImplBase.cpp diff --git a/src/system/boot/loader/file_systems/packagefs/packagefs.cpp b/src/system/boot/loader/file_systems/packagefs/packagefs.cpp index 66a608eaa2..390fe2b9ad 100644 --- a/src/system/boot/loader/file_systems/packagefs/packagefs.cpp +++ b/src/system/boot/loader/file_systems/packagefs/packagefs.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2011, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2011-2013, Ingo Weinhold, ingo_weinhold@gmx.de. * Distributed under the terms of the MIT License. */ @@ -9,12 +9,12 @@ #include #include -#include #include #include #include #include #include +#include #include #include @@ -41,6 +41,7 @@ using namespace BPackageKit; using namespace BPackageKit::BHPKG; +using BPackageKit::BHPKG::BPrivate::PackageFileHeapReader; using BPackageKit::BHPKG::BPrivate::PackageReaderImpl; @@ -143,7 +144,7 @@ struct PackageFile : PackageNode { off_t Size() const { - return fData.UncompressedSize(); + return fData.Size(); } private: @@ -232,85 +233,60 @@ private: }; -// #pragma mark - FileDataReader +// #pragma mark - PackageLoaderErrorOutput -struct FileDataReader { - FileDataReader(int fd, const BPackageData& data) - : - fDataReader(fd), - fData(data), - fPackageDataReader(NULL) +struct PackageLoaderErrorOutput : BErrorOutput { + PackageLoaderErrorOutput() { } - ~FileDataReader() + virtual void PrintErrorVarArgs(const char* format, va_list args) { - delete fPackageDataReader; } - - status_t Init(BPackageDataReaderFactory* dataReaderFactory) - { - if (fData.IsEncodedInline()) - return B_OK; - - return dataReaderFactory->CreatePackageDataReader(&fDataReader, - fData, fPackageDataReader); - } - - status_t ReadData(off_t offset, void* buffer, size_t bufferSize) - { - if (fData.IsEncodedInline()) { - BBufferDataReader dataReader(fData.InlineData(), - fData.CompressedSize()); - return dataReader.ReadData(offset, buffer, bufferSize); - } - - return fPackageDataReader->ReadData(offset, buffer, bufferSize); - } - -private: - BFDDataReader fDataReader; - BPackageData fData; - BAbstractBufferedDataReader* fPackageDataReader; }; // #pragma mark - PackageVolume -struct PackageVolume : BReferenceable { +struct PackageVolume : BReferenceable, private PackageLoaderErrorOutput { PackageVolume() : fNextNodeID(1), fRootDirectory(this, S_IFDIR), - fBufferCache(B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, 2), - fDataReaderFactory(&fBufferCache), + fHeapReader(NULL), fFD(-1) { } ~PackageVolume() { + delete fHeapReader; + if (fFD >= 0) close(fFD); } - status_t Init(int fd) + status_t Init(int fd, const PackageFileHeapReader* heapReader) { status_t error = fRootDirectory.Init(&fRootDirectory, ".", NextNodeID()); if (error != B_OK) return error; - error = fBufferCache.Init(); - if (error != B_OK) - return error; - fFD = dup(fd); if (fFD < 0) return errno; + // clone a heap reader and adjust it for our use + fHeapReader = heapReader->Clone(); + if (fHeapReader == NULL) + return B_NO_MEMORY; + + fHeapReader->SetErrorOutput(this); + fHeapReader->SetFD(fFD); + return B_OK; } @@ -325,46 +301,20 @@ struct PackageVolume : BReferenceable { } status_t CreateFileDataReader(const BPackageData& data, - FileDataReader*& _fileDataReader) + BAbstractBufferedDataReader*& _reader) { - // create the file reader object - FileDataReader* fileDataReader = new(std::nothrow) FileDataReader(fFD, - data); - if (fileDataReader == NULL) - return B_NO_MEMORY; - ObjectDeleter fileDataReaderDeleter(fileDataReader); - - status_t error = fileDataReader->Init(&fDataReaderFactory); - if (error != B_OK) - return error; - - _fileDataReader = fileDataReaderDeleter.Detach(); - return B_OK; + return BPackageDataReaderFactory().CreatePackageDataReader(fHeapReader, + data, _reader); } private: ino_t fNextNodeID; PackageDirectory fRootDirectory; - BBlockBufferPoolNoLock fBufferCache; - BPackageDataReaderFactory fDataReaderFactory; + PackageFileHeapReader* fHeapReader; int fFD; }; -// #pragma mark - PackageLoaderErrorOutput - - -struct PackageLoaderErrorOutput : BErrorOutput { - PackageLoaderErrorOutput() - { - } - - virtual void PrintErrorVarArgs(const char* format, va_list args) - { - } -}; - - // #pragma mark - PackageLoaderContentHandler @@ -503,7 +453,8 @@ struct File : ::Node { bufferSize = size - pos; if (bufferSize > 0) { - FileDataReader* dataReader = (FileDataReader*)cookie; + BAbstractBufferedDataReader* dataReader + = (BAbstractBufferedDataReader*)cookie; status_t error = dataReader->ReadData(pos, buffer, bufferSize); if (error != B_OK) return error; @@ -529,7 +480,7 @@ struct File : ::Node { if ((mode & O_ACCMODE) != O_RDONLY && (mode & O_ACCMODE) != O_RDWR) return B_NOT_ALLOWED; - FileDataReader* dataReader; + BAbstractBufferedDataReader* dataReader; status_t error = fFile->Volume()->CreateFileDataReader(fFile->Data(), dataReader); if (error != B_OK) @@ -542,7 +493,8 @@ struct File : ::Node { virtual status_t Close(void* cookie) { - FileDataReader* dataReader = (FileDataReader*)cookie; + BAbstractBufferedDataReader* dataReader + = (BAbstractBufferedDataReader*)cookie; delete dataReader; Release(); return B_OK; @@ -807,7 +759,7 @@ packagefs_mount_file(int fd, ::Directory*& _mountedDirectory) { PackageLoaderErrorOutput errorOutput; PackageReaderImpl packageReader(&errorOutput); - status_t error = packageReader.Init(fd, false); + status_t error = packageReader.Init(fd, false); if (error != B_OK) RETURN_ERROR(error); @@ -817,7 +769,7 @@ packagefs_mount_file(int fd, ::Directory*& _mountedDirectory) return B_NO_MEMORY; BReference volumeReference(volume, true); - error = volume->Init(fd); + error = volume->Init(fd, packageReader.HeapReader()); if (error != B_OK) RETURN_ERROR(error);