diff --git a/headers/build/private/package/hpkg/DataWriters.h b/headers/build/private/package/hpkg/DataWriters.h new file mode 100644 index 0000000000..1f9f8962ca --- /dev/null +++ b/headers/build/private/package/hpkg/DataWriters.h @@ -0,0 +1 @@ +#include <../private/package/hpkg/DataWriters.h> diff --git a/headers/build/private/package/hpkg/PackageFileHeapAccessorBase.h b/headers/build/private/package/hpkg/PackageFileHeapAccessorBase.h new file mode 100644 index 0000000000..f9c224e97f --- /dev/null +++ b/headers/build/private/package/hpkg/PackageFileHeapAccessorBase.h @@ -0,0 +1 @@ +#include <../private/package/hpkg/PackageFileHeapAccessorBase.h> diff --git a/headers/build/private/package/hpkg/PackageFileHeapReader.h b/headers/build/private/package/hpkg/PackageFileHeapReader.h new file mode 100644 index 0000000000..e051dd2d6f --- /dev/null +++ b/headers/build/private/package/hpkg/PackageFileHeapReader.h @@ -0,0 +1 @@ +#include <../private/package/hpkg/PackageFileHeapReader.h> diff --git a/headers/build/private/package/hpkg/PackageFileHeapWriter.h b/headers/build/private/package/hpkg/PackageFileHeapWriter.h new file mode 100644 index 0000000000..88fd3c8057 --- /dev/null +++ b/headers/build/private/package/hpkg/PackageFileHeapWriter.h @@ -0,0 +1 @@ +#include <../private/package/hpkg/PackageFileHeapWriter.h> diff --git a/headers/os/package/hpkg/HPKGDefs.h b/headers/os/package/hpkg/HPKGDefs.h index d4f90156c1..7cd931b52a 100644 --- a/headers/os/package/hpkg/HPKGDefs.h +++ b/headers/os/package/hpkg/HPKGDefs.h @@ -20,7 +20,7 @@ enum { B_HPKG_VERSION = 2, // B_HPKG_REPO_MAGIC = 'hpkr', - B_HPKG_REPO_VERSION = 1 + B_HPKG_REPO_VERSION = 2 }; @@ -98,37 +98,34 @@ enum BHPKGAttributeID { B_HPKG_ATTRIBUTE_ID_FILE_ATTRIBUTE = 11, B_HPKG_ATTRIBUTE_ID_FILE_ATTRIBUTE_TYPE = 12, B_HPKG_ATTRIBUTE_ID_DATA = 13, - B_HPKG_ATTRIBUTE_ID_DATA_SIZE = 14, - B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION = 15, - B_HPKG_ATTRIBUTE_ID_DATA_CHUNK_SIZE = 16, - B_HPKG_ATTRIBUTE_ID_SYMLINK_PATH = 17, - B_HPKG_ATTRIBUTE_ID_PACKAGE_NAME = 18, - B_HPKG_ATTRIBUTE_ID_PACKAGE_SUMMARY = 19, - B_HPKG_ATTRIBUTE_ID_PACKAGE_DESCRIPTION = 20, - B_HPKG_ATTRIBUTE_ID_PACKAGE_VENDOR = 21, - B_HPKG_ATTRIBUTE_ID_PACKAGE_PACKAGER = 22, - B_HPKG_ATTRIBUTE_ID_PACKAGE_FLAGS = 23, - B_HPKG_ATTRIBUTE_ID_PACKAGE_ARCHITECTURE = 24, - B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MAJOR = 25, - B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MINOR = 26, - B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MICRO = 27, - B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_REVISION = 28, - B_HPKG_ATTRIBUTE_ID_PACKAGE_COPYRIGHT = 29, - B_HPKG_ATTRIBUTE_ID_PACKAGE_LICENSE = 30, - B_HPKG_ATTRIBUTE_ID_PACKAGE_PROVIDES = 31, - B_HPKG_ATTRIBUTE_ID_PACKAGE_REQUIRES = 32, - B_HPKG_ATTRIBUTE_ID_PACKAGE_SUPPLEMENTS = 33, - B_HPKG_ATTRIBUTE_ID_PACKAGE_CONFLICTS = 34, - B_HPKG_ATTRIBUTE_ID_PACKAGE_FRESHENS = 35, - B_HPKG_ATTRIBUTE_ID_PACKAGE_REPLACES = 36, - B_HPKG_ATTRIBUTE_ID_PACKAGE_RESOLVABLE_OPERATOR = 37, - B_HPKG_ATTRIBUTE_ID_PACKAGE_CHECKSUM = 38, - B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_PRE_RELEASE = 39, - B_HPKG_ATTRIBUTE_ID_PACKAGE_PROVIDES_COMPATIBLE = 40, - B_HPKG_ATTRIBUTE_ID_PACKAGE_URL = 41, - B_HPKG_ATTRIBUTE_ID_PACKAGE_SOURCE_URL = 42, - B_HPKG_ATTRIBUTE_ID_PACKAGE_INSTALL_PATH = 43, - B_HPKG_ATTRIBUTE_ID_PACKAGE_BASE_PACKAGE = 44, + B_HPKG_ATTRIBUTE_ID_SYMLINK_PATH = 14, + B_HPKG_ATTRIBUTE_ID_PACKAGE_NAME = 15, + B_HPKG_ATTRIBUTE_ID_PACKAGE_SUMMARY = 16, + B_HPKG_ATTRIBUTE_ID_PACKAGE_DESCRIPTION = 17, + B_HPKG_ATTRIBUTE_ID_PACKAGE_VENDOR = 18, + B_HPKG_ATTRIBUTE_ID_PACKAGE_PACKAGER = 19, + B_HPKG_ATTRIBUTE_ID_PACKAGE_FLAGS = 20, + B_HPKG_ATTRIBUTE_ID_PACKAGE_ARCHITECTURE = 21, + B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MAJOR = 22, + B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MINOR = 23, + B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MICRO = 24, + B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_REVISION = 25, + B_HPKG_ATTRIBUTE_ID_PACKAGE_COPYRIGHT = 26, + B_HPKG_ATTRIBUTE_ID_PACKAGE_LICENSE = 27, + B_HPKG_ATTRIBUTE_ID_PACKAGE_PROVIDES = 28, + B_HPKG_ATTRIBUTE_ID_PACKAGE_REQUIRES = 29, + B_HPKG_ATTRIBUTE_ID_PACKAGE_SUPPLEMENTS = 30, + B_HPKG_ATTRIBUTE_ID_PACKAGE_CONFLICTS = 31, + B_HPKG_ATTRIBUTE_ID_PACKAGE_FRESHENS = 32, + B_HPKG_ATTRIBUTE_ID_PACKAGE_REPLACES = 33, + B_HPKG_ATTRIBUTE_ID_PACKAGE_RESOLVABLE_OPERATOR = 34, + B_HPKG_ATTRIBUTE_ID_PACKAGE_CHECKSUM = 35, + B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_PRE_RELEASE = 36, + B_HPKG_ATTRIBUTE_ID_PACKAGE_PROVIDES_COMPATIBLE = 37, + B_HPKG_ATTRIBUTE_ID_PACKAGE_URL = 38, + B_HPKG_ATTRIBUTE_ID_PACKAGE_SOURCE_URL = 39, + B_HPKG_ATTRIBUTE_ID_PACKAGE_INSTALL_PATH = 40, + B_HPKG_ATTRIBUTE_ID_PACKAGE_BASE_PACKAGE = 41, // B_HPKG_ATTRIBUTE_ID_ENUM_COUNT, }; @@ -154,9 +151,7 @@ enum { B_HPKG_DEFAULT_FILE_TYPE = B_HPKG_FILE_TYPE_FILE, B_HPKG_DEFAULT_FILE_PERMISSIONS = 0644, B_HPKG_DEFAULT_DIRECTORY_PERMISSIONS = 0755, - B_HPKG_DEFAULT_SYMLINK_PERMISSIONS = 0777, - B_HPKG_DEFAULT_DATA_COMPRESSION = B_HPKG_COMPRESSION_NONE, - B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB = 64 * 1024 + B_HPKG_DEFAULT_SYMLINK_PERMISSIONS = 0777 }; diff --git a/headers/os/package/hpkg/PackageData.h b/headers/os/package/hpkg/PackageData.h index d0ef1c0799..fa0fa4811a 100644 --- a/headers/os/package/hpkg/PackageData.h +++ b/headers/os/package/hpkg/PackageData.h @@ -18,14 +18,10 @@ class BPackageData { public: BPackageData(); - uint64 CompressedSize() const - { return fCompressedSize; } - uint64 UncompressedSize() const - { return fUncompressedSize; } + uint64 Size() const + { return fSize; } uint64 Offset() const { return fEncodedInline ? 0 : fOffset; } - uint32 Compression() const { return fCompression; } - uint32 ChunkSize() const { return fChunkSize; } bool IsEncodedInline() const { return fEncodedInline; } @@ -34,23 +30,13 @@ public: void SetData(uint64 size, uint64 offset); void SetData(uint8 size, const void* data); - void SetCompression(uint32 compression) - { fCompression = compression; } - void SetUncompressedSize(uint64 size) - { fUncompressedSize = size; } - void SetChunkSize(uint32 size) - { fChunkSize = size; } - private: - uint64 fCompressedSize; - uint64 fUncompressedSize; + uint64 fSize : 63; + bool fEncodedInline : 1; union { uint64 fOffset; uint8 fInlineData[B_HPKG_MAX_INLINE_DATA_SIZE]; }; - uint32 fChunkSize; - uint32 fCompression; - bool fEncodedInline; }; diff --git a/headers/os/package/hpkg/PackageDataReader.h b/headers/os/package/hpkg/PackageDataReader.h index 1152fbaf49..0dcebf9397 100644 --- a/headers/os/package/hpkg/PackageDataReader.h +++ b/headers/os/package/hpkg/PackageDataReader.h @@ -20,15 +20,12 @@ class BPackageData; class BPackageDataReaderFactory { public: - BPackageDataReaderFactory( - BBufferPool* bufferPool); + BPackageDataReaderFactory(); - status_t CreatePackageDataReader(BDataReader* dataReader, + status_t CreatePackageDataReader( + BAbstractBufferedDataReader* dataReader, const BPackageData& data, BAbstractBufferedDataReader*& _reader); - -private: - BBufferPool* fBufferPool; }; diff --git a/headers/os/package/hpkg/PackageReader.h b/headers/os/package/hpkg/PackageReader.h index e559c35e10..3828a4b410 100644 --- a/headers/os/package/hpkg/PackageReader.h +++ b/headers/os/package/hpkg/PackageReader.h @@ -19,6 +19,8 @@ namespace BPrivate { } using BPrivate::PackageReaderImpl; + +class BAbstractBufferedDataReader; class BErrorOutput; class BLowLevelPackageContentHandler; class BPackageContentHandler; @@ -26,8 +28,7 @@ class BPackageContentHandler; class BPackageReader { public: - BPackageReader( - BErrorOutput* errorOutput); + BPackageReader(BErrorOutput* errorOutput); ~BPackageReader(); status_t Init(const char* fileName); @@ -38,6 +39,10 @@ public: contentHandler); int PackageFileFD(); + + BAbstractBufferedDataReader* HeapReader() const; + // Only valid as long as the reader lives. + private: PackageReaderImpl* fImpl; }; diff --git a/headers/os/package/hpkg/RepositoryReader.h b/headers/os/package/hpkg/RepositoryReader.h index f42c077d7d..d7e3539bbd 100644 --- a/headers/os/package/hpkg/RepositoryReader.h +++ b/headers/os/package/hpkg/RepositoryReader.h @@ -19,6 +19,7 @@ namespace BPrivate { } using BPrivate::RepositoryReaderImpl; + class BErrorOutput; class BRepositoryContentHandler; diff --git a/headers/private/package/hpkg/DataWriters.h b/headers/private/package/hpkg/DataWriters.h new file mode 100644 index 0000000000..26f67cf411 --- /dev/null +++ b/headers/private/package/hpkg/DataWriters.h @@ -0,0 +1,101 @@ +/* + * Copyright 2011, Oliver Tappe + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef _PACKAGE__HPKG__PRIVATE__DATA_WRITERS_H_ +#define _PACKAGE__HPKG__PRIVATE__DATA_WRITERS_H_ + + +#include +#include + + +namespace BPackageKit { + +namespace BHPKG { + + +class BErrorOutput; + + +namespace BPrivate { + + +class AbstractDataWriter { +public: + AbstractDataWriter(); + virtual ~AbstractDataWriter(); + + uint64 BytesWritten() const + { return fBytesWritten; } + + virtual status_t WriteDataNoThrow(const void* buffer, + size_t size) = 0; + + void WriteDataThrows(const void* buffer, + size_t size); + +protected: + uint64 fBytesWritten; +}; + + +class FDDataWriter : public AbstractDataWriter { +public: + FDDataWriter(int fd, off_t offset, + BErrorOutput* errorOutput); + + virtual status_t WriteDataNoThrow(const void* buffer, + size_t size); + + off_t Offset() const + { return fOffset; } + +private: + int fFD; + off_t fOffset; + BErrorOutput* fErrorOutput; +}; + + +class ZlibDataWriter : public AbstractDataWriter, private BDataOutput { +public: + ZlibDataWriter(AbstractDataWriter* dataWriter); + + void Init(); + void Finish(); + + virtual status_t WriteDataNoThrow(const void* buffer, + size_t size); + +private: + // BDataOutput + virtual status_t WriteData(const void* buffer, size_t size); + +private: + AbstractDataWriter* fDataWriter; + ZlibCompressor fCompressor; +}; + + +// inline implementations + + +inline void +AbstractDataWriter::WriteDataThrows(const void* buffer, size_t size) +{ + status_t error = WriteDataNoThrow(buffer, size); + if (error != B_OK) + throw status_t(error); +} + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit + + +#endif // _PACKAGE__HPKG__PRIVATE__DATA_WRITERS_H_ diff --git a/headers/private/package/hpkg/HPKGDefsPrivate.h b/headers/private/package/hpkg/HPKGDefsPrivate.h index 7b29f00a2d..dd2b82062c 100644 --- a/headers/private/package/hpkg/HPKGDefsPrivate.h +++ b/headers/private/package/hpkg/HPKGDefsPrivate.h @@ -18,7 +18,7 @@ namespace BHPKG { namespace BPrivate { -// header +// package file header struct hpkg_header { uint32 magic; // "hpkg" uint16 header_size; @@ -26,40 +26,41 @@ struct hpkg_header { uint64 total_size; // heap -// uint64 heap_compression; + uint32 heap_compression; + uint32 heap_chunk_size; + uint64 heap_size_compressed; + uint64 heap_size_uncompressed; // package attributes section - uint32 attributes_compression; - uint32 attributes_length_compressed; - uint32 attributes_length_uncompressed; + uint32 attributes_length; uint32 attributes_strings_length; uint32 attributes_strings_count; // TOC section - uint32 toc_compression; - uint64 toc_length_compressed; - uint64 toc_length_uncompressed; + uint64 toc_length; uint64 toc_strings_length; uint64 toc_strings_count; }; -// header +// repository file header struct hpkg_repo_header { uint32 magic; // "hpkr" uint16 header_size; uint16 version; uint64 total_size; + // heap + uint32 heap_compression; + uint32 heap_chunk_size; + uint64 heap_size_compressed; + uint64 heap_size_uncompressed; + // repository info section - uint32 info_compression; - uint32 info_length_compressed; - uint32 info_length_uncompressed; + uint32 info_length; // package attributes section - uint32 packages_compression; - uint64 packages_length_compressed; - uint64 packages_length_uncompressed; + uint64 packages_length; uint64 packages_strings_length; uint64 packages_strings_count; }; diff --git a/headers/private/package/hpkg/PackageFileHeapAccessorBase.h b/headers/private/package/hpkg/PackageFileHeapAccessorBase.h new file mode 100644 index 0000000000..1cf9bd25c2 --- /dev/null +++ b/headers/private/package/hpkg/PackageFileHeapAccessorBase.h @@ -0,0 +1,143 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_ACCESSOR_BASE_H_ +#define _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_ACCESSOR_BASE_H_ + + +#include + +#include + + +namespace BPackageKit { + +namespace BHPKG { + + +class BErrorOutput; + + +namespace BPrivate { + + +class PackageFileHeapAccessorBase : public BAbstractBufferedDataReader { +public: + class OffsetArray; + +public: + PackageFileHeapAccessorBase( + BErrorOutput* errorOutput, int fd, + off_t heapOffset); + virtual ~PackageFileHeapAccessorBase(); + + off_t HeapOffset() const + { return fHeapOffset; } + off_t CompressedHeapSize() const + { return fCompressedHeapSize; } + uint64 UncompressedHeapSize() const + { return fUncompressedHeapSize; } + size_t ChunkSize() const + { return kChunkSize; } + + // normally used after cloning a PackageFileHeapReader only + void SetErrorOutput(BErrorOutput* errorOutput) + { fErrorOutput = errorOutput; } + void SetFD(int fd) + { fFD = fd; } + + // BAbstractBufferedDataReader + virtual status_t ReadDataToOutput(off_t offset, size_t size, + BDataOutput* output); + +public: + static const size_t kChunkSize = 64 * 1024; + +protected: + virtual status_t ReadAndDecompressChunk(size_t chunkIndex, + void* compressedDataBuffer, + void* uncompressedDataBuffer) = 0; + status_t ReadAndDecompressChunkData(uint64 offset, + size_t compressedSize, + size_t uncompressedSize, + void* compressedDataBuffer, + void* uncompressedDataBuffer); + status_t ReadFileData(uint64 offset, void* buffer, + size_t size); + +protected: + BErrorOutput* fErrorOutput; + int fFD; + off_t fHeapOffset; + uint64 fCompressedHeapSize; + uint64 fUncompressedHeapSize; +}; + + +/*! Stores the chunk offsets in a compact way, while still providing quick + access. + - The object doesn't store the number of chunks/offsets it contains. During + initialization the chunk count is provided. Later, when getting an offset, + the caller is responsible for ensuring a valid index. + - The first (index 0) chunk offset is omitted, since it is always 0. + - The chunk offsets that fit in a 32 bit number use only one 32 bit element + in the offsets array. + - The chunk offsets that don't fit in a 32 bit number use two elements in + the offsets array. + Memory use is one pointer, if the chunk count is <= 1 (uncompressed heap size + <= 64 KiB). Afterwards it's one pointer plus 32 bit per chunk as long as the + last offset still fits 32 bit (compressed heap size < 4GiB). For any further + chunks it is 64 bit per chunk. So, for the common case we use sizeof(void*) + plus 1 KiB per 16 MiB of uncompressed heap, or about 64 KiB per 1 GiB. Which + seems reasonable for packagefs to keep in memory. + */ +class PackageFileHeapAccessorBase::OffsetArray { +public: + OffsetArray(); + ~OffsetArray(); + + bool InitChunksOffsets(size_t totalChunkCount, + size_t baseIndex, const uint16* chunkSizes, + size_t chunkCount); + + bool Init(size_t totalChunkCount, + const OffsetArray& other); + // "copy" init + + uint64 operator[](size_t index) const; + +private: + uint32* fOffsets; + // - NULL, if chunkCount <= 1 + // - element 0 contains the number of 32 bit + // offsets that follow, or is 0, when all + // offsets are 32 bit only + // - the following offsets use two elements + // each (lower followed by upper 32 bit) + // to represent the 64 bit value +}; + + +inline uint64 +PackageFileHeapAccessorBase::OffsetArray::operator[](size_t index) const +{ + if (index == 0) + return 0; + + if (fOffsets[0] == 0 || index < fOffsets[0]) + return fOffsets[index]; + + index += index - fOffsets[0]; + return fOffsets[index] | ((uint64)fOffsets[index + 1] << 32); +} + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit + + +#endif // _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_ACCESSOR_BASE_H_ diff --git a/headers/private/package/hpkg/PackageFileHeapReader.h b/headers/private/package/hpkg/PackageFileHeapReader.h new file mode 100644 index 0000000000..de1c36e2ad --- /dev/null +++ b/headers/private/package/hpkg/PackageFileHeapReader.h @@ -0,0 +1,57 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_READER_H_ +#define _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_READER_H_ + + +#include +#include + + +namespace BPackageKit { + +namespace BHPKG { + + +class BDataReader; +class BErrorOutput; + + +namespace BPrivate { + + +class PackageFileHeapReader : public PackageFileHeapAccessorBase { +public: + PackageFileHeapReader(BErrorOutput* errorOutput, + int fd, off_t heapOffset, + off_t compressedHeapSize, + uint64 uncompressedHeapSize); + ~PackageFileHeapReader(); + + status_t Init(); + + PackageFileHeapReader* Clone() const; + + const OffsetArray& Offsets() const + { return fOffsets; } + +protected: + virtual status_t ReadAndDecompressChunk(size_t chunkIndex, + void* compressedDataBuffer, + void* uncompressedDataBuffer); + +private: + OffsetArray fOffsets; +}; + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit + + +#endif // _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_READER_H_ diff --git a/headers/private/package/hpkg/PackageFileHeapWriter.h b/headers/private/package/hpkg/PackageFileHeapWriter.h new file mode 100644 index 0000000000..3b4d24e590 --- /dev/null +++ b/headers/private/package/hpkg/PackageFileHeapWriter.h @@ -0,0 +1,99 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_WRITER_H_ +#define _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_WRITER_H_ + + +#include +#include +#include + + +namespace BPrivate { + template class RangeArray; +} + + +namespace BPackageKit { + +namespace BHPKG { + + +class BDataReader; +class BErrorOutput; + + +namespace BPrivate { + + +class PackageFileHeapReader; + + +class PackageFileHeapWriter : public PackageFileHeapAccessorBase, + private AbstractDataWriter { +public: + PackageFileHeapWriter(BErrorOutput* errorOutput, + int fd, off_t heapOffset); + ~PackageFileHeapWriter(); + + void Init(); + void Reinit(PackageFileHeapReader* heapReader); + + AbstractDataWriter* DataWriter() + { return this; } + + status_t AddData(BDataReader& dataReader, off_t size, + uint64& _offset); + void RemoveDataRanges( + const ::BPrivate::RangeArray& + ranges); + // doesn't truncate the file + status_t Finish(); + +protected: + virtual status_t ReadAndDecompressChunk(size_t chunkIndex, + void* compressedDataBuffer, + void* uncompressedDataBuffer); + +private: + // AbstractDataWriter + virtual status_t WriteDataNoThrow(const void* buffer, + size_t size); + +private: + struct Chunk; + struct ChunkSegment; + struct ChunkBuffer; + +private: + void _Uninit(); + + status_t _FlushPendingData(); + status_t _WriteChunk(const void* data, size_t size, + bool mayCompress); + status_t _WriteDataCompressed(const void* data, + size_t size); + status_t _WriteDataUncompressed(const void* data, + size_t size); + + void _PushChunks(ChunkBuffer& chunkBuffer, + uint64 startOffset, uint64 endOffset); + +private: + void* fPendingDataBuffer; + void* fCompressedDataBuffer; + size_t fPendingDataSize; + Array fOffsets; +}; + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit + + +#endif // _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_WRITER_H_ diff --git a/headers/private/package/hpkg/PackageReaderImpl.h b/headers/private/package/hpkg/PackageReaderImpl.h index b35ac5ec08..7dd0b82cf1 100644 --- a/headers/private/package/hpkg/PackageReaderImpl.h +++ b/headers/private/package/hpkg/PackageReaderImpl.h @@ -25,8 +25,7 @@ namespace BPrivate { class PackageReaderImpl : public ReaderImplBase { typedef ReaderImplBase inherited; public: - PackageReaderImpl( - BErrorOutput* errorOutput); + PackageReaderImpl(BErrorOutput* errorOutput); ~PackageReaderImpl(); status_t Init(const char* fileName); @@ -41,13 +40,15 @@ public: uint64 HeapOffset() const; uint64 HeapSize() const; + PackageFileHeapReader* HeapReader() const + { return inherited::HeapReader(); } + protected: // from ReaderImplBase virtual status_t ReadAttributeValue(uint8 type, uint8 encoding, AttributeValue& _value); private: - struct DataAttributeHandler; struct AttributeAttributeHandler; struct EntryAttributeHandler; struct RootAttributeHandler; @@ -59,11 +60,10 @@ private: status_t _GetTOCBuffer(size_t size, const void*& _buffer); private: - uint64 fTotalSize; uint64 fHeapOffset; uint64 fHeapSize; - SectionInfo fTOCSection; + PackageFileSection fTOCSection; }; diff --git a/headers/private/package/hpkg/PackageWriterImpl.h b/headers/private/package/hpkg/PackageWriterImpl.h index 88febb119b..306f5ccd26 100644 --- a/headers/private/package/hpkg/PackageWriterImpl.h +++ b/headers/private/package/hpkg/PackageWriterImpl.h @@ -35,6 +35,8 @@ namespace BPrivate { struct hpkg_header; +class PackageFileHeapWriter; + class PackageWriterImpl : public WriterImplBase { typedef WriterImplBase inherited; @@ -78,30 +80,16 @@ private: Entry* entry, const char* fileName, char* pathBuffer); void _CompactHeap(); - void _MoveHeapChunk(off_t fromOffset, off_t toOffset, - off_t size); void _AttributeRemoved(Attribute* attribute); - void _WriteTOC(hpkg_header& header); - int32 _WriteTOCCompressed( - uint64& _uncompressedStringsSize, - uint64& _uncompressedMainSize, - uint64& _tocUncompressedSize); - int32 _WriteTOCUncompressed( - uint64& _uncompressedStringsSize, - uint64& _uncompressedMainSize, - uint64& _tocUncompressedSize); - int32 _WriteTOCSections(uint64& _stringsSize, - uint64& _mainSize); + void _WriteTOC(hpkg_header& header, uint64& _length); void _WriteAttributeChildren(Attribute* attribute); - void _WritePackageAttributes(hpkg_header& header); + void _WritePackageAttributes(hpkg_header& header, + uint64& _length); uint32 _WritePackageAttributesCompressed( uint32& _stringsLengthUncompressed, uint32& _attributesLengthUncompressed); - uint32 _WritePackageAttributesUncompressed( - uint32& _stringsLengthUncompressed, - uint32& _attributesLengthUncompressed); void _AddEntry(int dirFD, Entry* entry, const char* fileName, char* pathBuffer); @@ -125,8 +113,6 @@ private: status_t _AddData(BDataReader& dataReader, off_t size); - status_t _WriteUncompressedData(BDataReader& dataReader, - off_t size, uint64 writeOffset); status_t _WriteZlibCompressedData( BDataReader& dataReader, off_t size, uint64 writeOffset, @@ -136,12 +122,9 @@ private: BPackageWriterListener* fListener; off_t fHeapOffset; - off_t fHeapEnd; + uint16 fHeaderSize; - ::BPrivate::RangeArray* fHeapRangesToRemove; - - void* fDataBuffer; - const size_t fDataBufferSize; + ::BPrivate::RangeArray* fHeapRangesToRemove; Entry* fRootEntry; diff --git a/headers/private/package/hpkg/ReaderImplBase.h b/headers/private/package/hpkg/ReaderImplBase.h index 75273e92c2..0ed343eb82 100644 --- a/headers/private/package/hpkg/ReaderImplBase.h +++ b/headers/private/package/hpkg/ReaderImplBase.h @@ -7,6 +7,10 @@ #define _PACKAGE__HPKG__PRIVATE__READER_IMPL_BASE_H_ +#include +#include + +#include #include #include @@ -27,176 +31,73 @@ class BErrorOutput; namespace BPrivate { +class PackageFileHeapReader; + + +struct PackageFileSection { + uint32 uncompressedLength; + uint8* data; + uint64 offset; + uint64 currentOffset; + uint64 stringsLength; + uint64 stringsCount; + char** strings; + const char* name; + + PackageFileSection(const char* _name) + : + data(NULL), + strings(NULL), + name(_name) + { + } + + ~PackageFileSection() + { + delete[] strings; + delete[] data; + } +}; + + class ReaderImplBase { protected: - ReaderImplBase( + ReaderImplBase(const char* fileType, BErrorOutput* errorOutput); virtual ~ReaderImplBase(); - virtual status_t Init(int fd, bool keepFD); - int FD() const; BErrorOutput* ErrorOutput() const; + PackageFileHeapReader* HeapReader() const + { return fHeapReader; } + protected: - struct AttributeHandlerContext { - BErrorOutput* errorOutput; - union { - BPackageContentHandler* packageContentHandler; - BLowLevelPackageContentHandler* lowLevelHandler; - }; - bool hasLowLevelHandler; - - uint64 heapOffset; - uint64 heapSize; - - BHPKGPackageSectionID section; - - AttributeHandlerContext(BErrorOutput* errorOutput, - BPackageContentHandler* packageContentHandler, - BHPKGPackageSectionID section); - - AttributeHandlerContext(BErrorOutput* errorOutput, - BLowLevelPackageContentHandler* lowLevelHandler, - BHPKGPackageSectionID section); - - void ErrorOccurred(); - }; - + class AttributeHandlerContext; + class AttributeHandler; + class IgnoreAttributeHandler; + class PackageVersionAttributeHandler; + class PackageResolvableAttributeHandler; + class PackageResolvableExpressionAttributeHandler; + class PackageAttributeHandler; + class LowLevelAttributeHandler; typedef BPackageAttributeValue AttributeValue; - - struct AttributeHandler - : SinglyLinkedListLinkImpl { - virtual ~AttributeHandler(); - - void SetLevel(int level); - virtual status_t HandleAttribute( - AttributeHandlerContext* context, uint8 id, - const AttributeValue& value, AttributeHandler** _handler); - - virtual status_t Delete(AttributeHandlerContext* context); - - protected: - int fLevel; - }; - - - struct IgnoreAttributeHandler : AttributeHandler { - }; - - - struct PackageVersionAttributeHandler : AttributeHandler { - PackageVersionAttributeHandler( - BPackageInfoAttributeValue& packageInfoValue, - BPackageVersionData& versionData, bool notify); - - virtual status_t HandleAttribute( - AttributeHandlerContext* context, uint8 id, - const AttributeValue& value, AttributeHandler** _handler); - - virtual status_t Delete(AttributeHandlerContext* context); - - private: - BPackageInfoAttributeValue& fPackageInfoValue; - BPackageVersionData& fPackageVersionData; - bool fNotify; - }; - - - struct PackageResolvableAttributeHandler : AttributeHandler { - PackageResolvableAttributeHandler( - BPackageInfoAttributeValue& packageInfoValue); - - virtual status_t HandleAttribute( - AttributeHandlerContext* context, uint8 id, - const AttributeValue& value, AttributeHandler** _handler); - - virtual status_t Delete(AttributeHandlerContext* context); - - private: - BPackageInfoAttributeValue& fPackageInfoValue; - }; - - - struct PackageResolvableExpressionAttributeHandler - : AttributeHandler { - PackageResolvableExpressionAttributeHandler( - BPackageInfoAttributeValue& packageInfoValue); - - virtual status_t HandleAttribute( - AttributeHandlerContext* context, uint8 id, - const AttributeValue& value, AttributeHandler** _handler); - - virtual status_t Delete(AttributeHandlerContext* context); - - private: - BPackageInfoAttributeValue& fPackageInfoValue; - }; - - - struct PackageAttributeHandler : AttributeHandler { - virtual status_t HandleAttribute( - AttributeHandlerContext* context, uint8 id, - const AttributeValue& value, AttributeHandler** _handler); - - private: - BPackageInfoAttributeValue fPackageInfoValue; - }; - - - struct LowLevelAttributeHandler : AttributeHandler { - LowLevelAttributeHandler(); - LowLevelAttributeHandler(uint8 id, - const BPackageAttributeValue& value, void* parentToken, - void* token); - - virtual status_t HandleAttribute( - AttributeHandlerContext* context, uint8 id, - const AttributeValue& value, AttributeHandler** _handler); - virtual status_t Delete(AttributeHandlerContext* context); - - private: - void* fParentToken; - void* fToken; - uint8 fID; - AttributeValue fValue; - }; - - - struct SectionInfo { - uint32 compression; - uint32 compressedLength; - uint32 uncompressedLength; - uint8* data; - uint64 offset; - uint64 currentOffset; - uint64 stringsLength; - uint64 stringsCount; - char** strings; - const char* name; - - SectionInfo(const char* _name) - : - data(NULL), - strings(NULL), - name(_name) - { - } - - ~SectionInfo() - { - delete[] strings; - delete[] data; - } - }; - typedef SinglyLinkedList AttributeHandlerList; protected: - const char* CheckCompression( - const SectionInfo& section) const; + template + status_t Init(int fd, bool keepFD, Header& header); + status_t InitHeapReader(uint32 compression, + uint32 chunkSize, off_t offset, + uint64 compressedSize, + uint64 uncompressedSize); + status_t InitSection(PackageFileSection& section, + uint64 endOffset, uint64 length, + uint64 maxSaneLength, uint64 stringsLength, + uint64 stringsCount); + status_t PrepareSection(PackageFileSection& section); status_t ParseStrings(); @@ -214,8 +115,7 @@ protected: status_t ReadBuffer(off_t offset, void* buffer, size_t size); - status_t ReadCompressedBuffer( - const SectionInfo& section); + status_t ReadSection(const PackageFileSection& section); inline AttributeHandler* CurrentAttributeHandler() const; inline void PushAttributeHandler( @@ -223,13 +123,15 @@ protected: inline AttributeHandler* PopAttributeHandler(); inline void ClearAttributeHandlerStack(); - inline SectionInfo* CurrentSection(); - inline void SetCurrentSection(SectionInfo* section); + inline PackageFileSection* CurrentSection(); + inline void SetCurrentSection(PackageFileSection* section); protected: - SectionInfo fPackageAttributesSection; + PackageFileSection fPackageAttributesSection; private: + status_t _Init(int fd, bool keepFD); + status_t _ParseAttributeTree( AttributeHandlerContext* context); @@ -247,11 +149,14 @@ private: size_t* _stringLength = NULL); private: + const char* fFileType; BErrorOutput* fErrorOutput; int fFD; bool fOwnsFD; - SectionInfo* fCurrentSection; + PackageFileHeapReader* fHeapReader; + + PackageFileSection* fCurrentSection; AttributeHandlerList fAttributeHandlerStack; @@ -260,6 +165,232 @@ private: }; +// #pragma mark - attribute handlers + + +class ReaderImplBase::AttributeHandlerContext { +public: + BErrorOutput* errorOutput; + union { + BPackageContentHandler* packageContentHandler; + BLowLevelPackageContentHandler* lowLevelHandler; + }; + bool hasLowLevelHandler; + + BHPKGPackageSectionID section; + +public: + AttributeHandlerContext( + BErrorOutput* errorOutput, + BPackageContentHandler* + packageContentHandler, + BHPKGPackageSectionID section); + AttributeHandlerContext( + BErrorOutput* errorOutput, + BLowLevelPackageContentHandler* + lowLevelHandler, + BHPKGPackageSectionID section); + + void ErrorOccurred(); +}; + + +class ReaderImplBase::AttributeHandler + : public SinglyLinkedListLinkImpl { +public: + virtual ~AttributeHandler(); + + void SetLevel(int level); + virtual status_t HandleAttribute( + AttributeHandlerContext* context, uint8 id, + const AttributeValue& value, + AttributeHandler** _handler); + + virtual status_t Delete(AttributeHandlerContext* context); + +protected: + int fLevel; +}; + + +class ReaderImplBase::IgnoreAttributeHandler : public AttributeHandler { +}; + + +class ReaderImplBase::PackageVersionAttributeHandler : public AttributeHandler { +public: + PackageVersionAttributeHandler( + BPackageInfoAttributeValue& + packageInfoValue, + BPackageVersionData& versionData, + bool notify); + + virtual status_t HandleAttribute( + AttributeHandlerContext* context, uint8 id, + const AttributeValue& value, + AttributeHandler** _handler); + + virtual status_t Delete(AttributeHandlerContext* context); + +private: + BPackageInfoAttributeValue& fPackageInfoValue; + BPackageVersionData& fPackageVersionData; + bool fNotify; +}; + + +class ReaderImplBase::PackageResolvableAttributeHandler + : public AttributeHandler { +public: + PackageResolvableAttributeHandler( + BPackageInfoAttributeValue& + packageInfoValue); + + virtual status_t HandleAttribute( + AttributeHandlerContext* context, uint8 id, + const AttributeValue& value, + AttributeHandler** _handler); + + virtual status_t Delete(AttributeHandlerContext* context); + +private: + BPackageInfoAttributeValue& fPackageInfoValue; +}; + + +class ReaderImplBase::PackageResolvableExpressionAttributeHandler + : public AttributeHandler { +public: + PackageResolvableExpressionAttributeHandler( + BPackageInfoAttributeValue& + packageInfoValue); + + virtual status_t HandleAttribute( + AttributeHandlerContext* context, uint8 id, + const AttributeValue& value, + AttributeHandler** _handler); + + virtual status_t Delete(AttributeHandlerContext* context); + +private: + BPackageInfoAttributeValue& fPackageInfoValue; +}; + + +class ReaderImplBase::PackageAttributeHandler : public AttributeHandler { +public: + virtual status_t HandleAttribute( + AttributeHandlerContext* context, uint8 id, + const AttributeValue& value, + AttributeHandler** _handler); + +private: + BPackageInfoAttributeValue fPackageInfoValue; +}; + + +class ReaderImplBase::LowLevelAttributeHandler : public AttributeHandler { +public: + LowLevelAttributeHandler(); + LowLevelAttributeHandler(uint8 id, + const BPackageAttributeValue& value, + void* parentToken, void* token); + + virtual status_t HandleAttribute( + AttributeHandlerContext* context, uint8 id, + const AttributeValue& value, + AttributeHandler** _handler); + virtual status_t Delete(AttributeHandlerContext* context); + +private: + void* fParentToken; + void* fToken; + uint8 fID; + AttributeValue fValue; +}; + + +// #pragma mark - template and inline methods + + +template +status_t +ReaderImplBase::Init(int fd, bool keepFD, Header& header) +{ + status_t error = _Init(fd, keepFD); + if (error != B_OK) + return error; + + // stat the file + struct stat st; + if (fstat(FD(), &st) < 0) { + ErrorOutput()->PrintError("Error: Failed to access %s file: %s\n", + fFileType, strerror(errno)); + return errno; + } + + // read the header + if ((error = ReadBuffer(0, &header, sizeof(header))) != B_OK) + return error; + + // check the header + + // magic + if (B_BENDIAN_TO_HOST_INT32(header.magic) != kMagic) { + ErrorOutput()->PrintError("Error: Invalid %s file: Invalid " + "magic\n", fFileType); + return B_BAD_DATA; + } + + // version + if (B_BENDIAN_TO_HOST_INT16(header.version) != kVersion) { + ErrorOutput()->PrintError("Error: Invalid/unsupported %s file " + "version (%d)\n", fFileType, + B_BENDIAN_TO_HOST_INT16(header.version)); + return B_MISMATCHED_VALUES; + } + + // header size + uint64 heapOffset = B_BENDIAN_TO_HOST_INT16(header.header_size); + if (heapOffset < (off_t)sizeof(header)) { + ErrorOutput()->PrintError("Error: Invalid %s file: Invalid header " + "size (%" B_PRIu64 ")\n", fFileType, heapOffset); + return B_BAD_DATA; + } + + // total size + uint64 totalSize = B_BENDIAN_TO_HOST_INT64(header.total_size); + if (totalSize != (uint64)st.st_size) { + ErrorOutput()->PrintError("Error: Invalid %s file: Total size in " + "header (%" B_PRIu64 ") doesn't agree with total file size (%" + B_PRIdOFF ")\n", fFileType, totalSize, st.st_size); + return B_BAD_DATA; + } + + // heap size + uint64 compressedHeapSize + = B_BENDIAN_TO_HOST_INT64(header.heap_size_compressed); + if (compressedHeapSize > totalSize + || heapOffset > totalSize - compressedHeapSize) { + ErrorOutput()->PrintError("Error: Invalid %s file: Heap size in " + "header (%" B_PRIu64 ") doesn't agree with total file size (%" + B_PRIu64 ") and heap offset (%" B_PRIu64 ")\n", fFileType, + compressedHeapSize, totalSize, heapOffset); + return B_BAD_DATA; + } + + error = InitHeapReader( + B_BENDIAN_TO_HOST_INT32(header.heap_compression), + B_BENDIAN_TO_HOST_INT32(header.heap_chunk_size), heapOffset, + compressedHeapSize, + B_BENDIAN_TO_HOST_INT64(header.heap_size_uncompressed)); + if (error != B_OK) + return error; + + return B_OK; +} + + inline int ReaderImplBase::FD() const { @@ -274,7 +405,7 @@ ReaderImplBase::ErrorOutput() const } -ReaderImplBase::SectionInfo* +PackageFileSection* ReaderImplBase::CurrentSection() { return fCurrentSection; @@ -282,7 +413,7 @@ ReaderImplBase::CurrentSection() void -ReaderImplBase::SetCurrentSection(SectionInfo* section) +ReaderImplBase::SetCurrentSection(PackageFileSection* section) { fCurrentSection = section; } diff --git a/headers/private/package/hpkg/RepositoryReaderImpl.h b/headers/private/package/hpkg/RepositoryReaderImpl.h index 5e17a009ab..4d3951282f 100644 --- a/headers/private/package/hpkg/RepositoryReaderImpl.h +++ b/headers/private/package/hpkg/RepositoryReaderImpl.h @@ -25,8 +25,7 @@ namespace BPrivate { class RepositoryReaderImpl : public ReaderImplBase { typedef ReaderImplBase inherited; public: - RepositoryReaderImpl( - BErrorOutput* errorOutput); + RepositoryReaderImpl(BErrorOutput* errorOutput); ~RepositoryReaderImpl(); status_t Init(const char* fileName); @@ -42,7 +41,6 @@ private: struct RootAttributeHandler; private: - SectionInfo fRepositoryInfoSection; BRepositoryInfo fRepositoryInfo; }; diff --git a/headers/private/package/hpkg/RepositoryWriterImpl.h b/headers/private/package/hpkg/RepositoryWriterImpl.h index 0d538448e0..3909202fd0 100644 --- a/headers/private/package/hpkg/RepositoryWriterImpl.h +++ b/headers/private/package/hpkg/RepositoryWriterImpl.h @@ -50,10 +50,9 @@ private: status_t _RegisterCurrentPackageInfo(); status_t _WriteRepositoryInfo(hpkg_repo_header& header, - ssize_t& _infoLengthCompressed); - off_t _WritePackageAttributes( - hpkg_repo_header& header, off_t startOffset, - ssize_t& _packagesLengthCompressed); + uint64& _length); + void _WritePackageAttributes( + hpkg_repo_header& header, uint64& _length); struct PackageNameSet; diff --git a/headers/private/package/hpkg/WriterImplBase.h b/headers/private/package/hpkg/WriterImplBase.h index 54858bf7b5..0965bcf731 100644 --- a/headers/private/package/hpkg/WriterImplBase.h +++ b/headers/private/package/hpkg/WriterImplBase.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -29,16 +30,19 @@ class BErrorOutput; namespace BPrivate { +class AbstractDataWriter; +class PackageFileHeapWriter; + struct hpkg_header; + class WriterImplBase { public: - WriterImplBase(BErrorOutput* errorOutput); + WriterImplBase(const char* fileType, + BErrorOutput* errorOutput); ~WriterImplBase(); protected: - - struct AttributeValue { union { int64 signedInt; @@ -92,63 +96,10 @@ protected: void _DeleteChildren(); }; - - struct AbstractDataWriter { - AbstractDataWriter(); - virtual ~AbstractDataWriter(); - - uint64 BytesWritten() const; - - virtual status_t WriteDataNoThrow(const void* buffer, - size_t size) = 0; - - void WriteDataThrows(const void* buffer, size_t size); - - protected: - uint64 fBytesWritten; - }; - - - struct FDDataWriter : AbstractDataWriter { - FDDataWriter(int fd, off_t offset, BErrorOutput* errorOutput); - - virtual status_t WriteDataNoThrow(const void* buffer, - size_t size); - - off_t Offset() const; - - private: - int fFD; - off_t fOffset; - BErrorOutput* fErrorOutput; - }; - - - struct ZlibDataWriter : AbstractDataWriter, private BDataOutput { - ZlibDataWriter(AbstractDataWriter* dataWriter); - - void Init(); - - void Finish(); - - virtual status_t WriteDataNoThrow(const void* buffer, - size_t size); - - private: - // BDataOutput - virtual status_t WriteData(const void* buffer, size_t size); - - private: - AbstractDataWriter* fDataWriter; - ZlibCompressor fCompressor; - }; - - typedef DoublyLinkedList PackageAttributeList; protected: - status_t Init(const char* fileName, const char* type, - uint32 flags); + status_t Init(const char* fileName, uint32 flags); void RegisterPackageInfo( PackageAttributeList& attributeList, @@ -180,13 +131,16 @@ protected: void WriteAttributeValue(const AttributeValue& value, uint8 encoding); void WriteUnsignedLEB128(uint64 value); - inline void WriteString(const char* string); template inline void Write(const Type& value); + inline void WriteString(const char* string); + inline void WriteBuffer(const void* data, size_t size); + // appends data to the heap - void WriteBuffer(const void* buffer, size_t size, + void RawWriteBuffer(const void* buffer, size_t size, off_t offset); + // writes to the file directly inline int FD() const; inline uint32 Flags() const; @@ -197,11 +151,11 @@ protected: inline const StringCache& PackageStringCache() const; inline StringCache& PackageStringCache(); - inline AbstractDataWriter* DataWriter() const; - inline void SetDataWriter(AbstractDataWriter* dataWriter); - inline void SetFinished(bool finished); +protected: + PackageFileHeapWriter* fHeapWriter; + private: static const BHPKGAttributeID kDefaultVersionAttributeID = B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MAJOR; @@ -211,6 +165,7 @@ private: const PackageAttributeList& attributes); private: + const char* fFileType; BErrorOutput* fErrorOutput; const char* fFileName; uint32 fFlags; @@ -224,41 +179,25 @@ private: }; -inline uint64 -WriterImplBase::AbstractDataWriter::BytesWritten() const -{ - return fBytesWritten; -} - - -inline void -WriterImplBase::AbstractDataWriter::WriteDataThrows(const void* buffer, - size_t size) -{ - status_t error = WriteDataNoThrow(buffer, size); - if (error != B_OK) - throw status_t(error); -} - -inline off_t -WriterImplBase::FDDataWriter::Offset() const -{ - return fOffset; -} - - template inline void WriterImplBase::Write(const Type& value) { - fDataWriter->WriteDataThrows(&value, sizeof(Type)); + WriteBuffer(&value, sizeof(Type)); } inline void WriterImplBase::WriteString(const char* string) { - fDataWriter->WriteDataThrows(string, strlen(string) + 1); + WriteBuffer(string, strlen(string) + 1); +} + + +inline void +WriterImplBase::WriteBuffer(const void* data, size_t size) +{ + fDataWriter->WriteDataThrows(data, size); } @@ -276,20 +215,6 @@ WriterImplBase::Flags() const } -inline WriterImplBase::AbstractDataWriter* -WriterImplBase::DataWriter() const -{ - return fDataWriter; -} - - -inline void -WriterImplBase::SetDataWriter(AbstractDataWriter* dataWriter) -{ - fDataWriter = dataWriter; -} - - inline const WriterImplBase::PackageAttributeList& WriterImplBase::PackageAttributes() const { diff --git a/src/add-ons/kernel/file_systems/packagefs/Jamfile b/src/add-ons/kernel/file_systems/packagefs/Jamfile index 28f9826890..e2061a8a75 100644 --- a/src/add-ons/kernel/file_systems/packagefs/Jamfile +++ b/src/add-ons/kernel/file_systems/packagefs/Jamfile @@ -83,6 +83,8 @@ HAIKU_PACKAGE_FS_PACKAGE_READER_SOURCES = PackageDataReader.cpp PackageEntry.cpp PackageEntryAttribute.cpp + PackageFileHeapAccessorBase.cpp + PackageFileHeapReader.cpp PackageReaderImpl.cpp ReaderImplBase.cpp diff --git a/src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp b/src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp index f8bb5b0e7d..81ec8cbc18 100644 --- a/src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp @@ -28,17 +28,9 @@ using BPackageKit::BHPKG::BFDDataReader; static status_t -read_package_data(const PackageData& data, BDataReader* dataReader, - off_t offset, void* buffer, size_t* bufferSize) +read_package_data(const PackageData& data, BDataReader* reader, off_t offset, + void* buffer, size_t* bufferSize) { - // create a PackageDataReader - BAbstractBufferedDataReader* reader; - status_t error = GlobalFactory::Default()->CreatePackageDataReader( - dataReader, data, reader); - if (error != B_OK) - RETURN_ERROR(error); - ObjectDeleter readerDeleter(reader); - // check the offset if (offset < 0 || (uint64)offset > data.UncompressedSize()) return B_BAD_VALUE; @@ -131,19 +123,20 @@ UnpackingAttributeCookie::ReadAttribute(PackageNode* packageNode, const PackageData& data = attribute->Data(); if (data.IsEncodedInline()) { // inline data - BBufferDataReader dataReader(data.InlineData(), data.CompressedSize()); + BBufferDataReader dataReader(data.InlineData(), + data.UncompressedSize()); return read_package_data(data, &dataReader, offset, buffer, bufferSize); } - // data not inline -- open the package + // data not inline -- let the package create a data reader for us Package* package = packageNode->GetPackage(); - int fd = package->Open(); - if (fd < 0) - RETURN_ERROR(fd); - PackageCloser packageCloser(package); + BAbstractBufferedDataReader* reader; + status_t error = package->CreateDataReader(data, reader); + if (error != B_OK) + return error; + ObjectDeleter readerDeleter(reader); - BFDDataReader dataReader(fd); - return read_package_data(data, &dataReader, offset, buffer, bufferSize); + return read_package_data(data, reader, offset, buffer, bufferSize); } diff --git a/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.cpp b/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.cpp index 9f9d2a3f1e..f7d44141f5 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.cpp @@ -20,13 +20,10 @@ static const uint32 kMaxCachedBuffers = 32; GlobalFactory::GlobalFactory() : - fBufferPool(BPackageKit::BHPKG::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, + fBufferPool(BPackageKit::BHPKG::V1::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, kMaxCachedBuffers), - fPackageDataReaderFactoryV1(&fBufferPool), - fPackageDataReaderFactoryV2(&fBufferPool) + fPackageDataReaderFactory(&fBufferPool) { - STATIC_ASSERT((int)BPackageKit::BHPKG::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB - == (int)BPackageKit::BHPKG::V1::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB); } @@ -73,20 +70,10 @@ GlobalFactory::Default() status_t GlobalFactory::CreatePackageDataReader(BDataReader* dataReader, - const PackageData& data, BAbstractBufferedDataReader*& _reader) + const PackageDataV1& data, BAbstractBufferedDataReader*& _reader) { - switch (data.Version()) { - case 1: - return fPackageDataReaderFactoryV1.CreatePackageDataReader( - dataReader, data.DataV1(), _reader); - - case 2: - return fPackageDataReaderFactoryV2.CreatePackageDataReader( - dataReader, data.DataV2(), _reader); - - default: - return B_NOT_SUPPORTED; - } + return fPackageDataReaderFactory.CreatePackageDataReader(dataReader, data, + _reader); } diff --git a/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.h b/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.h index ca06a9d4b9..46f75da820 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.h +++ b/src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.h @@ -1,27 +1,24 @@ /* - * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2009-2013, Ingo Weinhold, ingo_weinhold@gmx.de. * Distributed under the terms of the MIT License. */ #ifndef GLOBAL_FACTORY_H #define GLOBAL_FACTORY_H -#include #include #include "BlockBufferPoolKernel.h" +#include "PackageData.h" +using BPackageKit::BHPKG::BBufferPool; using BPackageKit::BHPKG::BDataReader; using BPackageKit::BHPKG::BPackageData; using BPackageKit::BHPKG::BAbstractBufferedDataReader; typedef BPackageKit::BHPKG::V1::BPackageDataReaderFactory BPackageDataReaderFactoryV1; -typedef BPackageKit::BHPKG::BPackageDataReaderFactory - BPackageDataReaderFactoryV2; - -class PackageData; class GlobalFactory { @@ -35,7 +32,7 @@ public: static GlobalFactory* Default(); status_t CreatePackageDataReader(BDataReader* dataReader, - const PackageData& data, + const PackageDataV1& data, BAbstractBufferedDataReader*& _reader); private: @@ -45,8 +42,7 @@ private: static GlobalFactory* sDefaultInstance; BlockBufferPoolKernel fBufferPool; - BPackageDataReaderFactoryV1 fPackageDataReaderFactoryV1; - BPackageDataReaderFactoryV2 fPackageDataReaderFactoryV2; + BPackageDataReaderFactoryV1 fPackageDataReaderFactory; }; #endif // GLOBAL_FACTORY_H diff --git a/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp b/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp index ad73f22a02..094b231850 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/package/Package.cpp @@ -12,17 +12,21 @@ #include #include -#include #include +#include #include #include -#include #include #include + +#include +#include +#include #include #include #include "DebugSupport.h" +#include "GlobalFactory.h" #include "PackageDirectory.h" #include "PackageFile.h" #include "PackageSymlink.h" @@ -32,10 +36,12 @@ using namespace BPackageKit; -typedef BPackageKit::BHPKG::BErrorOutput BErrorOutput; -typedef BPackageKit::BHPKG::BPackageInfoAttributeValue - BPackageInfoAttributeValue; -typedef BPackageKit::BHPKG::BPackageVersionData BPackageVersionData; +using BPackageKit::BHPKG::BDataOutput; +using BPackageKit::BHPKG::BErrorOutput; +using BPackageKit::BHPKG::BFDDataReader; +using BPackageKit::BHPKG::BPackageInfoAttributeValue; +using BPackageKit::BHPKG::BPackageVersionData; +using BPackageKit::BHPKG::BPrivate::PackageFileHeapReader; // current format version types typedef BPackageKit::BHPKG::BPackageContentHandler BPackageContentHandler; @@ -581,6 +587,129 @@ private: }; +// #pragma mark - HeapReader + + +struct Package::HeapReader { + virtual ~HeapReader() + { + } + + virtual void UpdateFD(int fd) = 0; + + virtual status_t CreateDataReader(const PackageData& data, + BAbstractBufferedDataReader*& _reader) = 0; +}; + + +// #pragma mark - HeapReaderV1 + + +struct Package::HeapReaderV1 : public HeapReader, private BDataReader { +public: + HeapReaderV1(int fd) + : + fFileReader(fd) + { + } + + ~HeapReaderV1() + { + } + + virtual void UpdateFD(int fd) + { + fFileReader.SetFD(fd); + } + + virtual status_t CreateDataReader(const PackageData& data, + BAbstractBufferedDataReader*& _reader) + { + return GlobalFactory::Default()->CreatePackageDataReader(this, + data.DataV1(), _reader); + } + +private: + // BDataReader + + virtual status_t ReadData(off_t offset, void* buffer, size_t size) + { + return fFileReader.ReadData(offset, buffer, size); + } + +private: + BFDDataReader fFileReader; +}; + + +// #pragma mark - HeapReaderV2 + + +struct Package::HeapReaderV2 : public HeapReader, + private BAbstractBufferedDataReader, private BErrorOutput { +public: + HeapReaderV2() + : + fHeapReader(NULL) + { + } + + ~HeapReaderV2() + { + delete fHeapReader; + } + + status_t Init(const PackageFileHeapReader* heapReader, int fd) + { + fHeapReader = heapReader->Clone(); + if (fHeapReader == NULL) + return B_NO_MEMORY; + + fHeapReader->SetErrorOutput(this); + fHeapReader->SetFD(fd); + + return B_OK; + } + + virtual void UpdateFD(int fd) + { + fHeapReader->SetFD(fd); + } + + virtual status_t CreateDataReader(const PackageData& data, + BAbstractBufferedDataReader*& _reader) + { + return BPackageKit::BHPKG::BPackageDataReaderFactory() + .CreatePackageDataReader(this, data.DataV2(), _reader); + } + +private: + // BAbstractBufferedDataReader + + virtual status_t ReadData(off_t offset, void* buffer, size_t size) + { + return fHeapReader->ReadData(offset, buffer, size); + } + + virtual status_t ReadDataToOutput(off_t offset, size_t size, + BDataOutput* output) + { + return fHeapReader->ReadDataToOutput(offset, size, output); + } + +private: + // BErrorOutput + + virtual void PrintErrorVarArgs(const char* format, va_list args) + { +// TODO:... + } + +private: + PackageFileHeapReader* fHeapReader; +}; + + // #pragma mark - Package @@ -595,6 +724,7 @@ Package::Package(::Volume* volume, dev_t deviceID, ino_t nodeID) fLinkDirectory(NULL), fFD(-1), fOpenCount(0), + fHeapReader(NULL), fNodeID(nodeID), fDeviceID(deviceID) { @@ -604,6 +734,8 @@ Package::Package(::Volume* volume, dev_t deviceID, ino_t nodeID) Package::~Package() { + delete fHeapReader; + while (PackageNode* node = fNodes.RemoveHead()) node->ReleaseReference(); @@ -652,7 +784,22 @@ Package::Load() if (error != B_OK) RETURN_ERROR(error); - RETURN_ERROR(packageReader.ParseContent(&handler)); + error = packageReader.ParseContent(&handler); + if (error != B_OK) + RETURN_ERROR(error); + + // create a heap reader + HeapReaderV2* heapReader = new(std::nothrow) HeapReaderV2; + if (heapReader == NULL) + RETURN_ERROR(B_NO_MEMORY); + + error = heapReader->Init(packageReader.HeapReader(), fd); + if (error != B_OK) { + RETURN_ERROR(error); + } + + fHeapReader = heapReader; + return B_OK; } if (error != B_MISMATCHED_VALUES) @@ -671,7 +818,16 @@ Package::Load() if (error != B_OK) RETURN_ERROR(error); - RETURN_ERROR(packageReader.ParseContent(&handler)); + error = packageReader.ParseContent(&handler); + if (error != B_OK) + RETURN_ERROR(error); + + // create a heap reader + fHeapReader = new(std::nothrow) HeapReaderV1(fd); + if (fHeapReader == NULL) + RETURN_ERROR(B_NO_MEMORY); + + return B_OK; } @@ -765,6 +921,10 @@ Package::Open() } fOpenCount = 1; + + if (fHeapReader != NULL) + fHeapReader->UpdateFD(fFD); + return fFD; } @@ -781,5 +941,19 @@ Package::Close() if (--fOpenCount == 0) { close(fFD); fFD = -1; + + if (fHeapReader != NULL) + fHeapReader->UpdateFD(fFD); } } + + +status_t +Package::CreateDataReader(const PackageData& data, + BAbstractBufferedDataReader*& _reader) +{ + if (fHeapReader == NULL) + return B_BAD_VALUE; + + return fHeapReader->CreateDataReader(data, _reader); +} diff --git a/src/add-ons/kernel/file_systems/packagefs/package/Package.h b/src/add-ons/kernel/file_systems/packagefs/package/Package.h index bb239fad22..b75996bf04 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/Package.h +++ b/src/add-ons/kernel/file_systems/packagefs/package/Package.h @@ -6,6 +6,7 @@ #define PACKAGE_H +#include #include #include @@ -23,6 +24,7 @@ using BPackageKit::BPackageArchitecture; +using BPackageKit::BHPKG::BAbstractBufferedDataReader; class PackageLinkDirectory; @@ -82,6 +84,9 @@ public: int Open(); void Close(); + status_t CreateDataReader(const PackageData& data, + BAbstractBufferedDataReader*& _reader); + const PackageNodeList& Nodes() const { return fNodes; } const ResolvableList& Resolvables() const { return fResolvables; } @@ -92,6 +97,9 @@ private: struct LoaderErrorOutput; struct LoaderContentHandler; struct LoaderContentHandlerV1; + struct HeapReader; + struct HeapReaderV1; + struct HeapReaderV2; private: mutex fLock; @@ -104,6 +112,7 @@ private: PackageLinkDirectory* fLinkDirectory; int fFD; uint32 fOpenCount; + HeapReader* fHeapReader; Package* fFileNameHashTableNext; ino_t fNodeID; dev_t fDeviceID; diff --git a/src/add-ons/kernel/file_systems/packagefs/package/PackageData.h b/src/add-ons/kernel/file_systems/packagefs/package/PackageData.h index 313531e1d9..6597aa30ca 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/PackageData.h +++ b/src/add-ons/kernel/file_systems/packagefs/package/PackageData.h @@ -83,7 +83,7 @@ PackageData::CompressedSize() const { if (fVersion == 1) return DataV1().CompressedSize(); - return DataV2().CompressedSize(); + return DataV2().Size(); } @@ -92,7 +92,7 @@ PackageData::UncompressedSize() const { if (fVersion == 1) return DataV1().UncompressedSize(); - return DataV2().UncompressedSize(); + return DataV2().Size(); } diff --git a/src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp b/src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp index 297515468d..08106735b7 100644 --- a/src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp +++ b/src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp @@ -48,10 +48,10 @@ private: struct PackageFile::DataAccessor { - DataAccessor(PackageData* data) + DataAccessor(Package* package, PackageData* data) : + fPackage(package), fData(data), - fDataReader(NULL), fReader(NULL), fFileCache(NULL) { @@ -62,27 +62,15 @@ struct PackageFile::DataAccessor { { file_cache_delete(fFileCache); delete fReader; - delete fDataReader; mutex_destroy(&fLock); } status_t Init(dev_t deviceID, ino_t nodeID, int fd) { - // create a BDataReader for the compressed data - if (fData->IsEncodedInline()) { - fDataReader = new(std::nothrow) BBufferDataReader( - fData->InlineData(), fData->CompressedSize()); - } else - fDataReader = new(std::nothrow) BFDDataReader(fd); - - if (fDataReader == NULL) - RETURN_ERROR(B_NO_MEMORY); - - // create a PackageDataReader - status_t error = GlobalFactory::Default()->CreatePackageDataReader( - fDataReader, *fData, fReader); + // create a reader for the data + status_t error = fPackage->CreateDataReader(*fData, fReader); if (error != B_OK) - RETURN_ERROR(error); + return error; // create a file cache fFileCache = file_cache_create(deviceID, nodeID, @@ -128,8 +116,8 @@ struct PackageFile::DataAccessor { private: mutex fLock; + Package* fPackage; PackageData* fData; - BDataReader* fDataReader; BAbstractBufferedDataReader* fReader; void* fFileCache; }; @@ -168,7 +156,7 @@ PackageFile::VFSInit(dev_t deviceID, ino_t nodeID) PackageCloser packageCloser(fPackage); // create the data accessor - fDataAccessor = new(std::nothrow) DataAccessor(&fData); + fDataAccessor = new(std::nothrow) DataAccessor(GetPackage(), &fData); if (fDataAccessor == NULL) RETURN_ERROR(B_NO_MEMORY); diff --git a/src/bin/package/command_dump.cpp b/src/bin/package/command_dump.cpp index c25437e63b..93401cdf2e 100644 --- a/src/bin/package/command_dump.cpp +++ b/src/bin/package/command_dump.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2009-2013, Ingo Weinhold, ingo_weinhold@gmx.de. * Distributed under the terms of the MIT License. */ diff --git a/src/bin/package/command_extract.cpp b/src/bin/package/command_extract.cpp index 9d85a1fc73..ae5a6070bd 100644 --- a/src/bin/package/command_extract.cpp +++ b/src/bin/package/command_extract.cpp @@ -44,7 +44,9 @@ using BPackageKit::BHPKG::BAbstractBufferedDataReader; using BPackageKit::BHPKG::BBlockBufferPoolNoLock; using BPackageKit::BHPKG::BBufferDataReader; +using BPackageKit::BHPKG::BBufferPool; using BPackageKit::BHPKG::BDataReader; +using BPackageKit::BHPKG::BErrorOutput; using BPackageKit::BHPKG::BFDDataReader; using BPackageKit::BHPKG::BPackageInfoAttributeValue; using BPackageKit::BHPKG::BStandardErrorOutput; @@ -54,38 +56,93 @@ struct VersionPolicyV1 { typedef BPackageKit::BHPKG::V1::BPackageContentHandler PackageContentHandler; typedef BPackageKit::BHPKG::V1::BPackageData PackageData; - typedef BPackageKit::BHPKG::V1::BPackageDataReaderFactory - PackageDataReaderFactory; typedef BPackageKit::BHPKG::V1::BPackageEntry PackageEntry; typedef BPackageKit::BHPKG::V1::BPackageEntryAttribute PackageEntryAttribute; typedef BPackageKit::BHPKG::V1::BPackageReader PackageReader; + typedef BDataReader HeapReaderBase; - static const int kDefaultDataChunkSize - = BPackageKit::BHPKG::V1::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB; + static inline size_t BufferSize() + { + return BPackageKit::BHPKG::V1::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB; + } static inline const char* PackageInfoFileName() { return BPackageKit::BHPKG::V1::B_HPKG_PACKAGE_INFO_FILE_NAME; } + + static inline uint64 PackageDataCompressedSize(const PackageData& data) + { + return data.CompressedSize(); + } + + static inline uint64 PackageDataUncompressedSize(const PackageData& data) + { + return data.UncompressedSize(); + } + + static status_t GetHeapReader(PackageReader& packageReader, + HeapReaderBase*& _heapReader, bool& _mustDelete) + { + _heapReader = new(std::nothrow) BFDDataReader( + packageReader.PackageFileFD()); + _mustDelete = false; + return _heapReader != NULL ? B_OK : B_NO_MEMORY; + } + + static status_t CreatePackageDataReader(BBufferPool* bufferPool, + HeapReaderBase* heapReader, const PackageData& data, + BAbstractBufferedDataReader*& _reader) + { + return BPackageKit::BHPKG::V1::BPackageDataReaderFactory(bufferPool) + .CreatePackageDataReader(heapReader, data, _reader); + } }; struct VersionPolicyV2 { typedef BPackageKit::BHPKG::BPackageContentHandler PackageContentHandler; typedef BPackageKit::BHPKG::BPackageData PackageData; - typedef BPackageKit::BHPKG::BPackageDataReaderFactory - PackageDataReaderFactory; typedef BPackageKit::BHPKG::BPackageEntry PackageEntry; typedef BPackageKit::BHPKG::BPackageEntryAttribute PackageEntryAttribute; typedef BPackageKit::BHPKG::BPackageReader PackageReader; + typedef BAbstractBufferedDataReader HeapReaderBase; - static const int kDefaultDataChunkSize - = BPackageKit::BHPKG::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB; + static inline size_t BufferSize() + { + return 64 * 1024; + } static inline const char* PackageInfoFileName() { return BPackageKit::BHPKG::B_HPKG_PACKAGE_INFO_FILE_NAME; } + + static inline uint64 PackageDataCompressedSize(const PackageData& data) + { + return data.Size(); + } + + static inline uint64 PackageDataUncompressedSize(const PackageData& data) + { + return data.Size(); + } + + static status_t GetHeapReader(PackageReader& packageReader, + HeapReaderBase*& _heapReader, bool& _mustDelete) + { + _heapReader = packageReader.HeapReader(); + _mustDelete = false; + return B_OK; + } + + static status_t CreatePackageDataReader(BBufferPool* bufferPool, + HeapReaderBase* heapReader, const PackageData& data, + BAbstractBufferedDataReader*& _reader) + { + return BPackageKit::BHPKG::BPackageDataReaderFactory() + .CreatePackageDataReader(heapReader, data, _reader); + } }; @@ -227,10 +284,11 @@ private: template struct PackageContentExtractHandler : VersionPolicy::PackageContentHandler { - PackageContentExtractHandler(int packageFileFD) + PackageContentExtractHandler(BBufferPool* bufferPool, + typename VersionPolicy::HeapReaderBase* heapReader) : - fBufferPool(VersionPolicy::kDefaultDataChunkSize, 2), - fPackageFileReader(packageFileFD), + fBufferPool(bufferPool), + fPackageFileReader(heapReader), fDataBuffer(NULL), fDataBufferSize(0), fRootFilterEntry(NULL, NULL, true), @@ -247,11 +305,7 @@ struct PackageContentExtractHandler : VersionPolicy::PackageContentHandler { status_t Init() { - status_t error = fBufferPool.Init(); - if (error != B_OK) - return error; - - error = fRootFilterEntry.Init(); + status_t error = fRootFilterEntry.Init(); if (error != B_OK) return error; @@ -434,15 +488,8 @@ struct PackageContentExtractHandler : VersionPolicy::PackageContentHandler { } // write data - status_t error; - const typename VersionPolicy::PackageData& data = entry->Data(); - if (data.IsEncodedInline()) { - BBufferDataReader dataReader(data.InlineData(), - data.CompressedSize()); - error = _ExtractFileData(&dataReader, data, fd); - } else - error = _ExtractFileData(&fPackageFileReader, data, fd); - + status_t error = _ExtractFileData(fPackageFileReader, entry->Data(), + fd); if (error != B_OK) return error; } else if (S_ISLNK(entry->Mode())) { @@ -528,14 +575,8 @@ struct PackageContentExtractHandler : VersionPolicy::PackageContentHandler { } // write data - status_t error; - const typename VersionPolicy::PackageData& data = attribute->Data(); - if (data.IsEncodedInline()) { - BBufferDataReader dataReader(data.InlineData(), - data.CompressedSize()); - error = _ExtractFileData(&dataReader, data, fd); - } else - error = _ExtractFileData(&fPackageFileReader, data, fd); + status_t error = _ExtractFileData(fPackageFileReader, attribute->Data(), + fd); fs_close_attr(fd); @@ -642,20 +683,20 @@ private: return path; } - status_t _ExtractFileData(BDataReader* dataReader, + status_t _ExtractFileData( + typename VersionPolicy::HeapReaderBase* dataReader, const typename VersionPolicy::PackageData& data, int fd) { // create a PackageDataReader BAbstractBufferedDataReader* reader; - status_t error = typename VersionPolicy::PackageDataReaderFactory( - &fBufferPool) - .CreatePackageDataReader(dataReader, data, reader); + status_t error = VersionPolicy::CreatePackageDataReader(fBufferPool, + dataReader, data, reader); if (error != B_OK) return error; ObjectDeleter readerDeleter(reader); // write the data - off_t bytesRemaining = data.UncompressedSize(); + off_t bytesRemaining = VersionPolicy::PackageDataUncompressedSize(data); off_t offset = 0; while (bytesRemaining > 0) { // read @@ -688,14 +729,14 @@ private: } private: - BBlockBufferPoolNoLock fBufferPool; - BFDDataReader fPackageFileReader; - void* fDataBuffer; - size_t fDataBufferSize; - Entry fRootFilterEntry; - int fBaseDirectory; - const char* fInfoFileName; - bool fErrorOccurred; + BBufferPool* fBufferPool; + typename VersionPolicy::HeapReaderBase* fPackageFileReader; + void* fDataBuffer; + size_t fDataBufferSize; + Entry fRootFilterEntry; + int fBaseDirectory; + const char* fInfoFileName; + bool fErrorOccurred; }; @@ -707,6 +748,12 @@ do_extract(const char* packageFileName, const char* changeToDirectory, { // open package BStandardErrorOutput errorOutput; + BBlockBufferPoolNoLock bufferPool(VersionPolicy::BufferSize(), 2); + if (bufferPool.Init() != B_OK) { + errorOutput.PrintError("Error: Out of memory!\n"); + exit(1); + } + typename VersionPolicy::PackageReader packageReader(&errorOutput); status_t error = packageReader.Init(packageFileName); if (error != B_OK) { @@ -715,8 +762,20 @@ do_extract(const char* packageFileName, const char* changeToDirectory, exit(1); } - PackageContentExtractHandler handler( - packageReader.PackageFileFD()); + typename VersionPolicy::HeapReaderBase* heapReader; + bool mustDeleteHeapReader; + error = VersionPolicy::GetHeapReader(packageReader, heapReader, + mustDeleteHeapReader); + if (error != B_OK) { + fprintf(stderr, "Error: Failed to create heap reader: \"%s\"\n", + strerror(error)); + exit(1); + } + ObjectDeleter heapReaderDeleter( + mustDeleteHeapReader ? heapReader : NULL); + + PackageContentExtractHandler handler(&bufferPool, + heapReader); error = handler.Init(); if (error != B_OK) exit(1); @@ -727,7 +786,7 @@ do_extract(const char* packageFileName, const char* changeToDirectory, for (int i = 0; i < explicitEntryCount; i++) { const char* entryName = explicitEntries[i]; if (entryName[0] == '\0' || entryName[0] == '/') { - fprintf(stderr, "Error: Invalid entry name: \"%s\".", + fprintf(stderr, "Error: Invalid entry name: \"%s\"\n", entryName); exit(1); } diff --git a/src/bin/package/command_list.cpp b/src/bin/package/command_list.cpp index de9220c217..b6f0b4147a 100644 --- a/src/bin/package/command_list.cpp +++ b/src/bin/package/command_list.cpp @@ -29,7 +29,9 @@ using namespace BPackageKit; +using BPackageKit::BHPKG::BErrorOutput; using BPackageKit::BHPKG::BPackageInfoAttributeValue; +using BPackageKit::BHPKG::BStandardErrorOutput; struct VersionPolicyV1 { @@ -39,6 +41,12 @@ struct VersionPolicyV1 { typedef BPackageKit::BHPKG::V1::BPackageEntryAttribute PackageEntryAttribute; typedef BPackageKit::BHPKG::V1::BPackageReader PackageReader; + + static inline uint64 PackageDataSize( + const BPackageKit::BHPKG::V1::BPackageData& data) + { + return data.UncompressedSize(); + } }; struct VersionPolicyV2 { @@ -46,6 +54,12 @@ struct VersionPolicyV2 { typedef BPackageKit::BHPKG::BPackageEntry PackageEntry; typedef BPackageKit::BHPKG::BPackageEntryAttribute PackageEntryAttribute; typedef BPackageKit::BHPKG::BPackageReader PackageReader; + + static inline uint64 PackageDataSize( + const BPackageKit::BHPKG::BPackageData& data) + { + return data.Size(); + } }; @@ -67,7 +81,8 @@ struct PackageContentListHandler : VersionPolicy::PackageContentHandler { // name and size printf("%-*s", indentation < 32 ? 32 - indentation : 0, entry->Name()); - printf(" %8llu", (unsigned long long)entry->Data().UncompressedSize()); + printf(" %8llu", + (unsigned long long)VersionPolicy::PackageDataSize(entry->Data())); // time struct tm* time = localtime(&entry->ModifiedTime().tv_sec); @@ -113,7 +128,8 @@ struct PackageContentListHandler : VersionPolicy::PackageContentHandler { printf("%*s<", indentation, ""); printf("%-*s %8llu", indentation < 31 ? 31 - indentation : 0, attribute->Name(), - (unsigned long long)attribute->Data().UncompressedSize()); + (unsigned long long)VersionPolicy::PackageDataSize( + attribute->Data())); uint32 type = attribute->Type(); if (isprint(type & 0xff) && isprint((type >> 8) & 0xff) @@ -352,7 +368,7 @@ do_list(const char* packageFileName, bool listAttributes, bool filePathsOnly, bool ignoreVersionError) { // open package - BHPKG::BStandardErrorOutput errorOutput; + BStandardErrorOutput errorOutput; typename VersionPolicy::PackageReader packageReader(&errorOutput); status_t error = packageReader.Init(packageFileName); if (error != B_OK) { diff --git a/src/build/libpackage/Jamfile b/src/build/libpackage/Jamfile index e55836571e..0b911b3988 100644 --- a/src/build/libpackage/Jamfile +++ b/src/build/libpackage/Jamfile @@ -28,6 +28,7 @@ HPKG_SOURCES = PoolBuffer.cpp DataOutput.cpp DataReader.cpp + DataWriters.cpp ErrorOutput.cpp FDDataReader.cpp HPKGDefs.cpp @@ -36,6 +37,9 @@ HPKG_SOURCES = PackageDataReader.cpp PackageEntry.cpp PackageEntryAttribute.cpp + PackageFileHeapAccessorBase.cpp + PackageFileHeapReader.cpp + PackageFileHeapWriter.cpp PackageReader.cpp PackageReaderImpl.cpp PackageWriter.cpp diff --git a/src/kits/package/Jamfile b/src/kits/package/Jamfile index ac7bd2c0ab..8ea102622a 100644 --- a/src/kits/package/Jamfile +++ b/src/kits/package/Jamfile @@ -18,6 +18,7 @@ HPKG_SOURCES = BufferPool.cpp DataOutput.cpp DataReader.cpp + DataWriters.cpp ErrorOutput.cpp FDDataReader.cpp HPKGDefs.cpp @@ -26,6 +27,8 @@ HPKG_SOURCES = PackageDataReader.cpp PackageEntry.cpp PackageEntryAttribute.cpp + PackageFileHeapAccessorBase.cpp + PackageFileHeapReader.cpp PackageFileHeapWriter.cpp PackageReader.cpp PackageReaderImpl.cpp diff --git a/src/kits/package/hpkg/DataWriters.cpp b/src/kits/package/hpkg/DataWriters.cpp new file mode 100644 index 0000000000..29318c890d --- /dev/null +++ b/src/kits/package/hpkg/DataWriters.cpp @@ -0,0 +1,125 @@ +/* + * Copyright 2011, Oliver Tappe + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include +#include +#include + +#include + + +namespace BPackageKit { + +namespace BHPKG { + +namespace BPrivate { + + +// #pragma mark - AbstractDataWriter + + +AbstractDataWriter::AbstractDataWriter() + : + fBytesWritten(0) +{ +} + + +AbstractDataWriter::~AbstractDataWriter() +{ +} + + +// #pragma mark - FDDataWriter + + +FDDataWriter::FDDataWriter(int fd, off_t offset, BErrorOutput* errorOutput) + : + fFD(fd), + fOffset(offset), + fErrorOutput(errorOutput) +{ +} + + +status_t +FDDataWriter::WriteDataNoThrow(const void* buffer, size_t size) +{ + ssize_t bytesWritten = pwrite(fFD, buffer, size, fOffset); + if (bytesWritten < 0) { + fErrorOutput->PrintError( + "WriteDataNoThrow(%p, %lu) failed to write data: %s\n", buffer, + size, strerror(errno)); + return errno; + } + if ((size_t)bytesWritten != size) { + fErrorOutput->PrintError( + "WriteDataNoThrow(%p, %lu) failed to write all data\n", buffer, + size); + return B_ERROR; + } + + fOffset += size; + fBytesWritten += size; + return B_OK; +} + + +// #pragma mark - ZlibDataWriter + + +ZlibDataWriter::ZlibDataWriter(AbstractDataWriter* dataWriter) + : + fDataWriter(dataWriter), + fCompressor(this) +{ +} + + +void +ZlibDataWriter::Init() +{ + status_t error = fCompressor.Init(); + if (error != B_OK) + throw status_t(error); +} + + +void +ZlibDataWriter::Finish() +{ + status_t error = fCompressor.Finish(); + if (error != B_OK) + throw status_t(error); +} + + +status_t +ZlibDataWriter::WriteDataNoThrow(const void* buffer, + size_t size) +{ + status_t error = fCompressor.CompressNext(buffer, size); + if (error == B_OK) + fBytesWritten += size; + return error; +} + + +status_t +ZlibDataWriter::WriteData(const void* buffer, size_t size) +{ + return fDataWriter->WriteDataNoThrow(buffer, size); +} + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit diff --git a/src/kits/package/hpkg/PackageContentHandler.cpp b/src/kits/package/hpkg/PackageContentHandler.cpp index 70bfc7e015..584af3cb77 100644 --- a/src/kits/package/hpkg/PackageContentHandler.cpp +++ b/src/kits/package/hpkg/PackageContentHandler.cpp @@ -30,9 +30,6 @@ static const char* kAttributeNames[B_HPKG_ATTRIBUTE_ID_ENUM_COUNT + 1] = { "file:attribute", "file:attribute:type", "data", - "data:compression", - "data:size", - "data:chunk_size", "symlink:path", "package:name", "package:summary", diff --git a/src/kits/package/hpkg/PackageData.cpp b/src/kits/package/hpkg/PackageData.cpp index 31b288f3b0..2e008fca46 100644 --- a/src/kits/package/hpkg/PackageData.cpp +++ b/src/kits/package/hpkg/PackageData.cpp @@ -21,10 +21,7 @@ using namespace BPrivate; BPackageData::BPackageData() : - fCompressedSize(0), - fUncompressedSize(0), - fChunkSize(0), - fCompression(B_HPKG_COMPRESSION_NONE), + fSize(0), fEncodedInline(true) { } @@ -33,7 +30,7 @@ BPackageData::BPackageData() void BPackageData::SetData(uint64 size, uint64 offset) { - fUncompressedSize = fCompressedSize = size; + fSize = size; fOffset = offset; fEncodedInline = false; } @@ -42,7 +39,7 @@ BPackageData::SetData(uint64 size, uint64 offset) void BPackageData::SetData(uint8 size, const void* data) { - fUncompressedSize = fCompressedSize = size; + fSize = size; if (size > 0) memcpy(fInlineData, data, size); fEncodedInline = true; diff --git a/src/kits/package/hpkg/PackageDataReader.cpp b/src/kits/package/hpkg/PackageDataReader.cpp index 6cccfe8ed4..9c808fec44 100644 --- a/src/kits/package/hpkg/PackageDataReader.cpp +++ b/src/kits/package/hpkg/PackageDataReader.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2009-2013, Ingo Weinhold, ingo_weinhold@gmx.de. * Distributed under the terms of the MIT License. */ @@ -12,9 +12,6 @@ #include #include -#include -#include -#include #include #include @@ -27,59 +24,20 @@ namespace BHPKG { using namespace BPrivate; -// minimum/maximum zlib chunk size we consider sane -static const size_t kMinSaneZlibChunkSize = 1024; -static const size_t kMaxSaneZlibChunkSize = 10 * 1024 * 1024; - -// maximum number of entries in the zlib offset table buffer -static const uint32 kMaxZlibOffsetTableBufferSize = 512; - -static const size_t kUncompressedReaderBufferSize - = B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB; +// #pragma mark - PackageDataHeapReader -// #pragma mark - BPackageDataReader - - -class PackageDataReader : public BAbstractBufferedDataReader { +class PackageDataHeapReader : public BAbstractBufferedDataReader { public: - PackageDataReader(BDataReader* dataReader) + PackageDataHeapReader(BAbstractBufferedDataReader* dataReader, + const BPackageData& data) : - fDataReader(dataReader) + fDataReader(dataReader), + fOffset(data.Offset()), + fSize(data.Size()) { } - virtual ~PackageDataReader() - { - } - - virtual status_t Init(const BPackageData& data) = 0; - -protected: - BDataReader* fDataReader; -}; - - -// #pragma mark - UncompressedPackageDataReader - - -class UncompressedPackageDataReader : public PackageDataReader { -public: - UncompressedPackageDataReader(BDataReader* dataReader, - BBufferPool* bufferPool) - : - PackageDataReader(dataReader), - fBufferPool(bufferPool) - { - } - - status_t Init(const BPackageData& data) - { - fOffset = data.Offset(); - fSize = data.UncompressedSize(); - return B_OK; - } - virtual status_t ReadData(off_t offset, void* buffer, size_t size) { if (size == 0) @@ -106,332 +64,55 @@ public: if ((uint64)offset > fSize || size > fSize - offset) return B_BAD_VALUE; - // get a temporary buffer - PoolBuffer* buffer = fBufferPool->GetBuffer( - kUncompressedReaderBufferSize); - if (buffer == NULL) - return B_NO_MEMORY; - PoolBufferPutter bufferPutter(fBufferPool, &buffer); - - while (size > 0) { - // read into the buffer - size_t toRead = std::min(size, buffer->Size()); - status_t error = fDataReader->ReadData(fOffset + offset, - buffer->Buffer(), toRead); - if (error != B_OK) - return error; - - // write to the output - error = output->WriteData(buffer->Buffer(), toRead); - if (error != B_OK) - return error; - - offset += toRead; - size -= toRead; - } - - return B_OK; + return fDataReader->ReadDataToOutput(fOffset + offset, size, output); } private: - BBufferPool* fBufferPool; - uint64 fOffset; - uint64 fSize; + BAbstractBufferedDataReader* fDataReader; + uint64 fOffset; + uint64 fSize; }; -// #pragma mark - ZlibPackageDataReader +// #pragma mark - PackageDataHeapReader -class ZlibPackageDataReader : public PackageDataReader { +class PackageDataInlineReader : public BBufferDataReader { public: - ZlibPackageDataReader(BDataReader* dataReader, BBufferPool* bufferPool) + PackageDataInlineReader(const BPackageData& data) : - PackageDataReader(dataReader), - fBufferPool(bufferPool), - fUncompressBuffer(NULL), - fOffsetTable(NULL) + BBufferDataReader(fData.InlineData(), data.Size()), + fData(data) { } - ~ZlibPackageDataReader() - { - delete[] fOffsetTable; - - fBufferPool->PutBuffer(&fUncompressBuffer); - } - - status_t Init(const BPackageData& data) - { - fOffset = data.Offset(); - fCompressedSize = data.CompressedSize(); - fUncompressedSize = data.UncompressedSize(); - fChunkSize = data.ChunkSize(); - - // validate chunk size - if (fChunkSize == 0) - fChunkSize = B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB; - if (fChunkSize < kMinSaneZlibChunkSize - || fChunkSize > kMaxSaneZlibChunkSize) { - return B_BAD_DATA; - } - - fChunkCount = (fUncompressedSize + (fChunkSize - 1)) / fChunkSize; - fOffsetTableSize = (fChunkCount - 1) * sizeof(uint64); - if (fOffsetTableSize >= fCompressedSize) - return B_BAD_DATA; - - // allocate a buffer for the offset table - if (fChunkCount > 1) { - fOffsetTableBufferEntryCount = std::min(fChunkCount - 1, - (uint64)kMaxZlibOffsetTableBufferSize); - fOffsetTable = new(std::nothrow) uint64[ - fOffsetTableBufferEntryCount]; - if (fOffsetTable == NULL) - return B_NO_MEMORY; - - fOffsetTableIndex = -1; - // mark the table content invalid - } else - fChunkSize = fUncompressedSize; - - // mark uncompressed content invalid - fUncompressedChunk = -1; - - return B_OK; - } - - virtual status_t ReadDataToOutput(off_t offset, size_t size, - BDataOutput* output) - { - // check offset and size - if (size == 0) - return B_OK; - - if (offset < 0) - return B_BAD_VALUE; - - if ((uint64)offset > fUncompressedSize - || size > fUncompressedSize - offset) { - return B_BAD_VALUE; - } - - // get our uncompressed chunk buffer back, if possible - bool newBuffer; - if (fBufferPool->GetBuffer(fChunkSize, &fUncompressBuffer, &newBuffer) - == NULL) { - return B_NO_MEMORY; - } - PoolBufferPutter uncompressBufferPutter(fBufferPool, - &fUncompressBuffer); - - if (newBuffer) - fUncompressedChunk = -1; - - // uncompress - int64 chunkIndex = offset / fChunkSize; - off_t chunkOffset = chunkIndex * fChunkSize; - size_t inChunkOffset = offset - chunkOffset; - - while (size > 0) { - // read and uncompress the chunk - status_t error = _ReadChunk(chunkIndex); - if (error != B_OK) - return error; - - // write data to output - size_t toCopy = std::min(size, (size_t)fChunkSize - inChunkOffset); - error = output->WriteData( - (uint8*)fUncompressBuffer->Buffer() + inChunkOffset, toCopy); - if (error != B_OK) - return error; - - size -= toCopy; - - chunkIndex++; - chunkOffset += fChunkSize; - inChunkOffset = 0; - } - - return B_OK; - } - private: - status_t _ReadChunk(int64 chunkIndex) - { - if (chunkIndex == fUncompressedChunk) - return B_OK; - - // get the chunk offset and size - uint64 offset; - uint32 compressedSize; - status_t error = _GetCompressedChunkOffsetAndSize(chunkIndex, offset, - compressedSize); - if (error != B_OK) - return error; - - uint32 uncompressedSize = (uint64)chunkIndex + 1 < fChunkCount - ? fChunkSize : fUncompressedSize - chunkIndex * fChunkSize; - - // read the chunk - if (compressedSize == uncompressedSize) { - // the chunk is not compressed -- read it directly into the - // uncompressed buffer - error = fDataReader->ReadData(offset, fUncompressBuffer->Buffer(), - compressedSize); - } else { - // read to a read buffer and uncompress - PoolBuffer* readBuffer = fBufferPool->GetBuffer(fChunkSize); - if (readBuffer == NULL) - return B_NO_MEMORY; - PoolBufferPutter readBufferPutter(fBufferPool, readBuffer); - - error = fDataReader->ReadData(offset, readBuffer->Buffer(), - compressedSize); - if (error != B_OK) - return error; - - size_t actuallyUncompressedSize; - error = ZlibDecompressor::DecompressSingleBuffer( - readBuffer->Buffer(), compressedSize, - fUncompressBuffer->Buffer(), uncompressedSize, - actuallyUncompressedSize); - if (error == B_OK && actuallyUncompressedSize != uncompressedSize) - error = B_BAD_DATA; - } - - if (error != B_OK) { - // error reading/decompressing data -- mark the cached data invalid - fUncompressedChunk = -1; - return error; - } - - fUncompressedChunk = chunkIndex; - return B_OK; - } - - status_t _GetCompressedChunkOffsetAndSize(int64 chunkIndex, uint64& _offset, - uint32& _size) - { - // get the offset - uint64 offset; - if (chunkIndex == 0) { - // first chunk is at 0 - offset = 0; - } else { - status_t error = _GetCompressedChunkRelativeOffset(chunkIndex, - offset); - if (error != B_OK) - return error; - } - - // get the end offset - uint64 endOffset; - if ((uint64)chunkIndex + 1 == fChunkCount) { - // last chunk end with the end of the data - endOffset = fCompressedSize - fOffsetTableSize; - } else { - status_t error = _GetCompressedChunkRelativeOffset(chunkIndex + 1, - endOffset); - if (error != B_OK) - return error; - } - - // sanity check - if (endOffset < offset) - return B_BAD_DATA; - - _offset = fOffset + fOffsetTableSize + offset; - _size = endOffset - offset; - return B_OK; - } - - status_t _GetCompressedChunkRelativeOffset(int64 chunkIndex, - uint64& _offset) - { - if (fOffsetTableIndex < 0 || fOffsetTableIndex > chunkIndex - || fOffsetTableIndex + fOffsetTableBufferEntryCount <= chunkIndex) { - // read the table at the given index, or, if we can, the whole table - int64 readAtIndex = fChunkCount - 1 > fOffsetTableBufferEntryCount - ? chunkIndex : 1; - uint32 entriesToRead = std::min( - (uint64)fOffsetTableBufferEntryCount, - fChunkCount - readAtIndex); - - status_t error = fDataReader->ReadData( - fOffset + (readAtIndex - 1) * sizeof(uint64), - fOffsetTable, entriesToRead * sizeof(uint64)); - if (error != B_OK) { - fOffsetTableIndex = -1; - return error; - } - - fOffsetTableIndex = readAtIndex; - } - - // get and check the offset - _offset = fOffsetTable[chunkIndex - fOffsetTableIndex]; - if (_offset > fCompressedSize - fOffsetTableSize) - return B_BAD_DATA; - - return B_OK; - } - -private: - BBufferPool* fBufferPool; - PoolBuffer* fUncompressBuffer; - int64 fUncompressedChunk; - - uint64 fOffset; - uint64 fUncompressedSize; - uint64 fCompressedSize; - uint64 fOffsetTableSize; - uint64 fChunkCount; - uint32 fChunkSize; - uint32 fOffsetTableBufferEntryCount; - uint64* fOffsetTable; - int32 fOffsetTableIndex; + BPackageData fData; }; // #pragma mark - BPackageDataReaderFactory -BPackageDataReaderFactory::BPackageDataReaderFactory(BBufferPool* bufferPool) - : - fBufferPool(bufferPool) +BPackageDataReaderFactory::BPackageDataReaderFactory() { } status_t -BPackageDataReaderFactory::CreatePackageDataReader(BDataReader* dataReader, - const BPackageData& data, BAbstractBufferedDataReader*& _reader) +BPackageDataReaderFactory::CreatePackageDataReader( + BAbstractBufferedDataReader* dataReader, const BPackageData& data, + BAbstractBufferedDataReader*& _reader) { - PackageDataReader* reader; - - switch (data.Compression()) { - case B_HPKG_COMPRESSION_NONE: - reader = new(std::nothrow) UncompressedPackageDataReader( - dataReader, fBufferPool); - break; - case B_HPKG_COMPRESSION_ZLIB: - reader = new(std::nothrow) ZlibPackageDataReader(dataReader, - fBufferPool); - break; - default: - return B_BAD_VALUE; - } + BAbstractBufferedDataReader* reader; + if (data.IsEncodedInline()) + reader = new(std::nothrow) PackageDataInlineReader(data); + else + reader = new(std::nothrow) PackageDataHeapReader(dataReader, data); if (reader == NULL) return B_NO_MEMORY; - status_t error = reader->Init(data); - if (error != B_OK) { - delete reader; - return error; - } - _reader = reader; return B_OK; } diff --git a/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp b/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp new file mode 100644 index 0000000000..be7d453ea5 --- /dev/null +++ b/src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp @@ -0,0 +1,247 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include + + +namespace BPackageKit { + +namespace BHPKG { + +namespace BPrivate { + + +// #pragma mark - OffsetArray + + +PackageFileHeapAccessorBase::OffsetArray::OffsetArray() + : + fOffsets(NULL) +{ +} + + +PackageFileHeapAccessorBase::OffsetArray::~OffsetArray() +{ + delete[] fOffsets; +} + + +bool +PackageFileHeapAccessorBase::OffsetArray::InitChunksOffsets( + size_t totalChunkCount, size_t baseIndex, const uint16* chunkSizes, + size_t chunkCount) +{ + if (totalChunkCount <= 1) + return true; + + if (fOffsets == NULL) { + fOffsets = new(std::nothrow) uint32[totalChunkCount]; + if (fOffsets == NULL) + return false; + fOffsets[0] = 0; + // Value that serves as a marker that all offsets are 32 bit. We'll + // replace it, when necessary. + } + + uint64 offset = (*this)[baseIndex]; + for (size_t i = 0; i < chunkCount; i++) { + offset += (uint64)chunkSizes[i] + 1; + // the stored value is chunkSize - 1 + size_t index = baseIndex + i + 1; + // (baseIndex + i) is the index of the chunk whose size is stored in + // chunkSizes[i]. We compute the offset of the following element + // which we store at index (baseIndex + i + 1). + + if (offset <= ~(uint32)0) { + fOffsets[index] = (uint32)offset; + } else { + if (fOffsets[0] == 0) { + // Not scaled to allow for 64 bit offsets yet. Do that. + fOffsets[0] = index; + uint32* newOffsets = new(std::nothrow) uint32[ + 2 * totalChunkCount - fOffsets[0]]; + if (newOffsets == NULL) + return false; + + memcpy(newOffsets, fOffsets, + sizeof(newOffsets[0]) * fOffsets[0]); + + delete[] fOffsets; + fOffsets = newOffsets; + } + + index += index - fOffsets[0]; + fOffsets[index] = (uint32)offset; + fOffsets[index + 1] = uint32(offset >> 32); + } + } + + return true; +} + + +bool +PackageFileHeapAccessorBase::OffsetArray::Init(size_t totalChunkCount, + const OffsetArray& other) +{ + if (other.fOffsets == NULL) + return true; + + size_t elementCount = other.fOffsets[0] == 0 + ? totalChunkCount + : 2 * totalChunkCount - other.fOffsets[0]; + + fOffsets = new(std::nothrow) uint32[elementCount]; + if (fOffsets == NULL) + return false; + + memcpy(fOffsets, other.fOffsets, elementCount * sizeof(fOffsets[0])); + return true; +} + + +// #pragma mark - PackageFileHeapAccessorBase + + +PackageFileHeapAccessorBase::PackageFileHeapAccessorBase( + BErrorOutput* errorOutput, int fd, off_t heapOffset) + : + fErrorOutput(errorOutput), + fFD(fd), + fHeapOffset(heapOffset), + fCompressedHeapSize(0), + fUncompressedHeapSize(0) +{ +} + + +PackageFileHeapAccessorBase::~PackageFileHeapAccessorBase() +{ +} + + +status_t +PackageFileHeapAccessorBase::ReadDataToOutput(off_t offset, size_t size, + BDataOutput* output) +{ + if (size == 0) + return B_OK; + + if (offset < 0 || (uint64)offset > fUncompressedHeapSize + || size > fUncompressedHeapSize - offset) { + return B_BAD_VALUE; + } + + // allocate buffers for compressed and uncompressed data + uint16* compressedDataBuffer = (uint16*)malloc(kChunkSize); + uint16* uncompressedDataBuffer = (uint16*)malloc(kChunkSize); + MemoryDeleter compressedDataBufferDeleter(compressedDataBuffer); + MemoryDeleter uncompressedDataBufferDeleter(uncompressedDataBuffer); + if (compressedDataBuffer == NULL || uncompressedDataBuffer == NULL) + return B_NO_MEMORY; + + // read the data + size_t chunkIndex = size_t(offset / kChunkSize); + size_t inChunkOffset = (uint64)offset - (uint64)chunkIndex * kChunkSize; + size_t remainingBytes = size; + + while (remainingBytes > 0) { + status_t error = ReadAndDecompressChunk(chunkIndex, + compressedDataBuffer, uncompressedDataBuffer); + if (error != B_OK) + return error; + + size_t toWrite = std::min((size_t)kChunkSize - inChunkOffset, + remainingBytes); + // The last chunk may be shorter than kChunkSize, but since + // size (and thus remainingSize) had been clamped, that doesn't + // harm. + error = output->WriteData( + (char*)uncompressedDataBuffer + inChunkOffset, toWrite); + if (error != B_OK) + return error; + + remainingBytes -= toWrite; + chunkIndex++; + inChunkOffset = 0; + } + + return B_OK; +} + + +status_t +PackageFileHeapAccessorBase::ReadAndDecompressChunkData(uint64 offset, + size_t compressedSize, size_t uncompressedSize, void* compressedDataBuffer, + void* uncompressedDataBuffer) +{ + // if uncompressed, read directly into the uncompressed data buffer + if (compressedSize == uncompressedSize) + return ReadFileData(offset, uncompressedDataBuffer, compressedSize); + + // otherwise read into the other buffer and decompress + status_t error = ReadFileData(offset, compressedDataBuffer, compressedSize); + if (error != B_OK) + return error; + + size_t actualSize; + error = ZlibDecompressor::DecompressSingleBuffer(compressedDataBuffer, + compressedSize, uncompressedDataBuffer, uncompressedSize, actualSize); + if (error != B_OK) { + fErrorOutput->PrintError("Failed to decompress data chunk: %s\n", + strerror(error)); + return error; + } + + if (actualSize != uncompressedSize) { + fErrorOutput->PrintError("Failed to decompress data chunk: chunk " + "size mismatch\n"); + return B_ERROR; + } + + return B_OK; +} + + +status_t +PackageFileHeapAccessorBase::ReadFileData(uint64 offset, void* buffer, + size_t size) +{ + ssize_t bytesRead = pread(fFD, buffer, size, fHeapOffset + (off_t)offset); + if (bytesRead < 0) { + fErrorOutput->PrintError("_ReadData(%" B_PRIu64 "%p, %zu) failed to " + "read data: %s\n", offset, buffer, size, strerror(errno)); + return errno; + } + if ((size_t)bytesRead != size) { + fErrorOutput->PrintError("_ReadData(%" B_PRIu64 "%p, %zu) failed to " + "read all data\n", offset, buffer, size); + return B_ERROR; + } + + return B_OK; +} + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit diff --git a/src/kits/package/hpkg/PackageFileHeapReader.cpp b/src/kits/package/hpkg/PackageFileHeapReader.cpp new file mode 100644 index 0000000000..e02313ba87 --- /dev/null +++ b/src/kits/package/hpkg/PackageFileHeapReader.cpp @@ -0,0 +1,131 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include +#include + +#include +#include + +#include +#include + + +// minimum length of data we require before trying to zlib compress them +static const size_t kZlibCompressionSizeThreshold = 64; + + +namespace BPackageKit { + +namespace BHPKG { + +namespace BPrivate { + + +PackageFileHeapReader::PackageFileHeapReader(BErrorOutput* errorOutput, int fd, + off_t heapOffset, off_t compressedHeapSize, uint64 uncompressedHeapSize) + : + PackageFileHeapAccessorBase(errorOutput, fd, heapOffset), + fOffsets() +{ + fCompressedHeapSize = compressedHeapSize; + fUncompressedHeapSize = uncompressedHeapSize; +} + + +PackageFileHeapReader::~PackageFileHeapReader() +{ +} + + +status_t +PackageFileHeapReader::Init() +{ + if (fUncompressedHeapSize == 0) + return B_OK; + + // Determine number of chunks and adjust the compressed heap size (subtract + // the size of the chunk size array at the end). Note that the size of the + // last chunk has not been saved, since it size is implied. + ssize_t chunkCount = (fUncompressedHeapSize + kChunkSize - 1) / kChunkSize; + if (chunkCount <= 0) + return B_OK; + + fCompressedHeapSize -= (chunkCount - 1) * 2; + + // allocate a buffer + uint16* buffer = (uint16*)malloc(kChunkSize); + if (buffer == NULL) + return B_NO_MEMORY; + MemoryDeleter bufferDeleter(buffer); + + // read the chunk size array + size_t remainingChunks = chunkCount - 1; + size_t index = 0; + uint64 offset = fCompressedHeapSize; + while (remainingChunks > 0) { + size_t toRead = std::min(remainingChunks, kChunkSize / 2); + status_t error = ReadFileData(offset, buffer, toRead * 2); + if (error != B_OK) + return error; + + if (!fOffsets.InitChunksOffsets(chunkCount, index, buffer, toRead)) + return B_NO_MEMORY; + + remainingChunks -= toRead; + index += toRead; + offset += toRead * 2; + } + + return B_OK; +} + + +PackageFileHeapReader* +PackageFileHeapReader::Clone() const +{ + PackageFileHeapReader* clone = new(std::nothrow) PackageFileHeapReader( + fErrorOutput, fFD, fHeapOffset, fCompressedHeapSize, + fUncompressedHeapSize); + if (clone == NULL) + return NULL; + + ssize_t chunkCount = (fUncompressedHeapSize + kChunkSize - 1) / kChunkSize; + if (!clone->fOffsets.Init(chunkCount, fOffsets)) { + delete clone; + return NULL; + } + + return clone; +} + + +status_t +PackageFileHeapReader::ReadAndDecompressChunk(size_t chunkIndex, + void* compressedDataBuffer, void* uncompressedDataBuffer) +{ + uint64 offset = fOffsets[chunkIndex]; + bool isLastChunk + = uint64(chunkIndex + 1) * kChunkSize >= fUncompressedHeapSize; + size_t compressedSize = isLastChunk + ? fCompressedHeapSize - offset + : fOffsets[chunkIndex + 1] - offset; + size_t uncompressedSize = isLastChunk + ? fUncompressedHeapSize - (uint64)chunkIndex * kChunkSize + : kChunkSize; + + return ReadAndDecompressChunkData(offset, compressedSize, uncompressedSize, + compressedDataBuffer, uncompressedDataBuffer); +} + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit diff --git a/src/kits/package/hpkg/PackageFileHeapWriter.cpp b/src/kits/package/hpkg/PackageFileHeapWriter.cpp new file mode 100644 index 0000000000..d7fc523c97 --- /dev/null +++ b/src/kits/package/hpkg/PackageFileHeapWriter.cpp @@ -0,0 +1,645 @@ +/* + * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + + +// minimum length of data we require before trying to zlib compress them +static const size_t kZlibCompressionSizeThreshold = 64; + + +namespace BPackageKit { + +namespace BHPKG { + +namespace BPrivate { + + +struct PackageFileHeapWriter::Chunk { + uint64 offset; + uint32 compressedSize; + uint32 uncompressedSize; + void* buffer; +}; + + +struct PackageFileHeapWriter::ChunkSegment { + ssize_t chunkIndex; + uint32 toKeepOffset; + uint32 toKeepSize; +}; + + +struct PackageFileHeapWriter::ChunkBuffer { + ChunkBuffer(BErrorOutput* errorOutput, int fd, off_t heapOffset, + size_t bufferSize) + : + fErrorOutput(errorOutput), + fFD(fd), + fHeapOffset(heapOffset), + fChunks(), + fCurrentChunkIndex(0), + fNextReadIndex(0), + fSegments(), + fCurrentSegmentIndex(0), + fBuffers(), + fUnusedBuffers(), + fBufferSize(bufferSize) + { + } + + ~ChunkBuffer() + { + for (int32 i = 0; void* buffer = fBuffers.ItemAt(i); i++) + free(buffer); + } + + bool PushChunkSegment(uint64 chunkOffset, uint32 compressedSize, + uint32 uncompressedSize, uint32 toKeepOffset, uint32 toKeepSize) + { + ChunkSegment segment; + segment.toKeepOffset = toKeepOffset; + segment.toKeepSize = toKeepSize; + + // might refer to the last chunk + segment.chunkIndex = fChunks.Count() - 1; + + if (segment.chunkIndex < 0 + || fChunks.ElementAt(segment.chunkIndex).offset != chunkOffset) { + // no, need to push a new chunk + segment.chunkIndex++; + + Chunk chunk; + chunk.offset = chunkOffset; + chunk.compressedSize = compressedSize; + chunk.uncompressedSize = uncompressedSize; + chunk.buffer = NULL; + if (!fChunks.Add(chunk)) + return false; + } + + return fSegments.Add(segment); + } + + bool HasMoreSegments() const + { + return fCurrentSegmentIndex < fSegments.Count(); + } + + const ChunkSegment& CurrentSegment() const + { + return fSegments[fCurrentSegmentIndex]; + } + + const Chunk& ChunkAt(ssize_t index) const + { + return fChunks[index]; + } + + bool HasMoreChunksToRead() const + { + return fNextReadIndex < fChunks.Count(); + } + + bool HasBufferedChunk() const + { + return fCurrentChunkIndex < fNextReadIndex; + } + + uint64 NextReadOffset() const + { + return fChunks[fNextReadIndex].offset; + } + + void ReadNextChunk() + { + if (!HasMoreChunksToRead()) + throw status_t(B_BAD_VALUE); + + Chunk& chunk = fChunks[fNextReadIndex++]; + chunk.buffer = _GetBuffer(); + + ssize_t bytesRead = pread(fFD, chunk.buffer, chunk.compressedSize, + fHeapOffset + chunk.offset); + if (bytesRead < 0) { + fErrorOutput->PrintError("PackageFileHeapWriter::ChunkBuffer::" + "ReadNextChunk(offset: %" B_PRIu64 ", size: %" B_PRIu32 + ") failed to read data: %s\n", chunk.offset, + chunk.compressedSize, strerror(errno)); + throw status_t(errno); + } + + if ((size_t)bytesRead != chunk.compressedSize) { + fErrorOutput->PrintError("PackageFileHeapWriter::ChunkBuffer::" + "ReadNextChunk(offset: %" B_PRIu64 ", size: %" B_PRIu32 + ") failed to read all data\n", chunk.offset, + chunk.compressedSize); + throw status_t(B_ERROR); + } + } + + void CurrentSegmentDone() + { + // Unless the next segment refers to the same chunk, advance to the next + // chunk. + const ChunkSegment& segment = fSegments[fCurrentSegmentIndex++]; + if (!HasMoreSegments() + || segment.chunkIndex != CurrentSegment().chunkIndex) { + _PutBuffer(fChunks[fCurrentChunkIndex++].buffer); + } + } + +private: + void* _GetBuffer() + { + if (!fUnusedBuffers.IsEmpty()) + return fUnusedBuffers.RemoveItem(fUnusedBuffers.CountItems() - 1); + + void* buffer = malloc(fBufferSize); + if (buffer == NULL && !fBuffers.AddItem(buffer)) { + free(buffer); + throw std::bad_alloc(); + } + + return buffer; + } + + void _PutBuffer(void* buffer) + { + if (buffer != NULL && !fUnusedBuffers.AddItem(buffer)) { + fBuffers.RemoveItem(buffer); + free(buffer); + } + } + +private: + BErrorOutput* fErrorOutput; + int fFD; + off_t fHeapOffset; + + Array fChunks; + ssize_t fCurrentChunkIndex; + ssize_t fNextReadIndex; + + Array fSegments; + ssize_t fCurrentSegmentIndex; + + BList fBuffers; + BList fUnusedBuffers; + size_t fBufferSize; +}; + + +PackageFileHeapWriter::PackageFileHeapWriter(BErrorOutput* errorOutput, int fd, + off_t heapOffset) + : + PackageFileHeapAccessorBase(errorOutput, fd, heapOffset), + fPendingDataBuffer(NULL), + fCompressedDataBuffer(NULL), + fPendingDataSize(0), + fOffsets() +{ +} + + +PackageFileHeapWriter::~PackageFileHeapWriter() +{ + _Uninit(); +} + + +void +PackageFileHeapWriter::Init() +{ + // allocate data buffers + fPendingDataBuffer = malloc(kChunkSize); + fCompressedDataBuffer = malloc(kChunkSize); + if (fPendingDataBuffer == NULL || fCompressedDataBuffer == NULL) + throw std::bad_alloc(); +} + + +void +PackageFileHeapWriter::Reinit(PackageFileHeapReader* heapReader) +{ + fHeapOffset = heapReader->HeapOffset(); + fCompressedHeapSize = heapReader->CompressedHeapSize(); + fUncompressedHeapSize = heapReader->UncompressedHeapSize(); + fPendingDataSize = 0; + + // copy the offsets array + size_t chunkCount = (fUncompressedHeapSize + kChunkSize - 1) / kChunkSize; + if (chunkCount > 0) { + if (!fOffsets.AddUninitialized(chunkCount)) + throw std::bad_alloc(); + + for (size_t i = 0; i < chunkCount; i++) + fOffsets[i] = heapReader->Offsets()[i]; + } + + // If the last chunk is partial, read it in and remove it from the offsets. + size_t lastChunkSize = fUncompressedHeapSize % kChunkSize; + if (lastChunkSize != 0) { + status_t error = heapReader->ReadData( + fUncompressedHeapSize - lastChunkSize, fPendingDataBuffer, + lastChunkSize); + if (error != B_OK) + throw error; + + fCompressedHeapSize = fOffsets[fOffsets.Count() - 1]; + fOffsets.Remove(fOffsets.Count() - 1); + } +} + + +status_t +PackageFileHeapWriter::AddData(BDataReader& dataReader, off_t size, + uint64& _offset) +{ + _offset = fUncompressedHeapSize; + + // copy the data to the heap + off_t readOffset = 0; + off_t remainingSize = size; + while (remainingSize > 0) { + // read data into pending data buffer + size_t toCopy = std::min(remainingSize, + off_t(kChunkSize - fPendingDataSize)); + status_t error = dataReader.ReadData(readOffset, + (uint8*)fPendingDataBuffer + fPendingDataSize, toCopy); + if (error != B_OK) { + fErrorOutput->PrintError("Failed to read data: %s\n", + strerror(error)); + return error; + } + + fPendingDataSize += toCopy; + fUncompressedHeapSize += toCopy; + remainingSize -= toCopy; + readOffset += toCopy; + + if (fPendingDataSize == kChunkSize) { + error = _FlushPendingData(); + if (error != B_OK) + return error; + } + } + + return B_OK; +} + + +void +PackageFileHeapWriter::RemoveDataRanges( + const ::BPrivate::RangeArray& ranges) +{ + ssize_t rangeCount = ranges.CountRanges(); + if (rangeCount == 0) + return; + + if (fOffsets.IsEmpty()) { + fErrorOutput->PrintError("Can't remove ranges from empty heap\n"); + throw status_t(B_BAD_VALUE); + } + + // We potentially have to recompress all data from the first affected chunk + // to the end (minus the removed ranges, of course). As a basic algorithm we + // can use our usual data writing strategy, i.e. read a chunk, decompress it + // to a temporary buffer, and write the data to keep via AddData(). There + // are a few complications/optimizations, though: + // * As data moves to other chunks, it may actually compress worse than + // before. While unlikely, we still have to take care of this case by + // making sure our reading end is at least a complete uncompressed chunk + // ahead of the writing end. + // * When we run into the situation that we have to move complete aligned + // chunks, we want to avoid uncompressing and recompressing them + // needlessly. + + // Build a list of (possibly partial) chunks we want to keep. + + // the first partial chunk (if any) and all chunks between ranges + ChunkBuffer chunkBuffer(fErrorOutput, fFD, fHeapOffset, kChunkSize); + uint64 writeOffset = ranges[0].offset - ranges[0].offset % kChunkSize; + uint64 readOffset = writeOffset; + for (ssize_t i = 0; i < rangeCount; i++) { + const Range& range = ranges[i]; + if (range.size > 0) { + _PushChunks(chunkBuffer, readOffset, range.offset); + readOffset = range.offset + range.size; + } + } + + if (readOffset == writeOffset) { + fErrorOutput->PrintError("Only empty ranges to remove from heap\n"); + throw status_t(B_BAD_VALUE); + } + + // all chunks after the last range + _PushChunks(chunkBuffer, readOffset, fUncompressedHeapSize); + + // Reset our state to look like all chunks from the first affected one have + // been removed and re-add all data we want to keep. + + // truncate the offsets array and reset the heap end offset + ssize_t firstChunkIndex = ssize_t(writeOffset / kChunkSize); + fCompressedHeapSize = fOffsets[firstChunkIndex]; + fOffsets.Remove(firstChunkIndex, fOffsets.Count() - firstChunkIndex); + + // we need a decompression buffer + void* decompressionBuffer = malloc(kChunkSize); + if (decompressionBuffer == NULL) + throw std::bad_alloc(); + MemoryDeleter decompressionBufferDeleter(decompressionBuffer); + + const Chunk* decompressedChunk = NULL; + + while (chunkBuffer.HasMoreSegments()) { + const ChunkSegment& segment = chunkBuffer.CurrentSegment(); + + // If we have an aligned, complete chunk, copy its compressed data. + bool copyCompressed = fPendingDataSize == 0 && segment.toKeepOffset == 0 + && segment.toKeepSize == kChunkSize; + + // Read more chunks. We need at least one buffered one to do anything + // and we want to buffer as many as necessary to ensure we don't + // overwrite one we haven't buffered yet. + while (chunkBuffer.HasMoreChunksToRead() + && (!chunkBuffer.HasBufferedChunk() + || (!copyCompressed + && chunkBuffer.NextReadOffset() + < fCompressedHeapSize + kChunkSize))) { + // read chunk + chunkBuffer.ReadNextChunk(); + } + + // copy compressed chunk data, if possible + const Chunk& chunk = chunkBuffer.ChunkAt(segment.chunkIndex); + if (copyCompressed) { + status_t error = _WriteChunk(chunk.buffer, chunk.compressedSize, + false); + if (error != B_OK) + throw error; + continue; + } + + // decompress chunk, if compressed + void* uncompressedData; + if (chunk.uncompressedSize == chunk.compressedSize) { + uncompressedData = chunk.buffer; + } else if (decompressedChunk == &chunk) { + uncompressedData = decompressionBuffer; + } else { + size_t uncompressedSize; + status_t error = ZlibDecompressor::DecompressSingleBuffer( + chunk.buffer, chunk.compressedSize, + decompressionBuffer, chunk.uncompressedSize, uncompressedSize); + if (error != B_OK) { + fErrorOutput->PrintError("Failed to decompress data chunk: %s\n", + strerror(error)); + throw error; + } + + if (uncompressedSize != chunk.uncompressedSize) { + fErrorOutput->PrintError("Failed to decompress data chunk: chunk " + "size mismatch\n"); + throw status_t(B_ERROR); + } + + decompressedChunk = &chunk; + uncompressedData = decompressionBuffer; + } + + // add chunk data + WriteDataThrows((uint8*)uncompressedData + segment.toKeepOffset, + segment.toKeepSize); + + chunkBuffer.CurrentSegmentDone(); + } +} + + +status_t +PackageFileHeapWriter::Finish() +{ + // flush pending data, if any + status_t error = _FlushPendingData(); + if (error != B_OK) + return error; + + // write chunk sizes table + + // We don't need to write the last chunk size, since it is implied by the + // total size minus the sum of all other chunk sizes. + ssize_t offsetCount = fOffsets.Count(); + if (offsetCount < 2) + return B_OK; + + // Convert the offsets to 16 bit sizes and write them. We use the (no longer + // used) pending data buffer for the conversion. + uint16* buffer = (uint16*)fPendingDataBuffer; + for (ssize_t offsetIndex = 1; offsetIndex < offsetCount;) { + ssize_t toWrite = std::min(offsetCount - offsetIndex, + ssize_t(kChunkSize / 2)); + + for (ssize_t i = 0; i < toWrite; i++, offsetIndex++) { + // store chunkSize - 1, so it fits 16 bit (chunks cannot be empty) + buffer[i] = uint16(fOffsets[offsetIndex] - fOffsets[offsetIndex - 1] + - 1); + } + + error = _WriteDataUncompressed(buffer, toWrite * 2); + if (error != B_OK) + return error; + } + + return B_OK; +} + + +status_t +PackageFileHeapWriter::ReadAndDecompressChunk(size_t chunkIndex, + void* compressedDataBuffer, void* uncompressedDataBuffer) +{ + if (uint64(chunkIndex + 1) * kChunkSize > fUncompressedHeapSize) { + // The chunk has not been written to disk yet. Its data are still in the + // pending data buffer. + memcpy(uncompressedDataBuffer, fPendingDataBuffer, fPendingDataSize); + // TODO: This can be optimized. Since we write to a BDataOutput anyway, + // there's no need to copy the data. + return B_OK; + } + + uint64 offset = fOffsets[chunkIndex]; + size_t compressedSize = chunkIndex + 1 == (size_t)fOffsets.Count() + ? fCompressedHeapSize - offset + : fOffsets[chunkIndex + 1] - offset; + + return ReadAndDecompressChunkData(offset, compressedSize, kChunkSize, + compressedDataBuffer, uncompressedDataBuffer); +} + + +status_t +PackageFileHeapWriter::WriteDataNoThrow(const void* buffer, size_t size) +{ + BBufferDataReader reader(buffer, size); + uint64 dummyOffset; + return AddData(reader, size, dummyOffset); +} + + +void +PackageFileHeapWriter::_Uninit() +{ + free(fPendingDataBuffer); + free(fCompressedDataBuffer); + fPendingDataBuffer = NULL; + fCompressedDataBuffer = NULL; +} + + +status_t +PackageFileHeapWriter::_FlushPendingData() +{ + if (fPendingDataSize == 0) + return B_OK; + + status_t error = _WriteChunk(fPendingDataBuffer, fPendingDataSize, true); + if (error == B_OK) + fPendingDataSize = 0; + + return error; +} + + +status_t +PackageFileHeapWriter::_WriteChunk(const void* data, size_t size, + bool mayCompress) +{ + // add offset + if (!fOffsets.Add(fCompressedHeapSize)) { + fErrorOutput->PrintError("Out of memory!\n"); + return B_NO_MEMORY; + } + + // Try to use zlib compression only for data large enough. + bool compress = mayCompress && size >= (off_t)kZlibCompressionSizeThreshold; + if (compress) { + status_t error = _WriteDataCompressed(data, size); + if (error != B_OK) { + if (error != B_BUFFER_OVERFLOW) + return error; + compress = false; + } + } + + // Write uncompressed, if necessary. + if (!compress) { + status_t error = _WriteDataUncompressed(data, size); + if (error != B_OK) + return error; + } + + return B_OK; +} + + +status_t +PackageFileHeapWriter::_WriteDataCompressed(const void* data, size_t size) +{ + size_t compressedSize; + status_t error = ZlibCompressor::CompressSingleBuffer(data, size, + fCompressedDataBuffer, size, compressedSize); + if (error != B_OK) + return error; + + // only use compressed data when we've actually saved space + if (compressedSize == size) + return B_BUFFER_OVERFLOW; + + return _WriteDataUncompressed(fCompressedDataBuffer, compressedSize); +} + + +status_t +PackageFileHeapWriter::_WriteDataUncompressed(const void* data, size_t size) +{ + ssize_t bytesWritten = pwrite(fFD, data, size, + fHeapOffset + (off_t)fCompressedHeapSize); + if (bytesWritten < 0) { + fErrorOutput->PrintError("Failed to write data: %s\n", strerror(errno)); + return errno; + } + if ((size_t)bytesWritten != size) { + fErrorOutput->PrintError("Failed to write all data\n"); + return B_ERROR; + } + + fCompressedHeapSize += bytesWritten; + + return B_OK; +} + + +void +PackageFileHeapWriter::_PushChunks(ChunkBuffer& chunkBuffer, uint64 startOffset, + uint64 endOffset) +{ + if (endOffset > fUncompressedHeapSize) { + fErrorOutput->PrintError("Invalid range to remove from heap\n"); + throw status_t(B_BAD_VALUE); + } + + ssize_t chunkIndex = startOffset / kChunkSize; + uint64 uncompressedChunkOffset = (uint64)chunkIndex * kChunkSize; + + while (startOffset < endOffset) { + uint32 inChunkOffset = uint32(startOffset - uncompressedChunkOffset); + uint32 uncompressedChunkSize = chunkIndex + 1 < fOffsets.Count() + ? kChunkSize + : fUncompressedHeapSize - uncompressedChunkOffset; + uint64 compressedChunkOffset = fOffsets[chunkIndex]; + uint32 compressedChunkSize = chunkIndex + 1 < fOffsets.Count() + ? fOffsets[chunkIndex + 1] - compressedChunkOffset + : fCompressedHeapSize - compressedChunkOffset; + uint32 toKeepSize = uint32(std::min( + (uint64)uncompressedChunkSize - inChunkOffset, + endOffset - startOffset)); + + if (!chunkBuffer.PushChunkSegment(compressedChunkOffset, + compressedChunkSize, uncompressedChunkSize, inChunkOffset, + toKeepSize)) { + throw std::bad_alloc(); + } + + startOffset += toKeepSize; + chunkIndex++; + uncompressedChunkOffset += uncompressedChunkSize; + } +} + + +} // namespace BPrivate + +} // namespace BHPKG + +} // namespace BPackageKit diff --git a/src/kits/package/hpkg/PackageReader.cpp b/src/kits/package/hpkg/PackageReader.cpp index 549bf2f9e6..8453d25b40 100644 --- a/src/kits/package/hpkg/PackageReader.cpp +++ b/src/kits/package/hpkg/PackageReader.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2009-2013, Ingo Weinhold, ingo_weinhold@gmx.de. * Distributed under the terms of the MIT License. */ @@ -9,6 +9,8 @@ #include #include + +#include #include @@ -80,6 +82,14 @@ BPackageReader::PackageFileFD() } + +BAbstractBufferedDataReader* +BPackageReader::HeapReader() const +{ + return fImpl != NULL ? fImpl->HeapReader() : NULL; +} + + } // namespace BHPKG } // namespace BPackageKit diff --git a/src/kits/package/hpkg/PackageReaderImpl.cpp b/src/kits/package/hpkg/PackageReaderImpl.cpp index 40551e7b8b..1446c31ae5 100644 --- a/src/kits/package/hpkg/PackageReaderImpl.cpp +++ b/src/kits/package/hpkg/PackageReaderImpl.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -48,80 +49,16 @@ static const size_t kMaxTOCSize = 64 * 1024 * 1024; static const size_t kMaxPackageAttributesSize = 1 * 1024 * 1024; -// #pragma mark - DataAttributeHandler - - -struct PackageReaderImpl::DataAttributeHandler : AttributeHandler { - DataAttributeHandler(BPackageData* data) - : - fData(data) - { - } - - static status_t InitData(AttributeHandlerContext* context, - BPackageData* data, const AttributeValue& value) - { - if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) - data->SetData(value.data.size, value.data.raw); - else - data->SetData(value.data.size, value.data.offset); - - data->SetUncompressedSize(value.data.size); - - return B_OK; - } - - static status_t Create(AttributeHandlerContext* context, - BPackageData* data, const AttributeValue& value, - AttributeHandler*& _handler) - { - DataAttributeHandler* handler = new(std::nothrow) DataAttributeHandler( - data); - if (handler == NULL) - return B_NO_MEMORY; - - InitData(context, data, value); - - _handler = handler; - return B_OK; - } - - virtual status_t HandleAttribute(AttributeHandlerContext* context, - uint8 id, const AttributeValue& value, AttributeHandler** _handler) - { - switch (id) { - case B_HPKG_ATTRIBUTE_ID_DATA_SIZE: - fData->SetUncompressedSize(value.unsignedInt); - return B_OK; - - case B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION: - { - switch (value.unsignedInt) { - case B_HPKG_COMPRESSION_NONE: - case B_HPKG_COMPRESSION_ZLIB: - break; - default: - context->errorOutput->PrintError("Error: Invalid " - "compression type for data (%llu)\n", - value.unsignedInt); - return B_BAD_DATA; - } - - fData->SetCompression(value.unsignedInt); - return B_OK; - } - - case B_HPKG_ATTRIBUTE_ID_DATA_CHUNK_SIZE: - fData->SetChunkSize(value.unsignedInt); - return B_OK; - } - - return AttributeHandler::HandleAttribute(context, id, value, _handler); - } - -private: - BPackageData* fData; -}; +static status_t +set_package_data_from_attribute_value(const BPackageAttributeValue& value, + BPackageData& data) +{ + if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) + data.SetData(value.data.size, value.data.raw); + else + data.SetData(value.data.size, value.data.offset); + return B_OK; +} // #pragma mark - AttributeAttributeHandler @@ -140,12 +77,8 @@ struct PackageReaderImpl::AttributeAttributeHandler : AttributeHandler { { switch (id) { case B_HPKG_ATTRIBUTE_ID_DATA: - if (_handler != NULL) { - return DataAttributeHandler::Create(context, - &fAttribute.Data(), value, *_handler); - } - return DataAttributeHandler::InitData(context, - &fAttribute.Data(), value); + return set_package_data_from_attribute_value(value, + fAttribute.Data()); case B_HPKG_ATTRIBUTE_ID_FILE_ATTRIBUTE_TYPE: fAttribute.SetType(value.unsignedInt); @@ -279,12 +212,8 @@ struct PackageReaderImpl::EntryAttributeHandler : AttributeHandler { } case B_HPKG_ATTRIBUTE_ID_DATA: - if (_handler != NULL) { - return DataAttributeHandler::Create(context, &fEntry.Data(), - value, *_handler); - } - return DataAttributeHandler::InitData(context, &fEntry.Data(), - value); + return set_package_data_from_attribute_value(value, + fEntry.Data()); case B_HPKG_ATTRIBUTE_ID_SYMLINK_PATH: fEntry.SetSymlinkPath(value.string); @@ -378,7 +307,7 @@ struct PackageReaderImpl::RootAttributeHandler : PackageAttributeHandler { PackageReaderImpl::PackageReaderImpl(BErrorOutput* errorOutput) : - inherited(errorOutput), + inherited("package", errorOutput), fTOCSection("TOC") { } @@ -407,179 +336,39 @@ PackageReaderImpl::Init(const char* fileName) status_t PackageReaderImpl::Init(int fd, bool keepFD) { - status_t error = inherited::Init(fd, keepFD); - if (error != B_OK) - return error; - - // stat it - struct stat st; - if (fstat(FD(), &st) < 0) { - ErrorOutput()->PrintError("Error: Failed to access package file: %s\n", - strerror(errno)); - return errno; - } - - // read the header hpkg_header header; - if ((error = ReadBuffer(0, &header, sizeof(header))) != B_OK) + status_t error = inherited::Init( + fd, keepFD, header); + if (error != B_OK) return error; + fHeapSize = HeapReader()->UncompressedHeapSize(); - // check the header - - // magic - if (B_BENDIAN_TO_HOST_INT32(header.magic) != B_HPKG_MAGIC) { - ErrorOutput()->PrintError("Error: Invalid package file: Invalid " - "magic\n"); - return B_BAD_DATA; - } - - // version - if (B_BENDIAN_TO_HOST_INT16(header.version) != B_HPKG_VERSION) { - ErrorOutput()->PrintError("Error: Invalid/unsupported package file " - "version (%d)\n", B_BENDIAN_TO_HOST_INT16(header.version)); - return B_MISMATCHED_VALUES; - } - - // header size - fHeapOffset = B_BENDIAN_TO_HOST_INT16(header.header_size); - if ((size_t)fHeapOffset < sizeof(hpkg_header)) { - ErrorOutput()->PrintError("Error: Invalid package file: Invalid header " - "size (%llu)\n", fHeapOffset); - return B_BAD_DATA; - } - - // total size - fTotalSize = B_BENDIAN_TO_HOST_INT64(header.total_size); - if (fTotalSize != (uint64)st.st_size) { - ErrorOutput()->PrintError("Error: Invalid package file: Total size in " - "header (%llu) doesn't agree with total file size (%lld)\n", - fTotalSize, st.st_size); - return B_BAD_DATA; - } - - // package attributes length and compression - fPackageAttributesSection.compression - = B_BENDIAN_TO_HOST_INT32(header.attributes_compression); - fPackageAttributesSection.compressedLength - = B_BENDIAN_TO_HOST_INT32(header.attributes_length_compressed); - fPackageAttributesSection.uncompressedLength - = B_BENDIAN_TO_HOST_INT32(header.attributes_length_uncompressed); - fPackageAttributesSection.stringsLength - = B_BENDIAN_TO_HOST_INT32(header.attributes_strings_length); - fPackageAttributesSection.stringsCount - = B_BENDIAN_TO_HOST_INT32(header.attributes_strings_count); - - if (const char* errorString = CheckCompression( - fPackageAttributesSection)) { - ErrorOutput()->PrintError("Error: Invalid package file: package " - "attributes section: %s\n", errorString); - return B_BAD_DATA; - } - - // TOC length and compression - fTOCSection.compression = B_BENDIAN_TO_HOST_INT32(header.toc_compression); - fTOCSection.compressedLength - = B_BENDIAN_TO_HOST_INT64(header.toc_length_compressed); - fTOCSection.uncompressedLength - = B_BENDIAN_TO_HOST_INT64(header.toc_length_uncompressed); - - if (const char* errorString = CheckCompression(fTOCSection)) { - ErrorOutput()->PrintError("Error: Invalid package file: TOC section: " - "%s\n", errorString); - return B_BAD_DATA; - } - - // TOC subsections - fTOCSection.stringsLength - = B_BENDIAN_TO_HOST_INT64(header.toc_strings_length); - fTOCSection.stringsCount - = B_BENDIAN_TO_HOST_INT64(header.toc_strings_count); - - if (fTOCSection.stringsLength > fTOCSection.uncompressedLength - || fTOCSection.stringsCount > fTOCSection.stringsLength) { - ErrorOutput()->PrintError("Error: Invalid package file: Invalid TOC " - "subsections description\n"); - return B_BAD_DATA; - } - - // check whether the sections fit together - if (fPackageAttributesSection.compressedLength > fTotalSize - || fTOCSection.compressedLength - > fTotalSize - fPackageAttributesSection.compressedLength - || fHeapOffset - > fTotalSize - fPackageAttributesSection.compressedLength - - fTOCSection.compressedLength) { - ErrorOutput()->PrintError("Error: Invalid package file: The sum of the " - "sections sizes is greater than the package size\n"); - return B_BAD_DATA; - } - - fPackageAttributesSection.offset - = fTotalSize - fPackageAttributesSection.compressedLength; - fTOCSection.offset = fPackageAttributesSection.offset - - fTOCSection.compressedLength; - fHeapSize = fTOCSection.offset - fHeapOffset; - - // TOC size sanity check - if (fTOCSection.uncompressedLength > kMaxTOCSize) { - ErrorOutput()->PrintError("Error: Package file TOC section size " - "is %llu bytes. This is beyond the reader's sanity limit\n", - fTOCSection.uncompressedLength); - return B_UNSUPPORTED; - } - - // package attributes size sanity check - if (fPackageAttributesSection.uncompressedLength - > kMaxPackageAttributesSize) { - ErrorOutput()->PrintError( - "Error: Package file package attributes section size " - "is %llu bytes. This is beyond the reader's sanity limit\n", - fPackageAttributesSection.uncompressedLength); - return B_UNSUPPORTED; - } - - // read in the complete TOC - fTOCSection.data - = new(std::nothrow) uint8[fTOCSection.uncompressedLength]; - if (fTOCSection.data == NULL) { - ErrorOutput()->PrintError("Error: Out of memory!\n"); - return B_NO_MEMORY; - } - error = ReadCompressedBuffer(fTOCSection); + // init package attributes section + error = InitSection(fPackageAttributesSection, fHeapSize, + B_BENDIAN_TO_HOST_INT32(header.attributes_length), + kMaxPackageAttributesSize, + B_BENDIAN_TO_HOST_INT32(header.attributes_strings_length), + B_BENDIAN_TO_HOST_INT32(header.attributes_strings_count)); if (error != B_OK) return error; - // read in the complete package attributes section - fPackageAttributesSection.data - = new(std::nothrow) uint8[fPackageAttributesSection.uncompressedLength]; - if (fPackageAttributesSection.data == NULL) { - ErrorOutput()->PrintError("Error: Out of memory!\n"); - return B_NO_MEMORY; - } - error = ReadCompressedBuffer(fPackageAttributesSection); + // init TOC section + error = InitSection(fTOCSection, fPackageAttributesSection.offset, + B_BENDIAN_TO_HOST_INT64(header.toc_length), kMaxTOCSize, + B_BENDIAN_TO_HOST_INT64(header.toc_strings_length), + B_BENDIAN_TO_HOST_INT64(header.toc_strings_count)); if (error != B_OK) return error; - // start parsing the TOC - fTOCSection.currentOffset = 0; - SetCurrentSection(&fTOCSection); - - // strings - error = ParseStrings(); + // prepare the sections for use + error = PrepareSection(fTOCSection); if (error != B_OK) return error; - // parse strings from package attributes section - fPackageAttributesSection.currentOffset = 0; - SetCurrentSection(&fPackageAttributesSection); - - // strings - error = ParseStrings(); + error = PrepareSection(fPackageAttributesSection); if (error != B_OK) return error; - SetCurrentSection(NULL); - return B_OK; } @@ -630,10 +419,6 @@ PackageReaderImpl::_ParseTOC(AttributeHandlerContext* context, fTOCSection.currentOffset = fTOCSection.stringsLength; SetCurrentSection(&fTOCSection); - // prepare attribute handler context - context->heapOffset = fHeapOffset; - context->heapSize = fHeapSize; - // init the attribute handler stack rootAttributeHandler->SetLevel(0); ClearAttributeHandlerStack(); @@ -688,7 +473,7 @@ PackageReaderImpl::ReadAttributeValue(uint8 type, uint8 encoding, return B_BAD_DATA; } - _value.SetToData(size, fHeapOffset + offset); + _value.SetToData(size, offset); } else if (encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) { if (size > B_HPKG_MAX_INLINE_DATA_SIZE) { ErrorOutput()->PrintError("Error: Invalid %s section: " diff --git a/src/kits/package/hpkg/PackageWriterImpl.cpp b/src/kits/package/hpkg/PackageWriterImpl.cpp index 3778cd36e9..fc407ea7fc 100644 --- a/src/kits/package/hpkg/PackageWriterImpl.cpp +++ b/src/kits/package/hpkg/PackageWriterImpl.cpp @@ -39,6 +39,7 @@ #include #include +#include #include #include @@ -156,12 +157,11 @@ struct PackageWriterImpl::Attribute struct PackageWriterImpl::PackageContentHandler : BLowLevelPackageContentHandler { PackageContentHandler(Attribute* rootAttribute, BErrorOutput* errorOutput, - StringCache& stringCache, uint64 heapOffset) + StringCache& stringCache) : fErrorOutput(errorOutput), fStringCache(stringCache), fRootAttribute(rootAttribute), - fHeapOffset(heapOffset), fErrorOccurred(false) { } @@ -212,7 +212,7 @@ struct PackageWriterImpl::PackageContentHandler case B_HPKG_ATTRIBUTE_TYPE_RAW: if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_HEAP) { attribute->value.SetToData(value.data.size, - value.data.offset - fHeapOffset); + value.data.offset); } else if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) { attribute->value.SetToData(value.data.size, value.data.raw); @@ -249,7 +249,6 @@ private: BErrorOutput* fErrorOutput; StringCache& fStringCache; Attribute* fRootAttribute; - uint64 fHeapOffset; bool fErrorOccurred; }; @@ -394,8 +393,8 @@ private: struct PackageWriterImpl::HeapAttributeOffsetter { - HeapAttributeOffsetter(const RangeArray& ranges, - const Array& deltas) + HeapAttributeOffsetter(const RangeArray& ranges, + const Array& deltas) : fRanges(ranges), fDeltas(deltas) @@ -409,7 +408,7 @@ struct PackageWriterImpl::HeapAttributeOffsetter { if (value.type == B_HPKG_ATTRIBUTE_TYPE_RAW && value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_HEAP) { - off_t delta = fDeltas[fRanges.InsertionIndex(value.data.offset)]; + uint64 delta = fDeltas[fRanges.InsertionIndex(value.data.offset)]; value.data.offset -= delta; } @@ -422,8 +421,8 @@ struct PackageWriterImpl::HeapAttributeOffsetter { } private: - const RangeArray& fRanges; - const Array& fDeltas; + const RangeArray& fRanges; + const Array& fDeltas; }; @@ -445,11 +444,9 @@ PackageWriterImpl::_AddAttribute(BHPKGAttributeID attributeID, Type value) PackageWriterImpl::PackageWriterImpl(BPackageWriterListener* listener) : - inherited(listener), + inherited("package", listener), fListener(listener), fHeapRangesToRemove(NULL), - fDataBuffer(NULL), - fDataBufferSize(2 * B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB), fRootEntry(NULL), fRootAttribute(NULL), fTopAttribute(NULL), @@ -461,10 +458,7 @@ PackageWriterImpl::PackageWriterImpl(BPackageWriterListener* listener) PackageWriterImpl::~PackageWriterImpl() { delete fRootAttribute; - delete fRootEntry; - - free(fDataBuffer); } @@ -566,7 +560,7 @@ status_t PackageWriterImpl::Finish() { try { - RangeArray heapRangesToRemove; + RangeArray heapRangesToRemove; fHeapRangesToRemove = &heapRangesToRemove; if ((Flags() & B_HPKG_WRITER_UPDATE_PACKAGE) != 0) { @@ -610,15 +604,10 @@ PackageWriterImpl::Finish() status_t PackageWriterImpl::_Init(const char* fileName, uint32 flags) { - status_t result = inherited::Init(fileName, "package", flags); + status_t result = inherited::Init(fileName, flags); if (result != B_OK) return result; - // allocate data buffer - fDataBuffer = malloc(fDataBufferSize); - if (fDataBuffer == NULL) - throw std::bad_alloc(); - if (fStringCache.Init() != B_OK) throw std::bad_alloc(); @@ -627,7 +616,7 @@ PackageWriterImpl::_Init(const char* fileName, uint32 flags) fRootAttribute = new Attribute(); - fHeapOffset = fHeapEnd = sizeof(hpkg_header); + fHeapOffset = fHeaderSize = sizeof(hpkg_header); fTopAttribute = fRootAttribute; // in update mode, parse the TOC @@ -638,19 +627,14 @@ PackageWriterImpl::_Init(const char* fileName, uint32 flags) return result; fHeapOffset = packageReader.HeapOffset(); - fHeapEnd = fHeapOffset + packageReader.HeapSize(); - PackageContentHandler handler(fRootAttribute, fListener, fStringCache, - fHeapOffset); + PackageContentHandler handler(fRootAttribute, fListener, fStringCache); result = packageReader.ParseContent(&handler); if (result != B_OK) return result; - if ((uint64)fHeapOffset > packageReader.HeapOffset()) { - fListener->PrintError("Unexpected heap offset in package file.\n"); - return B_BAD_DATA; - } + fHeapWriter->Reinit(packageReader.HeapReader()); } return B_OK; @@ -815,99 +799,22 @@ PackageWriterImpl::_UpdateReadPackageInfo() if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) data.SetData(value.data.size, value.data.raw); else - data.SetData(value.data.size, value.data.offset + fHeapOffset); - - // get the compression - uint8 compression = B_HPKG_DEFAULT_DATA_COMPRESSION; - if (Attribute* compressionAttribute = dataAttribute->ChildWithID( - B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION)) { - if (compressionAttribute->value.type != B_HPKG_ATTRIBUTE_TYPE_UINT) { - fListener->PrintError("%s entry in package file has an invalid " - "data compression attribute (not of type uint).\n", - B_HPKG_PACKAGE_INFO_FILE_NAME); - throw status_t(B_BAD_DATA); - } - compression = compressionAttribute->value.unsignedInt; - } - - data.SetCompression(compression); - - // get the size - uint64 size; - Attribute* sizeAttribute = dataAttribute->ChildWithID( - B_HPKG_ATTRIBUTE_ID_DATA_SIZE); - if (sizeAttribute == NULL) { - size = value.data.size; - } else if (sizeAttribute->value.type != B_HPKG_ATTRIBUTE_TYPE_UINT) { - fListener->PrintError("%s entry in package file has an invalid data " - "size attribute (not of type uint).\n", - B_HPKG_PACKAGE_INFO_FILE_NAME); - throw status_t(B_BAD_DATA); - } else - size = sizeAttribute->value.unsignedInt; - - data.SetUncompressedSize(size); - - // get the chunk size - uint64 chunkSize = compression == B_HPKG_COMPRESSION_ZLIB - ? B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB : 0; - if (Attribute* chunkSizeAttribute = dataAttribute->ChildWithID( - B_HPKG_ATTRIBUTE_ID_DATA_CHUNK_SIZE)) { - if (chunkSizeAttribute->value.type != B_HPKG_ATTRIBUTE_TYPE_UINT) { - fListener->PrintError("%s entry in package file has an invalid " - "data chunk size attribute (not of type uint).\n", - B_HPKG_PACKAGE_INFO_FILE_NAME); - throw status_t(B_BAD_DATA); - } - chunkSize = chunkSizeAttribute->value.unsignedInt; - } - - data.SetChunkSize(chunkSize); + data.SetData(value.data.size, value.data.offset); // read the value into a string BString valueString; - char* valueBuffer = valueString.LockBuffer(size); + char* valueBuffer = valueString.LockBuffer(value.data.size); if (valueBuffer == NULL) throw std::bad_alloc(); if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) { // data encoded inline -- just copy to buffer - if (size != value.data.size) { - fListener->PrintError("%s entry in package file has an invalid " - "data attribute (mismatching size).\n", - B_HPKG_PACKAGE_INFO_FILE_NAME); - throw status_t(B_BAD_DATA); - } memcpy(valueBuffer, value.data.raw, value.data.size); } else { // data on heap -- read from there - BBlockBufferPoolNoLock bufferCache(16 * 1024, 1); - status_t error = bufferCache.Init(); - if (error != B_OK) { - fListener->PrintError("Failed to initialize buffer cache: %s\n", - strerror(error)); - throw status_t(error); - } - - // create a PackageDataReader - BFDDataReader packageFileReader(FD()); - BAbstractBufferedDataReader* reader; - error = BPackageDataReaderFactory(&bufferCache) - .CreatePackageDataReader(&packageFileReader, data, reader); - if (error != B_OK) { - fListener->PrintError("Failed to create package data reader: %s\n", - strerror(error)); - throw status_t(error); - } - ObjectDeleter readerDeleter(reader); - - // read the data - error = reader->ReadData(0, valueBuffer, size); - if (error != B_OK) { - fListener->PrintError("Failed to read data of %s entry in package " - "file: %s\n", B_HPKG_PACKAGE_INFO_FILE_NAME, strerror(error)); - throw status_t(error); - } + status_t error = fHeapWriter->ReadData(data.Offset(), valueBuffer, + data.Size()); + throw error; } valueString.UnlockBuffer(); @@ -1101,8 +1008,8 @@ PackageWriterImpl::_CompactHeap() return; // compute the move deltas for the ranges - Array deltas; - off_t delta = 0; + Array deltas; + uint64 delta = 0; for (int32 i = 0; i < count; i++) { if (!deltas.Add(delta)) throw std::bad_alloc(); @@ -1117,73 +1024,8 @@ PackageWriterImpl::_CompactHeap() HeapAttributeOffsetter(*fHeapRangesToRemove, deltas).ProcessAttribute( fRootAttribute); - // move the heap chunks in the file around - off_t chunkOffset = fHeapOffset; - delta = 0; - - for (int32 i = 0; i < count; i++) { - const Range& range = fHeapRangesToRemove->RangeAt(i); - - if (delta > 0 && chunkOffset < range.offset) { - // move chunk - _MoveHeapChunk(chunkOffset, chunkOffset - delta, - range.offset - chunkOffset); - } - - chunkOffset = range.EndOffset(); - delta += range.size; - } - - // move the final chunk - off_t heapSize = fHeapEnd - fHeapOffset; - if (delta > 0 && chunkOffset < heapSize) { - _MoveHeapChunk(chunkOffset, chunkOffset - delta, - heapSize - chunkOffset); - } - - fHeapEnd -= delta; -} - - -void -PackageWriterImpl::_MoveHeapChunk(off_t fromOffset, off_t toOffset, off_t size) -{ - // convert heap offsets to file offsets - fromOffset += fHeapOffset; - toOffset += fHeapOffset; - - while (size > 0) { - size_t toCopy = std::min(size, (off_t)fDataBufferSize); - - // read data into buffer - ssize_t bytesRead = read_pos(FD(), fromOffset, fDataBuffer, toCopy); - if (bytesRead < 0) { - fListener->PrintError("Failed to read from package file: %s\n", - strerror(errno)); - throw status_t(errno); - } - if ((size_t)bytesRead < toCopy) { - fListener->PrintError("Failed to read from package file (wanted " - "%zu bytes, got %zd).\n", toCopy, bytesRead); - throw status_t(B_IO_ERROR); - } - - // write data to target offset - ssize_t bytesWritten = write_pos(FD(), toOffset, fDataBuffer, toCopy); - if (bytesWritten < 0) { - fListener->PrintError("Failed to write to package file: %s\n", - strerror(errno)); - throw status_t(errno); - } - if ((size_t)bytesWritten < toCopy) { - fListener->PrintError("Failed to write to package file.\n"); - throw status_t(B_IO_ERROR); - } - - fromOffset += toCopy; - toOffset += toCopy; - size -= toCopy; - } + // remove the ranges from the heap + fHeapWriter->RemoveDataRanges(*fHeapRangesToRemove); } @@ -1216,41 +1058,52 @@ PackageWriterImpl::_Finish() _AddEntry(AT_FDCWD, entry, entry->Name(), pathBuffer); } - off_t heapSize = fHeapEnd - fHeapOffset; - hpkg_header header; // write the TOC and package attributes - _WriteTOC(header); - _WritePackageAttributes(header); + uint64 tocLength; + _WriteTOC(header, tocLength); - off_t totalSize = fHeapEnd; + uint64 attributesLength; + _WritePackageAttributes(header, attributesLength); + + // flush the heap + status_t error = fHeapWriter->Finish(); + if (error != B_OK) + return error; + + uint64 compressedHeapSize = fHeapWriter->CompressedHeapSize(); + + header.heap_compression = B_HOST_TO_BENDIAN_INT32(B_HPKG_COMPRESSION_ZLIB); + header.heap_chunk_size = B_HOST_TO_BENDIAN_INT32(fHeapWriter->ChunkSize()); + header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64( + fHeapWriter->CompressedHeapSize()); + header.heap_size_uncompressed = B_HOST_TO_BENDIAN_INT64( + fHeapWriter->UncompressedHeapSize()); // Truncate the file to the size it is supposed to have. In update mode, it - // can be greater when one or more files are shrunk. In creation mode, when - // writing compressed TOC or package attributes yields a larger size than - // uncompressed, the file size may also be greater than it should be. + // can be greater when one or more files are shrunk. In creation mode it + // should already have the correct size. + off_t totalSize = fHeapWriter->HeapOffset() + (off_t)compressedHeapSize; if (ftruncate(FD(), totalSize) != 0) { fListener->PrintError("Failed to truncate package file to new " "size: %s\n", strerror(errno)); return errno; } - fListener->OnPackageSizeInfo(fHeapOffset, heapSize, - B_BENDIAN_TO_HOST_INT64(header.toc_length_compressed), - B_BENDIAN_TO_HOST_INT32(header.attributes_length_compressed), - totalSize); + fListener->OnPackageSizeInfo(fHeaderSize, compressedHeapSize, tocLength, + attributesLength, totalSize); // prepare the header // general header.magic = B_HOST_TO_BENDIAN_INT32(B_HPKG_MAGIC); - header.header_size = B_HOST_TO_BENDIAN_INT16((uint16)fHeapOffset); + header.header_size = B_HOST_TO_BENDIAN_INT16(fHeaderSize); header.version = B_HOST_TO_BENDIAN_INT16(B_HPKG_VERSION); header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize); // write the header - WriteBuffer(&header, sizeof(hpkg_header), 0); + RawWriteBuffer(&header, sizeof(hpkg_header), 0); SetFinished(true); return B_OK; @@ -1332,107 +1185,32 @@ PackageWriterImpl::_RegisterEntry(Entry* parent, const char* name, void -PackageWriterImpl::_WriteTOC(hpkg_header& header) +PackageWriterImpl::_WriteTOC(hpkg_header& header, uint64& _length) { - // prepare the writer (zlib writer on top of a file writer) - off_t startOffset = fHeapEnd; + // write the subsections + uint64 startOffset = fHeapWriter->UncompressedHeapSize(); - // write the sections - uint32 compression = B_HPKG_COMPRESSION_ZLIB; - uint64 uncompressedStringsSize; - uint64 uncompressedMainSize; - uint64 tocUncompressedSize; - int32 cachedStringsWritten = _WriteTOCCompressed(uncompressedStringsSize, - uncompressedMainSize, tocUncompressedSize); - - off_t endOffset = fHeapEnd; - - if (endOffset - startOffset >= (off_t)tocUncompressedSize) { - // the compressed section isn't shorter -- write uncompressed - fHeapEnd = startOffset; - compression = B_HPKG_COMPRESSION_NONE; - cachedStringsWritten = _WriteTOCUncompressed(uncompressedStringsSize, - uncompressedMainSize, tocUncompressedSize); - - endOffset = fHeapEnd; - } - - fListener->OnTOCSizeInfo(uncompressedStringsSize, uncompressedMainSize, - tocUncompressedSize); - - // update the header - - // TOC - header.toc_compression = B_HOST_TO_BENDIAN_INT32(compression); - header.toc_length_compressed = B_HOST_TO_BENDIAN_INT64( - endOffset - startOffset); - header.toc_length_uncompressed = B_HOST_TO_BENDIAN_INT64( - tocUncompressedSize); - - // TOC subsections - header.toc_strings_length = B_HOST_TO_BENDIAN_INT64( - uncompressedStringsSize); - header.toc_strings_count = B_HOST_TO_BENDIAN_INT64(cachedStringsWritten); -} - - -int32 -PackageWriterImpl::_WriteTOCCompressed(uint64& _uncompressedStringsSize, - uint64& _uncompressedMainSize, uint64& _tocUncompressedSize) -{ - FDDataWriter realWriter(FD(), fHeapEnd, fListener); - ZlibDataWriter zlibWriter(&realWriter); - SetDataWriter(&zlibWriter); - zlibWriter.Init(); - - // write the sections - int32 cachedStringsWritten - = _WriteTOCSections(_uncompressedStringsSize, _uncompressedMainSize); - - // finish the writer - zlibWriter.Finish(); - fHeapEnd = realWriter.Offset(); - SetDataWriter(NULL); - - _tocUncompressedSize = zlibWriter.BytesWritten(); - return cachedStringsWritten; -} - - -int32 -PackageWriterImpl::_WriteTOCUncompressed(uint64& _uncompressedStringsSize, - uint64& _uncompressedMainSize, uint64& _tocUncompressedSize) -{ - FDDataWriter realWriter(FD(), fHeapEnd, fListener); - SetDataWriter(&realWriter); - - // write the sections - int32 cachedStringsWritten - = _WriteTOCSections(_uncompressedStringsSize, _uncompressedMainSize); - - fHeapEnd = realWriter.Offset(); - SetDataWriter(NULL); - - _tocUncompressedSize = realWriter.BytesWritten(); - return cachedStringsWritten; -} - - -int32 -PackageWriterImpl::_WriteTOCSections(uint64& _stringsSize, uint64& _mainSize) -{ - // write the cached strings - uint64 cachedStringsOffset = DataWriter()->BytesWritten(); + // cached strings + uint64 cachedStringsOffset = fHeapWriter->UncompressedHeapSize(); int32 cachedStringsWritten = WriteCachedStrings(fStringCache, 2); - // write the main TOC section - uint64 mainOffset = DataWriter()->BytesWritten(); + // main TOC section + uint64 mainOffset = fHeapWriter->UncompressedHeapSize(); _WriteAttributeChildren(fRootAttribute); - _stringsSize = mainOffset - cachedStringsOffset; - _mainSize = DataWriter()->BytesWritten() - mainOffset; + // notify the listener + uint64 endOffset = fHeapWriter->UncompressedHeapSize(); + uint64 stringsSize = mainOffset - cachedStringsOffset; + uint64 mainSize = endOffset - mainOffset; + uint64 tocSize = endOffset - startOffset; + fListener->OnTOCSizeInfo(stringsSize, mainSize, tocSize); - return cachedStringsWritten; + // update the header + header.toc_length = B_HOST_TO_BENDIAN_INT64(tocSize); + header.toc_strings_length = B_HOST_TO_BENDIAN_INT64(stringsSize); + header.toc_strings_count = B_HOST_TO_BENDIAN_INT64(cachedStringsWritten); + + _length = tocSize; } @@ -1459,85 +1237,25 @@ PackageWriterImpl::_WriteAttributeChildren(Attribute* attribute) void -PackageWriterImpl::_WritePackageAttributes(hpkg_header& header) +PackageWriterImpl::_WritePackageAttributes(hpkg_header& header, uint64& _length) { - // write the package attributes (zlib writer on top of a file writer) - off_t startOffset = fHeapEnd; + // write cached strings and package attributes tree + off_t startOffset = fHeapWriter->UncompressedHeapSize(); - uint32 compression = B_HPKG_COMPRESSION_ZLIB; - uint32 stringsLengthUncompressed; - uint32 attributesLengthUncompressed; - uint32 stringsCount = _WritePackageAttributesCompressed( - stringsLengthUncompressed, attributesLengthUncompressed); + uint32 stringsLength; + uint32 stringsCount = WritePackageAttributes(PackageAttributes(), + stringsLength); - off_t endOffset = fHeapEnd; - - if ((off_t)attributesLengthUncompressed <= endOffset - startOffset) { - // the compressed section isn't shorter -- write uncompressed - fHeapEnd = startOffset; - compression = B_HPKG_COMPRESSION_NONE; - stringsCount = _WritePackageAttributesUncompressed( - stringsLengthUncompressed, attributesLengthUncompressed); - - endOffset = fHeapEnd; - } - - fListener->OnPackageAttributesSizeInfo(stringsCount, - attributesLengthUncompressed); + // notify listener + uint32 attributesLength = fHeapWriter->UncompressedHeapSize() - startOffset; + fListener->OnPackageAttributesSizeInfo(stringsCount, attributesLength); // update the header - header.attributes_compression = B_HOST_TO_BENDIAN_INT32(compression); - header.attributes_length_compressed - = B_HOST_TO_BENDIAN_INT32(endOffset - startOffset); - header.attributes_length_uncompressed - = B_HOST_TO_BENDIAN_INT32(attributesLengthUncompressed); + header.attributes_length = B_HOST_TO_BENDIAN_INT32(attributesLength); header.attributes_strings_count = B_HOST_TO_BENDIAN_INT32(stringsCount); - header.attributes_strings_length - = B_HOST_TO_BENDIAN_INT32(stringsLengthUncompressed); -} + header.attributes_strings_length = B_HOST_TO_BENDIAN_INT32(stringsLength); - -uint32 -PackageWriterImpl::_WritePackageAttributesCompressed( - uint32& _stringsLengthUncompressed, uint32& _attributesLengthUncompressed) -{ - off_t startOffset = fHeapEnd; - FDDataWriter realWriter(FD(), startOffset, fListener); - ZlibDataWriter zlibWriter(&realWriter); - SetDataWriter(&zlibWriter); - zlibWriter.Init(); - - // write cached strings and package attributes tree - uint32 stringsCount = WritePackageAttributes(PackageAttributes(), - _stringsLengthUncompressed); - - zlibWriter.Finish(); - fHeapEnd = realWriter.Offset(); - SetDataWriter(NULL); - - _attributesLengthUncompressed = zlibWriter.BytesWritten(); - return stringsCount; -} - - -uint32 -PackageWriterImpl::_WritePackageAttributesUncompressed( - uint32& _stringsLengthUncompressed, uint32& _attributesLengthUncompressed) -{ - off_t startOffset = fHeapEnd; - FDDataWriter realWriter(FD(), startOffset, fListener); - - SetDataWriter(&realWriter); - - // write cached strings and package attributes tree - uint32 stringsCount = WritePackageAttributes(PackageAttributes(), - _stringsLengthUncompressed); - - fHeapEnd = realWriter.Offset(); - SetDataWriter(NULL); - - _attributesLengthUncompressed = realWriter.BytesWritten(); - return stringsCount; + _length = attributesLength; } @@ -1804,178 +1522,13 @@ PackageWriterImpl::_AddData(BDataReader& dataReader, off_t size) return B_OK; } - // longer data -- try to compress - uint64 dataOffset = fHeapEnd; - - uint64 compression = B_HPKG_COMPRESSION_NONE; - uint64 compressedSize; - - status_t error = _WriteZlibCompressedData(dataReader, size, dataOffset, - compressedSize); - if (error == B_OK) { - compression = B_HPKG_COMPRESSION_ZLIB; - } else { - error = _WriteUncompressedData(dataReader, size, dataOffset); - compressedSize = size; - } + // add data to heap + uint64 dataOffset; + status_t error = fHeapWriter->AddData(dataReader, size, dataOffset); if (error != B_OK) return error; - fHeapEnd = dataOffset + compressedSize; - - // add data attribute - Attribute* dataAttribute = _AddDataAttribute(B_HPKG_ATTRIBUTE_ID_DATA, - compressedSize, dataOffset - fHeapOffset); - Stacker attributeAttributeStacker(fTopAttribute, dataAttribute); - - // if compressed, add compression attributes - if (compression != B_HPKG_COMPRESSION_NONE) { - _AddAttribute(B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION, compression); - _AddAttribute(B_HPKG_ATTRIBUTE_ID_DATA_SIZE, (uint64)size); - // uncompressed size - } - - return B_OK; -} - - -status_t -PackageWriterImpl::_WriteUncompressedData(BDataReader& dataReader, off_t size, - uint64 writeOffset) -{ - // copy the data to the heap - off_t readOffset = 0; - off_t remainingSize = size; - while (remainingSize > 0) { - // read data - size_t toCopy = std::min(remainingSize, (off_t)fDataBufferSize); - status_t error = dataReader.ReadData(readOffset, fDataBuffer, toCopy); - if (error != B_OK) { - fListener->PrintError("Failed to read data: %s\n", strerror(error)); - return error; - } - - // write to heap - ssize_t bytesWritten = pwrite(FD(), fDataBuffer, toCopy, writeOffset); - if (bytesWritten < 0) { - fListener->PrintError("Failed to write data: %s\n", - strerror(errno)); - return errno; - } - if ((size_t)bytesWritten != toCopy) { - fListener->PrintError("Failed to write all data\n"); - return B_ERROR; - } - - remainingSize -= toCopy; - readOffset += toCopy; - writeOffset += toCopy; - } - - return B_OK; -} - - -status_t -PackageWriterImpl::_WriteZlibCompressedData(BDataReader& dataReader, off_t size, - uint64 writeOffset, uint64& _compressedSize) -{ - // Use zlib compression only for data large enough. - if (size < (off_t)kZlibCompressionSizeThreshold) - return B_BAD_VALUE; - - // fDataBuffer is 2 * B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, so split it into - // two halves we can use for reading and compressing - const size_t chunkSize = B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB; - uint8* inputBuffer = (uint8*)fDataBuffer; - uint8* outputBuffer = (uint8*)fDataBuffer + chunkSize; - - // account for the offset table - uint64 chunkCount = (size + (chunkSize - 1)) / chunkSize; - off_t offsetTableOffset = writeOffset; - uint64* offsetTable = NULL; - if (chunkCount > 1) { - offsetTable = new uint64[chunkCount - 1]; - writeOffset = offsetTableOffset + (chunkCount - 1) * sizeof(uint64); - } - ArrayDeleter offsetTableDeleter(offsetTable); - - const uint64 dataOffset = writeOffset; - const uint64 dataEndLimit = offsetTableOffset + size; - - // read the data, compress them and write them to the heap - off_t readOffset = 0; - off_t remainingSize = size; - uint64 chunkIndex = 0; - while (remainingSize > 0) { - // read data - size_t toCopy = std::min(remainingSize, (off_t)chunkSize); - status_t error = dataReader.ReadData(readOffset, inputBuffer, toCopy); - if (error != B_OK) { - fListener->PrintError("Failed to read data: %s\n", strerror(error)); - return error; - } - - // compress - size_t compressedSize; - error = ZlibCompressor::CompressSingleBuffer(inputBuffer, toCopy, - outputBuffer, toCopy, compressedSize); - - const void* writeBuffer; - size_t bytesToWrite; - if (error == B_OK) { - writeBuffer = outputBuffer; - bytesToWrite = compressedSize; - } else { - if (error != B_BUFFER_OVERFLOW) - return error; - writeBuffer = inputBuffer; - bytesToWrite = toCopy; - } - - // check the total compressed data size - if (writeOffset + bytesToWrite >= dataEndLimit) - return B_BUFFER_OVERFLOW; - - if (chunkIndex > 0) - offsetTable[chunkIndex - 1] = writeOffset - dataOffset; - - // write to heap - ssize_t bytesWritten = pwrite(FD(), writeBuffer, bytesToWrite, - writeOffset); - if (bytesWritten < 0) { - fListener->PrintError("Failed to write data: %s\n", - strerror(errno)); - return errno; - } - if ((size_t)bytesWritten != bytesToWrite) { - fListener->PrintError("Failed to write all data\n"); - return B_ERROR; - } - - remainingSize -= toCopy; - readOffset += toCopy; - writeOffset += bytesToWrite; - chunkIndex++; - } - - // write the offset table - if (chunkCount > 1) { - size_t bytesToWrite = (chunkCount - 1) * sizeof(uint64); - ssize_t bytesWritten = pwrite(FD(), offsetTable, bytesToWrite, - offsetTableOffset); - if (bytesWritten < 0) { - fListener->PrintError("Failed to write data: %s\n", - strerror(errno)); - return errno; - } - if ((size_t)bytesWritten != bytesToWrite) { - fListener->PrintError("Failed to write all data\n"); - return B_ERROR; - } - } - - _compressedSize = writeOffset - offsetTableOffset; + _AddDataAttribute(B_HPKG_ATTRIBUTE_ID_DATA, size, dataOffset); return B_OK; } diff --git a/src/kits/package/hpkg/ReaderImplBase.cpp b/src/kits/package/hpkg/ReaderImplBase.cpp index 5137d70213..434d6efc1a 100644 --- a/src/kits/package/hpkg/ReaderImplBase.cpp +++ b/src/kits/package/hpkg/ReaderImplBase.cpp @@ -21,6 +21,7 @@ #include #include +#include #include @@ -536,12 +537,14 @@ ReaderImplBase::LowLevelAttributeHandler::Delete( // #pragma mark - ReaderImplBase -ReaderImplBase::ReaderImplBase(BErrorOutput* errorOutput) +ReaderImplBase::ReaderImplBase(const char* fileType, BErrorOutput* errorOutput) : fPackageAttributesSection("package attributes"), + fFileType(fileType), fErrorOutput(errorOutput), fFD(-1), fOwnsFD(false), + fHeapReader(NULL), fCurrentSection(NULL), fScratchBuffer(NULL), fScratchBufferSize(0) @@ -551,6 +554,8 @@ ReaderImplBase::ReaderImplBase(BErrorOutput* errorOutput) ReaderImplBase::~ReaderImplBase() { + delete fHeapReader; + if (fOwnsFD && fFD >= 0) close(fFD); @@ -559,7 +564,7 @@ ReaderImplBase::~ReaderImplBase() status_t -ReaderImplBase::Init(int fd, bool keepFD) +ReaderImplBase::_Init(int fd, bool keepFD) { fFD = fd; fOwnsFD = keepFD; @@ -576,27 +581,84 @@ ReaderImplBase::Init(int fd, bool keepFD) } -const char* -ReaderImplBase::CheckCompression(const SectionInfo& section) const +status_t +ReaderImplBase::InitHeapReader(uint32 compression, uint32 chunkSize, + off_t offset, uint64 compressedSize, uint64 uncompressedSize) { - switch (section.compression) { - case B_HPKG_COMPRESSION_NONE: - if (section.compressedLength != section.uncompressedLength) { - return "Uncompressed, but compressed and uncompressed length " - "don't match"; - } - return NULL; - - case B_HPKG_COMPRESSION_ZLIB: - if (section.compressedLength >= section.uncompressedLength) { - return "Compressed, but compressed length is not less than " - "uncompressed length"; - } - return NULL; - - default: - return "Invalid compression algorithm ID"; + if (compression != B_HPKG_COMPRESSION_ZLIB) { + fErrorOutput->PrintError("Error: Invalid heap compression\n"); + return B_BAD_DATA; } + + fHeapReader = new(std::nothrow) PackageFileHeapReader(fErrorOutput, fFD, + offset, compressedSize, uncompressedSize); + return fHeapReader->Init(); +} + + +status_t +ReaderImplBase::InitSection(PackageFileSection& section, uint64 endOffset, + uint64 length, uint64 maxSaneLength, uint64 stringsLength, + uint64 stringsCount) +{ + // check length vs. endOffset + if (length > endOffset) { + ErrorOutput()->PrintError("Error: %s file %s section size is %" + B_PRIu64 " bytes. This is greater than the available space\n", + fFileType, section.name, length); + return B_BAD_DATA; + } + + // check sanity length + if (maxSaneLength > 0 && length > maxSaneLength) { + ErrorOutput()->PrintError("Error: %s file %s section size is %" + B_PRIu64 " bytes. This is beyond the reader's sanity limit\n", + fFileType, section.name, length); + return B_NOT_SUPPORTED; + } + + // check strings subsection size/count + if ((stringsLength == 0) != (stringsCount == 0) || stringsLength > length) { + ErrorOutput()->PrintError("Error: strings subsection description of %s " + "file %s section is invalid (%" B_PRIu64 " strings, length: %" + B_PRIu64 ")\n", + fFileType, section.name, length, stringsCount, stringsLength); + return B_BAD_DATA; + } + + section.uncompressedLength = length; + section.offset = endOffset - length; + section.currentOffset = 0; + section.stringsLength = stringsLength; + section.stringsCount = stringsCount; + + return B_OK; +} + + +status_t +ReaderImplBase::PrepareSection(PackageFileSection& section) +{ + // allocate memory for the section data and read it in + section.data = new(std::nothrow) uint8[section.uncompressedLength]; + if (section.data == NULL) { + ErrorOutput()->PrintError("Error: Out of memory!\n"); + return B_NO_MEMORY; + } + + status_t error = ReadSection(section); + if (error != B_OK) + return error; + + // parse the section strings + section.currentOffset = 0; + SetCurrentSection(§ion); + + error = ParseStrings(); + if (error != B_OK) + return error; + + return B_OK; } @@ -982,8 +1044,9 @@ ReaderImplBase::_ReadSectionBuffer(void* buffer, size_t size) { if (size > fCurrentSection->uncompressedLength - fCurrentSection->currentOffset) { - fErrorOutput->PrintError("_ReadBuffer(%lu): read beyond %s end\n", - size, fCurrentSection->name); + fErrorOutput->PrintError( + "_ReadSectionBuffer(%lu): read beyond %s end\n", size, + fCurrentSection->name); return B_BAD_DATA; } @@ -1014,63 +1077,11 @@ ReaderImplBase::ReadBuffer(off_t offset, void* buffer, size_t size) status_t -ReaderImplBase::ReadCompressedBuffer(const SectionInfo& section) +ReaderImplBase::ReadSection(const PackageFileSection& section) { - uint32 compressedSize = section.compressedLength; - uint64 offset = section.offset; - - switch (section.compression) { - case B_HPKG_COMPRESSION_NONE: - return ReadBuffer(offset, section.data, compressedSize); - - case B_HPKG_COMPRESSION_ZLIB: - { - // init the decompressor - BBufferDataOutput bufferOutput(section.data, - section.uncompressedLength); - ZlibDecompressor decompressor(&bufferOutput); - status_t error = decompressor.Init(); - if (error != B_OK) - return error; - - while (compressedSize > 0) { - // read compressed buffer - size_t toRead = std::min((size_t)compressedSize, - fScratchBufferSize); - error = ReadBuffer(offset, fScratchBuffer, toRead); - if (error != B_OK) - return error; - - // uncompress - error = decompressor.DecompressNext(fScratchBuffer, toRead); - if (error != B_OK) - return error; - - compressedSize -= toRead; - offset += toRead; - } - - error = decompressor.Finish(); - if (error != B_OK) - return error; - - // verify that all data have been read - if (bufferOutput.BytesWritten() != section.uncompressedLength) { - fErrorOutput->PrintError("Error: Missing bytes in uncompressed " - "buffer!\n"); - return B_BAD_DATA; - } - - return B_OK; - } - - default: - { - fErrorOutput->PrintError("Error: Invalid compression type: %u\n", - section.compression); - return B_BAD_DATA; - } - } + BBufferDataOutput output(section.data, section.uncompressedLength); + return fHeapReader->ReadDataToOutput(section.offset, + section.uncompressedLength, &output); } diff --git a/src/kits/package/hpkg/RepositoryReaderImpl.cpp b/src/kits/package/hpkg/RepositoryReaderImpl.cpp index 5f315a4000..13f78a1471 100644 --- a/src/kits/package/hpkg/RepositoryReaderImpl.cpp +++ b/src/kits/package/hpkg/RepositoryReaderImpl.cpp @@ -20,6 +20,7 @@ #include #include +#include #include @@ -43,8 +44,7 @@ static const size_t kMaxPackageAttributesSize = 64 * 1024 * 1024; RepositoryReaderImpl::RepositoryReaderImpl(BErrorOutput* errorOutput) : - inherited(errorOutput), - fRepositoryInfoSection("repository info") + inherited("repository", errorOutput) { } @@ -73,146 +73,43 @@ RepositoryReaderImpl::Init(const char* fileName) status_t RepositoryReaderImpl::Init(int fd, bool keepFD) { - status_t error = inherited::Init(fd, keepFD); - if (error != B_OK) - return error; - - // stat it - struct stat st; - if (fstat(FD(), &st) < 0) { - ErrorOutput()->PrintError( - "Error: Failed to access repository file: %s\n", strerror(errno)); - return errno; - } - - // read the header hpkg_repo_header header; - if ((error = ReadBuffer(0, &header, sizeof(header))) != B_OK) - return error; - - // check the header - - // magic - if (B_BENDIAN_TO_HOST_INT32(header.magic) != B_HPKG_REPO_MAGIC) { - ErrorOutput()->PrintError("Error: Invalid repository file: Invalid " - "magic\n"); - return B_BAD_DATA; - } - - // header size - size_t headerSize = B_BENDIAN_TO_HOST_INT16(header.header_size); - if (headerSize < sizeof(hpkg_repo_header)) { - ErrorOutput()->PrintError("Error: Invalid repository file: Invalid " - "header size (%llu)\n", headerSize); - return B_BAD_DATA; - } - - // version - if (B_BENDIAN_TO_HOST_INT16(header.version) != B_HPKG_REPO_VERSION) { - ErrorOutput()->PrintError("Error: Invalid/unsupported repository file " - "version (%d)\n", B_BENDIAN_TO_HOST_INT16(header.version)); - return B_BAD_DATA; - } - - // total size - uint64 totalSize = B_BENDIAN_TO_HOST_INT64(header.total_size); - if (totalSize != (uint64)st.st_size) { - ErrorOutput()->PrintError("Error: Invalid repository file: Total size " - "in header (%llu) doesn't agree with total file size (%lld)\n", - totalSize, st.st_size); - return B_BAD_DATA; - } - - // repository info length and compression - fRepositoryInfoSection.compression - = B_BENDIAN_TO_HOST_INT32(header.info_compression); - fRepositoryInfoSection.compressedLength - = B_BENDIAN_TO_HOST_INT32(header.info_length_compressed); - fRepositoryInfoSection.uncompressedLength - = B_BENDIAN_TO_HOST_INT32(header.info_length_uncompressed); - - if (const char* errorString = CheckCompression(fRepositoryInfoSection)) { - ErrorOutput()->PrintError( - "Error: Invalid repository file: info section: %s\n", errorString); - return B_BAD_DATA; - } - - // package attributes length and compression - fPackageAttributesSection.compression - = B_BENDIAN_TO_HOST_INT32(header.packages_compression); - fPackageAttributesSection.compressedLength - = B_BENDIAN_TO_HOST_INT64(header.packages_length_compressed); - fPackageAttributesSection.uncompressedLength - = B_BENDIAN_TO_HOST_INT64(header.packages_length_uncompressed); - fPackageAttributesSection.stringsLength - = B_BENDIAN_TO_HOST_INT64(header.packages_strings_length); - fPackageAttributesSection.stringsCount - = B_BENDIAN_TO_HOST_INT64(header.packages_strings_count); - - if (const char* errorString = CheckCompression( - fPackageAttributesSection)) { - ErrorOutput()->PrintError("Error: Invalid repository file: package " - "attributes section: %s\n", errorString); - return B_BAD_DATA; - } - - // check whether the sections fit together - if (fPackageAttributesSection.compressedLength > totalSize - || fRepositoryInfoSection.compressedLength - > totalSize - fPackageAttributesSection.compressedLength) { - ErrorOutput()->PrintError("Error: Invalid repository file: The sum of " - "the sections sizes is greater than the repository size\n"); - return B_BAD_DATA; - } - - fPackageAttributesSection.offset - = totalSize - fPackageAttributesSection.compressedLength; - fRepositoryInfoSection.offset = fPackageAttributesSection.offset - - fRepositoryInfoSection.compressedLength; - - // repository info size sanity check - if (fRepositoryInfoSection.uncompressedLength > kMaxRepositoryInfoSize) { - ErrorOutput()->PrintError("Error: Repository file info section size " - "is %llu bytes. This is beyond the reader's sanity limit\n", - fRepositoryInfoSection.uncompressedLength); - return B_UNSUPPORTED; - } - - // package attributes size sanity check - if (fPackageAttributesSection.uncompressedLength - > kMaxPackageAttributesSize) { - ErrorOutput()->PrintError( - "Error: Package file package attributes section size " - "is %llu bytes. This is beyond the reader's sanity limit\n", - fPackageAttributesSection.uncompressedLength); - return B_UNSUPPORTED; - } - - // read in the complete repository info section - fRepositoryInfoSection.data - = new(std::nothrow) uint8[fRepositoryInfoSection.uncompressedLength]; - if (fRepositoryInfoSection.data == NULL) { - ErrorOutput()->PrintError("Error: Out of memory!\n"); - return B_NO_MEMORY; - } - error = ReadCompressedBuffer(fRepositoryInfoSection); + status_t error = inherited::Init(fd, keepFD, header); if (error != B_OK) return error; - // read in the complete package attributes section - fPackageAttributesSection.data - = new(std::nothrow) uint8[fPackageAttributesSection.uncompressedLength]; - if (fPackageAttributesSection.data == NULL) { - ErrorOutput()->PrintError("Error: Out of memory!\n"); - return B_NO_MEMORY; - } - error = ReadCompressedBuffer(fPackageAttributesSection); + // init package attributes section + error = InitSection(fPackageAttributesSection, + HeapReader()->UncompressedHeapSize(), + B_BENDIAN_TO_HOST_INT64(header.packages_length), + kMaxPackageAttributesSize, + B_BENDIAN_TO_HOST_INT64(header.packages_strings_length), + B_BENDIAN_TO_HOST_INT64(header.packages_strings_count)); + if (error != B_OK) + return error; + + // init repository info section + PackageFileSection repositoryInfoSection("repository info"); + error = InitSection(repositoryInfoSection, + fPackageAttributesSection.offset, + B_BENDIAN_TO_HOST_INT32(header.info_length), kMaxRepositoryInfoSize, 0, + 0); + if (error != B_OK) + return error; + + // prepare the sections for use + error = PrepareSection(repositoryInfoSection); + if (error != B_OK) + return error; + + error = PrepareSection(fPackageAttributesSection); if (error != B_OK) return error; // unarchive repository info BMessage repositoryInfoArchive; - error = repositoryInfoArchive.Unflatten((char*)fRepositoryInfoSection.data); + error = repositoryInfoArchive.Unflatten((char*)repositoryInfoSection.data); if (error != B_OK) { ErrorOutput()->PrintError( "Error: Unable to unflatten repository info archive!\n"); @@ -225,17 +122,6 @@ RepositoryReaderImpl::Init(int fd, bool keepFD) return error; } - // parse strings from package attributes section - fPackageAttributesSection.currentOffset = 0; - SetCurrentSection(&fPackageAttributesSection); - - // strings - error = ParseStrings(); - if (error != B_OK) - return error; - - SetCurrentSection(NULL); - return B_OK; } diff --git a/src/kits/package/hpkg/RepositoryWriterImpl.cpp b/src/kits/package/hpkg/RepositoryWriterImpl.cpp index c7c87dda7f..c8ceb33a6b 100644 --- a/src/kits/package/hpkg/RepositoryWriterImpl.cpp +++ b/src/kits/package/hpkg/RepositoryWriterImpl.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -42,28 +43,31 @@ using BPackageKit::BPrivate::HashableString; namespace { +// #pragma mark - PackageEntryDataFetcher + + struct PackageEntryDataFetcher { PackageEntryDataFetcher(BErrorOutput* errorOutput, BPackageData& packageData) : fErrorOutput(errorOutput), - fBufferPool(B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, 2), fPackageData(packageData) { } - status_t ReadIntoString(BDataReader* dataReader, BString& _contents) + status_t ReadIntoString(BAbstractBufferedDataReader* heapReader, + BString& _contents) { // create a PackageDataReader BAbstractBufferedDataReader* reader; - status_t result = BPackageDataReaderFactory(&fBufferPool) - .CreatePackageDataReader(dataReader, fPackageData, reader); + status_t result = BPackageDataReaderFactory() + .CreatePackageDataReader(heapReader, fPackageData, reader); if (result != B_OK) return result; ObjectDeleter readerDeleter(reader); // copy data into the given string - int32 bufferSize = fPackageData.UncompressedSize(); + int32 bufferSize = fPackageData.Size(); char* buffer = _contents.LockBuffer(bufferSize); if (buffer == NULL) return B_NO_MEMORY; @@ -81,17 +85,20 @@ struct PackageEntryDataFetcher { private: BErrorOutput* fErrorOutput; - BBlockBufferPoolNoLock fBufferPool; BPackageData& fPackageData; }; +// #pragma mark - PackageContentHandler + + struct PackageContentHandler : public BPackageInfoContentHandler { PackageContentHandler(BErrorOutput* errorOutput, BPackageInfo* packageInfo, - int packageFileFD, BRepositoryInfo* repositoryInfo) + BAbstractBufferedDataReader* heapReader, + BRepositoryInfo* repositoryInfo) : BPackageInfoContentHandler(*packageInfo, errorOutput), - fPackageFileReader(packageFileFD), + fHeapReader(heapReader), fRepositoryInfo(repositoryInfo) { } @@ -132,16 +139,7 @@ struct PackageContentHandler : public BPackageInfoContentHandler { PackageEntryDataFetcher dataFetcher(fErrorOutput, packageData); BString licenseText; - status_t result; - if (packageData.IsEncodedInline()) { - BBufferDataReader dataReader(packageData.InlineData(), - packageData.CompressedSize()); - result = dataFetcher.ReadIntoString(&dataReader, licenseText); - } else { - result - = dataFetcher.ReadIntoString(&fPackageFileReader, licenseText); - } - + status_t result = dataFetcher.ReadIntoString(fHeapReader, licenseText); if (result != B_OK) return result; @@ -165,24 +163,30 @@ struct PackageContentHandler : public BPackageInfoContentHandler { } private: - BPackageReader* fPackageReader; - BFDDataReader fPackageFileReader; - BRepositoryInfo* fRepositoryInfo; + BPackageReader* fPackageReader; + BAbstractBufferedDataReader* fHeapReader; + BRepositoryInfo* fRepositoryInfo; }; } // anonymous namespace +// #pragma mark - PackageNameSet + + struct RepositoryWriterImpl::PackageNameSet : public ::BPrivate::HashSet { }; +// #pragma mark - RepositoryWriterImpl + + RepositoryWriterImpl::RepositoryWriterImpl(BRepositoryWriterListener* listener, BRepositoryInfo* repositoryInfo) : - inherited(listener), + inherited("repository", listener), fListener(listener), fRepositoryInfo(repositoryInfo), fPackageCount(0), @@ -246,7 +250,7 @@ RepositoryWriterImpl::Finish() status_t RepositoryWriterImpl::_Init(const char* fileName) { - return inherited::Init(fileName, "repository", 0); + return inherited::Init(fileName, 0); } @@ -255,29 +259,34 @@ RepositoryWriterImpl::_Finish() { hpkg_repo_header header; - // write repository header - ssize_t infoLengthCompressed; - status_t result = _WriteRepositoryInfo(header, infoLengthCompressed); + // write repository info + uint64 infoLength; + status_t result = _WriteRepositoryInfo(header, infoLength); if (result != B_OK) return result; // write package attributes - ssize_t packagesLengthCompressed; - off_t totalSize = _WritePackageAttributes(header, - sizeof(header) + infoLengthCompressed, packagesLengthCompressed); + uint64 packagesLength; + _WritePackageAttributes(header, packagesLength); - fListener->OnRepositoryDone(sizeof(header), infoLengthCompressed, + // flush the heap writer + result = fHeapWriter->Finish(); + if (result != B_OK) + return result; + uint64 totalSize = fHeapWriter->HeapOffset() + + fHeapWriter->CompressedHeapSize(); + + fListener->OnRepositoryDone(sizeof(header), infoLength, fRepositoryInfo->LicenseNames().CountStrings(), fPackageCount, - packagesLengthCompressed, totalSize); + packagesLength, totalSize); - // general + // update the general header info and write the header header.magic = B_HOST_TO_BENDIAN_INT32(B_HPKG_REPO_MAGIC); header.header_size = B_HOST_TO_BENDIAN_INT16((uint16)sizeof(header)); header.version = B_HOST_TO_BENDIAN_INT16(B_HPKG_REPO_VERSION); header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize); - // write the header - WriteBuffer(&header, sizeof(header), 0); + RawWriteBuffer(&header, sizeof(header), 0); SetFinished(true); return B_OK; @@ -309,7 +318,7 @@ RepositoryWriterImpl::_AddPackage(const BEntry& packageEntry) // parse package PackageContentHandler contentHandler(fListener, &fPackageInfo, - packageReader.PackageFileFD(), fRepositoryInfo); + packageReader.HeapReader(), fRepositoryInfo); if ((result = packageReader.ParseContent(&contentHandler)) != B_OK) return result; @@ -384,8 +393,9 @@ RepositoryWriterImpl::_RegisterCurrentPackageInfo() status_t RepositoryWriterImpl::_WriteRepositoryInfo(hpkg_repo_header& header, - ssize_t& _infoLengthCompressed) + uint64& _length) { + // archive and flatten the repository info and write it BMessage archive; status_t result = fRepositoryInfo->Archive(&archive); if (result != B_OK) { @@ -400,71 +410,40 @@ RepositoryWriterImpl::_WriteRepositoryInfo(hpkg_repo_header& header, return result; } - off_t startOffset = sizeof(hpkg_repo_header); + WriteBuffer(buffer, flattenedSize); - // write the package attributes (zlib writer on top of a file writer) - FDDataWriter realWriter(FD(), startOffset, fListener); - ZlibDataWriter zlibWriter(&realWriter); - SetDataWriter(&zlibWriter); - zlibWriter.Init(); - - DataWriter()->WriteDataThrows(buffer, flattenedSize); - - zlibWriter.Finish(); - SetDataWriter(NULL); - - fListener->OnRepositoryInfoSectionDone(zlibWriter.BytesWritten()); - - _infoLengthCompressed = realWriter.BytesWritten(); + // notify listener + fListener->OnRepositoryInfoSectionDone(flattenedSize); // update the header - header.info_compression - = B_HOST_TO_BENDIAN_INT32(B_HPKG_COMPRESSION_ZLIB); - header.info_length_compressed - = B_HOST_TO_BENDIAN_INT32(_infoLengthCompressed); - header.info_length_uncompressed - = B_HOST_TO_BENDIAN_INT32(flattenedSize); + header.info_length = B_HOST_TO_BENDIAN_INT32(flattenedSize); + _length = flattenedSize; return B_OK; } -off_t +void RepositoryWriterImpl::_WritePackageAttributes(hpkg_repo_header& header, - off_t startOffset, ssize_t& _packagesLengthCompressed) + uint64& _length) { // write the package attributes (zlib writer on top of a file writer) - FDDataWriter realWriter(FD(), startOffset, fListener); - ZlibDataWriter zlibWriter(&realWriter); - SetDataWriter(&zlibWriter); - zlibWriter.Init(); + uint64 startOffset = fHeapWriter->UncompressedHeapSize(); - // write cached strings and package attributes tree - uint32 stringsLengthUncompressed; + uint32 stringsLength; uint32 stringsCount = WritePackageAttributes(PackageAttributes(), - stringsLengthUncompressed); + stringsLength); - zlibWriter.Finish(); - off_t endOffset = realWriter.Offset(); - SetDataWriter(NULL); + uint64 sectionSize = fHeapWriter->UncompressedHeapSize() - startOffset; - fListener->OnPackageAttributesSectionDone(stringsCount, - zlibWriter.BytesWritten()); - - _packagesLengthCompressed = endOffset - startOffset; + fListener->OnPackageAttributesSectionDone(stringsCount, sectionSize); // update the header - header.packages_compression - = B_HOST_TO_BENDIAN_INT32(B_HPKG_COMPRESSION_ZLIB); - header.packages_length_compressed - = B_HOST_TO_BENDIAN_INT64(_packagesLengthCompressed); - header.packages_length_uncompressed - = B_HOST_TO_BENDIAN_INT64(zlibWriter.BytesWritten()); + header.packages_length = B_HOST_TO_BENDIAN_INT64(sectionSize); header.packages_strings_count = B_HOST_TO_BENDIAN_INT64(stringsCount); - header.packages_strings_length - = B_HOST_TO_BENDIAN_INT64(stringsLengthUncompressed); + header.packages_strings_length = B_HOST_TO_BENDIAN_INT64(stringsLength); - return endOffset; + _length = sectionSize; } diff --git a/src/kits/package/hpkg/WriterImplBase.cpp b/src/kits/package/hpkg/WriterImplBase.cpp index 7a539a3795..88d9d9be52 100644 --- a/src/kits/package/hpkg/WriterImplBase.cpp +++ b/src/kits/package/hpkg/WriterImplBase.cpp @@ -18,11 +18,12 @@ #include -#include - #include #include +#include +#include + namespace BPackageKit { @@ -176,106 +177,9 @@ WriterImplBase::AttributeValue::_ApplicableIntEncoding(uint64 value) } -// #pragma mark - AbstractDataWriter - - -WriterImplBase::AbstractDataWriter::AbstractDataWriter() - : - fBytesWritten(0) -{ -} - - -WriterImplBase::AbstractDataWriter::~AbstractDataWriter() -{ -} - - -// #pragma mark - FDDataWriter - - -WriterImplBase::FDDataWriter::FDDataWriter(int fd, off_t offset, - BErrorOutput* errorOutput) - : - fFD(fd), - fOffset(offset), - fErrorOutput(errorOutput) -{ -} - - -status_t -WriterImplBase::FDDataWriter::WriteDataNoThrow(const void* buffer, size_t size) -{ - ssize_t bytesWritten = pwrite(fFD, buffer, size, fOffset); - if (bytesWritten < 0) { - fErrorOutput->PrintError( - "WriteDataNoThrow(%p, %lu) failed to write data: %s\n", buffer, - size, strerror(errno)); - return errno; - } - if ((size_t)bytesWritten != size) { - fErrorOutput->PrintError( - "WriteDataNoThrow(%p, %lu) failed to write all data\n", buffer, - size); - return B_ERROR; - } - - fOffset += size; - fBytesWritten += size; - return B_OK; -} - - -// #pragma mark - ZlibDataWriter - - -WriterImplBase::ZlibDataWriter::ZlibDataWriter(AbstractDataWriter* dataWriter) - : - fDataWriter(dataWriter), - fCompressor(this) -{ -} - - -void -WriterImplBase::ZlibDataWriter::Init() -{ - status_t error = fCompressor.Init(); - if (error != B_OK) - throw status_t(error); -} - - -void -WriterImplBase::ZlibDataWriter::Finish() -{ - status_t error = fCompressor.Finish(); - if (error != B_OK) - throw status_t(error); -} - - -status_t -WriterImplBase::ZlibDataWriter::WriteDataNoThrow(const void* buffer, - size_t size) -{ - status_t error = fCompressor.CompressNext(buffer, size); - if (error == B_OK) - fBytesWritten += size; - return error; -} - - -status_t -WriterImplBase::ZlibDataWriter::WriteData(const void* buffer, size_t size) -{ - return fDataWriter->WriteDataNoThrow(buffer, size); -} - - // #pragma mark - PackageAttribute + WriterImplBase::PackageAttribute::PackageAttribute(BHPKGAttributeID id_, uint8 type_, uint8 encoding_) : @@ -310,8 +214,10 @@ WriterImplBase::PackageAttribute::_DeleteChildren() // #pragma mark - WriterImplBase -WriterImplBase::WriterImplBase(BErrorOutput* errorOutput) +WriterImplBase::WriterImplBase(const char* fileType, BErrorOutput* errorOutput) : + fHeapWriter(NULL), + fFileType(fileType), fErrorOutput(errorOutput), fFileName(NULL), fFlags(0), @@ -324,6 +230,8 @@ WriterImplBase::WriterImplBase(BErrorOutput* errorOutput) WriterImplBase::~WriterImplBase() { + delete fHeapWriter; + if (fFD >= 0) close(fFD); @@ -335,7 +243,7 @@ WriterImplBase::~WriterImplBase() status_t -WriterImplBase::Init(const char* fileName, const char* type, uint32 flags) +WriterImplBase::Init(const char* fileName, uint32 flags) { if (fPackageStringCache.Init() != B_OK) throw std::bad_alloc(); @@ -347,14 +255,20 @@ WriterImplBase::Init(const char* fileName, const char* type, uint32 flags) fFD = open(fileName, openMode, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (fFD < 0) { - fErrorOutput->PrintError("Failed to open %s file \"%s\": %s\n", type, - fileName, strerror(errno)); + fErrorOutput->PrintError("Failed to open %s file \"%s\": %s\n", + fFileType, fileName, strerror(errno)); return errno; } fFileName = fileName; fFlags = flags; + // create heap writer + fHeapWriter = new PackageFileHeapWriter(fErrorOutput, FD(), + sizeof(hpkg_header)); + fHeapWriter->Init(); + fDataWriter = fHeapWriter->DataWriter(); + return B_OK; } @@ -670,8 +584,10 @@ WriterImplBase::WritePackageAttributes( uint32& _stringsLengthUncompressed) { // write the cached strings + uint64 startOffset = fHeapWriter->UncompressedHeapSize(); uint32 stringsCount = WriteCachedStrings(fPackageStringCache, 2); - _stringsLengthUncompressed = DataWriter()->BytesWritten(); + _stringsLengthUncompressed + = fHeapWriter->UncompressedHeapSize() - startOffset; _WritePackageAttributes(packageAttributes); @@ -760,18 +676,18 @@ WriterImplBase::WriteUnsignedLEB128(uint64 value) void -WriterImplBase::WriteBuffer(const void* buffer, size_t size, off_t offset) +WriterImplBase::RawWriteBuffer(const void* buffer, size_t size, off_t offset) { ssize_t bytesWritten = pwrite(fFD, buffer, size, offset); if (bytesWritten < 0) { fErrorOutput->PrintError( - "WriteBuffer(%p, %lu) failed to write data: %s\n", buffer, size, + "RawWriteBuffer(%p, %lu) failed to write data: %s\n", buffer, size, strerror(errno)); throw status_t(errno); } if ((size_t)bytesWritten != size) { fErrorOutput->PrintError( - "WriteBuffer(%p, %lu) failed to write all data\n", buffer, size); + "RawWriteBuffer(%p, %lu) failed to write all data\n", buffer, size); throw status_t(B_ERROR); } } diff --git a/src/kits/package/hpkg/v1/PackageDataReaderV1.cpp b/src/kits/package/hpkg/v1/PackageDataReaderV1.cpp index a77e0b0877..9fc006319f 100644 --- a/src/kits/package/hpkg/v1/PackageDataReaderV1.cpp +++ b/src/kits/package/hpkg/v1/PackageDataReaderV1.cpp @@ -397,6 +397,23 @@ private: }; +// #pragma mark - PackageDataHeapReader + + +class PackageDataInlineReader : public BBufferDataReader { +public: + PackageDataInlineReader(const BPackageData& data) + : + BBufferDataReader(fData.InlineData(), data.UncompressedSize()), + fData(data) + { + } + +private: + BPackageData fData; +}; + + // #pragma mark - BPackageDataReaderFactory @@ -411,6 +428,16 @@ status_t BPackageDataReaderFactory::CreatePackageDataReader(BDataReader* dataReader, const BPackageData& data, BAbstractBufferedDataReader*& _reader) { + if (data.IsEncodedInline()) { + BAbstractBufferedDataReader* reader + = new(std::nothrow) PackageDataInlineReader(data); + if (reader == NULL) + return B_NO_MEMORY; + + _reader = reader; + return B_OK; + } + PackageDataReader* reader; switch (data.Compression()) { diff --git a/src/system/boot/loader/file_systems/packagefs/Jamfile b/src/system/boot/loader/file_systems/packagefs/Jamfile index bd42445609..2798472348 100644 --- a/src/system/boot/loader/file_systems/packagefs/Jamfile +++ b/src/system/boot/loader/file_systems/packagefs/Jamfile @@ -35,6 +35,8 @@ BootStaticLibrary boot_packagefs : PackageDataReader.cpp PackageEntry.cpp PackageEntryAttribute.cpp + PackageFileHeapAccessorBase.cpp + PackageFileHeapReader.cpp PackageReaderImpl.cpp ReaderImplBase.cpp diff --git a/src/system/boot/loader/file_systems/packagefs/packagefs.cpp b/src/system/boot/loader/file_systems/packagefs/packagefs.cpp index 66a608eaa2..390fe2b9ad 100644 --- a/src/system/boot/loader/file_systems/packagefs/packagefs.cpp +++ b/src/system/boot/loader/file_systems/packagefs/packagefs.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2011, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2011-2013, Ingo Weinhold, ingo_weinhold@gmx.de. * Distributed under the terms of the MIT License. */ @@ -9,12 +9,12 @@ #include #include -#include #include #include #include #include #include +#include #include #include @@ -41,6 +41,7 @@ using namespace BPackageKit; using namespace BPackageKit::BHPKG; +using BPackageKit::BHPKG::BPrivate::PackageFileHeapReader; using BPackageKit::BHPKG::BPrivate::PackageReaderImpl; @@ -143,7 +144,7 @@ struct PackageFile : PackageNode { off_t Size() const { - return fData.UncompressedSize(); + return fData.Size(); } private: @@ -232,85 +233,60 @@ private: }; -// #pragma mark - FileDataReader +// #pragma mark - PackageLoaderErrorOutput -struct FileDataReader { - FileDataReader(int fd, const BPackageData& data) - : - fDataReader(fd), - fData(data), - fPackageDataReader(NULL) +struct PackageLoaderErrorOutput : BErrorOutput { + PackageLoaderErrorOutput() { } - ~FileDataReader() + virtual void PrintErrorVarArgs(const char* format, va_list args) { - delete fPackageDataReader; } - - status_t Init(BPackageDataReaderFactory* dataReaderFactory) - { - if (fData.IsEncodedInline()) - return B_OK; - - return dataReaderFactory->CreatePackageDataReader(&fDataReader, - fData, fPackageDataReader); - } - - status_t ReadData(off_t offset, void* buffer, size_t bufferSize) - { - if (fData.IsEncodedInline()) { - BBufferDataReader dataReader(fData.InlineData(), - fData.CompressedSize()); - return dataReader.ReadData(offset, buffer, bufferSize); - } - - return fPackageDataReader->ReadData(offset, buffer, bufferSize); - } - -private: - BFDDataReader fDataReader; - BPackageData fData; - BAbstractBufferedDataReader* fPackageDataReader; }; // #pragma mark - PackageVolume -struct PackageVolume : BReferenceable { +struct PackageVolume : BReferenceable, private PackageLoaderErrorOutput { PackageVolume() : fNextNodeID(1), fRootDirectory(this, S_IFDIR), - fBufferCache(B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, 2), - fDataReaderFactory(&fBufferCache), + fHeapReader(NULL), fFD(-1) { } ~PackageVolume() { + delete fHeapReader; + if (fFD >= 0) close(fFD); } - status_t Init(int fd) + status_t Init(int fd, const PackageFileHeapReader* heapReader) { status_t error = fRootDirectory.Init(&fRootDirectory, ".", NextNodeID()); if (error != B_OK) return error; - error = fBufferCache.Init(); - if (error != B_OK) - return error; - fFD = dup(fd); if (fFD < 0) return errno; + // clone a heap reader and adjust it for our use + fHeapReader = heapReader->Clone(); + if (fHeapReader == NULL) + return B_NO_MEMORY; + + fHeapReader->SetErrorOutput(this); + fHeapReader->SetFD(fFD); + return B_OK; } @@ -325,46 +301,20 @@ struct PackageVolume : BReferenceable { } status_t CreateFileDataReader(const BPackageData& data, - FileDataReader*& _fileDataReader) + BAbstractBufferedDataReader*& _reader) { - // create the file reader object - FileDataReader* fileDataReader = new(std::nothrow) FileDataReader(fFD, - data); - if (fileDataReader == NULL) - return B_NO_MEMORY; - ObjectDeleter fileDataReaderDeleter(fileDataReader); - - status_t error = fileDataReader->Init(&fDataReaderFactory); - if (error != B_OK) - return error; - - _fileDataReader = fileDataReaderDeleter.Detach(); - return B_OK; + return BPackageDataReaderFactory().CreatePackageDataReader(fHeapReader, + data, _reader); } private: ino_t fNextNodeID; PackageDirectory fRootDirectory; - BBlockBufferPoolNoLock fBufferCache; - BPackageDataReaderFactory fDataReaderFactory; + PackageFileHeapReader* fHeapReader; int fFD; }; -// #pragma mark - PackageLoaderErrorOutput - - -struct PackageLoaderErrorOutput : BErrorOutput { - PackageLoaderErrorOutput() - { - } - - virtual void PrintErrorVarArgs(const char* format, va_list args) - { - } -}; - - // #pragma mark - PackageLoaderContentHandler @@ -503,7 +453,8 @@ struct File : ::Node { bufferSize = size - pos; if (bufferSize > 0) { - FileDataReader* dataReader = (FileDataReader*)cookie; + BAbstractBufferedDataReader* dataReader + = (BAbstractBufferedDataReader*)cookie; status_t error = dataReader->ReadData(pos, buffer, bufferSize); if (error != B_OK) return error; @@ -529,7 +480,7 @@ struct File : ::Node { if ((mode & O_ACCMODE) != O_RDONLY && (mode & O_ACCMODE) != O_RDWR) return B_NOT_ALLOWED; - FileDataReader* dataReader; + BAbstractBufferedDataReader* dataReader; status_t error = fFile->Volume()->CreateFileDataReader(fFile->Data(), dataReader); if (error != B_OK) @@ -542,7 +493,8 @@ struct File : ::Node { virtual status_t Close(void* cookie) { - FileDataReader* dataReader = (FileDataReader*)cookie; + BAbstractBufferedDataReader* dataReader + = (BAbstractBufferedDataReader*)cookie; delete dataReader; Release(); return B_OK; @@ -807,7 +759,7 @@ packagefs_mount_file(int fd, ::Directory*& _mountedDirectory) { PackageLoaderErrorOutput errorOutput; PackageReaderImpl packageReader(&errorOutput); - status_t error = packageReader.Init(fd, false); + status_t error = packageReader.Init(fd, false); if (error != B_OK) RETURN_ERROR(error); @@ -817,7 +769,7 @@ packagefs_mount_file(int fd, ::Directory*& _mountedDirectory) return B_NO_MEMORY; BReference volumeReference(volume, true); - error = volume->Init(fd); + error = volume->Init(fd, packageReader.HeapReader()); if (error != B_OK) RETURN_ERROR(error);