NetServices: move HttpBuffer and HttpParser into their own header/source

Change-Id: I5bc0d9df6f94c2cf1c39baa6206bf6f1db284705

headers/private/netservices2/HttpStream.h

@@ -23,6 +23,7 @@ namespace Network {
 class BHttpFields;
 class BHttpRequest;
 class BHttpStatus;
+class HttpBuffer;
 
 
 class BAbstractDataStream {
@@ -62,77 +63,6 @@ private:
 using HttpTransferFunction = std::function<size_t (const std::byte*, size_t)>;
 
 
-class HttpBuffer {
-public:
-							HttpBuffer(size_t capacity = 8*1024);
-
-	ssize_t					ReadFrom(BDataIO* source);
-	void					WriteExactlyTo(BDataIO* target);
-	void					WriteTo(HttpTransferFunction func,
-								std::optional<size_t> maxSize = std::nullopt);
-	std::optional<BString>	GetNextLine();
-
-	size_t					RemainingBytes() noexcept;
-
-	void					Flush() noexcept;
-	void					Clear() noexcept;
-
-private:
-	std::vector<std::byte>	fBuffer;
-	size_t					fCurrentOffset = 0;
-};
-
-
-enum class HttpBodyInputStreamState {
-	ChunkSize,
-	ChunkEnd,
-	Chunk,
-	Trailers,
-	Done
-};
-
-
-class HttpParser {
-public:
-							HttpParser() {};
-
-	// HTTP Header
-	bool					ParseStatus(HttpBuffer& buffer, BHttpStatus& status);
-	bool					ParseFields(HttpBuffer& buffer, BHttpFields& fields);
-
-	// HTTP Body
-	void					SetGzipCompression(bool compression = true);
-	void					SetContentLength(std::optional<off_t> contentLength) noexcept;
-
-	size_t					ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody);
-	std::optional<off_t>	BodyBytesTotal() const noexcept { return fBodyBytesTotal; };
-	off_t					BodyBytesTransferred() const noexcept { return fTransferredBodySize; };
-	bool					Complete() const noexcept;
-
-private:
-	size_t					_ParseBodyRaw(HttpBuffer& buffer, HttpTransferFunction writeToBody);
-	size_t					_ParseBodyChunked(HttpBuffer& buffer, HttpTransferFunction writeToBody);
-	size_t					_ReadChunk(HttpBuffer& buffer, HttpTransferFunction writeToBody,
-								size_t maxSize, bool flush);
-	bool					_IsChunked() const noexcept;
-
-private:
-	off_t					fHeaderBytes = 0;
-
-	// Support for chunked transfers
-	HttpBodyInputStreamState fBodyState = HttpBodyInputStreamState::ChunkSize;
-	off_t					fRemainingChunkSize = 0;
-	bool					fLastChunk = false;
-
-	// Receive stats
-	std::optional<off_t>	fBodyBytesTotal = 0;
-	off_t					fTransferredBodySize = 0;
-
-	// Optional decompression
-	std::unique_ptr<BMallocIO>		fDecompressorStorage = nullptr;
-	std::unique_ptr<BDataIO>		fDecompressingStream = nullptr;
-
-};
 
 
 } // namespace Network

src/kits/network/libnetservices2/HttpBuffer.cpp

@@ -0,0 +1,179 @@
+/*
+ * Copyright 2022 Haiku Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Niels Sascha Reedijk, [email protected]
+ */
+
+#include "HttpBuffer.h"
+
+#include <DataIO.h>
+#include <NetServicesDefs.h>
+#include <String.h>
+
+using namespace BPrivate::Network;
+
+
+/*!
+	\brief Newline sequence
+
+	As per the RFC, defined as \r\n
+*/
+static constexpr std::array<std::byte, 2> kNewLine = {std::byte('\r'), std::byte('\n')};
+
+
+/*!
+	\brief Create a new HTTP buffer with \a capacity.
+*/
+HttpBuffer::HttpBuffer(size_t capacity)
+{
+	fBuffer.reserve(capacity);
+};
+
+
+/*!
+	\brief Load data from \a source into the spare capacity of this buffer.
+
+	\exception BNetworkRequestError When BDataIO::Read() returns any error other than B_WOULD_BLOCK
+
+	\retval B_WOULD_BLOCK The read call on the \a source was unsuccessful because it would block.
+	\retval >=0 The actual number of bytes read.
+*/
+ssize_t
+HttpBuffer::ReadFrom(BDataIO* source)
+{
+	// Remove any unused bytes at the beginning of the buffer
+	Flush();
+
+	auto currentSize = fBuffer.size();
+	auto remainingBufferSize = fBuffer.capacity() - currentSize;
+
+	// Adjust the buffer to the maximum size
+	fBuffer.resize(fBuffer.capacity());
+
+	ssize_t bytesRead = B_INTERRUPTED;
+	while (bytesRead == B_INTERRUPTED)
+		bytesRead = source->Read(fBuffer.data() + currentSize, remainingBufferSize);
+
+	if (bytesRead == B_WOULD_BLOCK || bytesRead == 0) {
+		fBuffer.resize(currentSize);
+		return bytesRead;
+	} else if (bytesRead < 0) {
+		throw BNetworkRequestError("BDataIO::Read()", BNetworkRequestError::NetworkError,
+			bytesRead);
+	}
+
+	// Adjust the buffer to the current size
+	fBuffer.resize(currentSize + bytesRead);
+
+	return bytesRead;
+}
+
+
+/*!
+	\brief Use BDataIO::WriteExactly() on target to write the contents of the buffer.
+*/
+void
+HttpBuffer::WriteExactlyTo(BDataIO* target)
+{
+	if (RemainingBytes() == 0)
+		return;
+
+	auto status = target->WriteExactly(fBuffer.data() + fCurrentOffset, RemainingBytes());
+	if (status != B_OK) {
+		throw BNetworkRequestError("BDataIO::WriteExactly()", BNetworkRequestError::SystemError,
+			status);
+	}
+
+	// Entire buffer is written; reset internal buffer
+	Clear();
+}
+
+
+/*!
+	\brief Write the contents of the buffer through the helper \a func.
+
+	\param func Handle the actual writing. The function accepts a pointer and a size as inputs
+		and should return the number of actual written bytes, which may be fewer than the number
+		of available bytes.
+*/
+void
+HttpBuffer::WriteTo(HttpTransferFunction func , std::optional<size_t> maxSize)
+{
+	if (RemainingBytes() == 0)
+		return;
+
+	auto size = RemainingBytes();
+	if (maxSize.has_value() && *maxSize < size)
+		size = *maxSize;
+
+	auto bytesWritten = func(fBuffer.data() + fCurrentOffset, size);
+	if (bytesWritten > size)
+		throw BRuntimeError(__PRETTY_FUNCTION__, "More bytes written than were made available");
+
+	fCurrentOffset += bytesWritten;
+}
+
+
+/*!
+	\brief Get the next line from this buffer.
+
+	This can be called iteratively until all lines in the current data are read. After using this
+	method, you should use Flush() to make sure that the read lines are cleared from the beginning
+	of the buffer.
+
+	\retval std::nullopt There are no more lines in the buffer.
+	\retval BString The next line.
+*/
+std::optional<BString>
+HttpBuffer::GetNextLine()
+{
+	auto offset = fBuffer.cbegin() + fCurrentOffset;
+	auto result = std::search(offset, fBuffer.cend(), kNewLine.cbegin(), kNewLine.cend());
+	if (result == fBuffer.cend())
+		return std::nullopt;
+
+	BString line(reinterpret_cast<const char*>(std::addressof(*offset)), std::distance(offset, result));
+	fCurrentOffset = std::distance(fBuffer.cbegin(), result) + 2;
+	return line;
+}
+
+
+/*!
+	\brief Get the number of remaining bytes in this buffer.
+*/
+size_t
+HttpBuffer::RemainingBytes() noexcept
+{
+	return fBuffer.size() - fCurrentOffset;
+}
+
+
+/*!
+	\brief Move data to the beginning of the buffer to clear at the back.
+
+	The GetNextLine() increases the offset of the internal buffer. This call moves remaining data
+	to the beginning of the buffer sets the correct size, making the remainder of the capacity
+	available for further reading.
+*/
+void
+HttpBuffer::Flush() noexcept
+{
+	if (fCurrentOffset > 0) {
+		auto end = fBuffer.cbegin() + fCurrentOffset;
+		fBuffer.erase(fBuffer.cbegin(), end);
+		fCurrentOffset = 0;
+	}
+}
+
+
+/*!
+	\brief Clear the internal buffer
+*/
+void
+HttpBuffer::Clear() noexcept
+{
+	fBuffer.clear();
+	fCurrentOffset = 0;
+}

src/kits/network/libnetservices2/HttpBuffer.h

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2022 Haiku Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ */
+
+#ifndef _B_HTTP_BUFFER_H_
+#define _B_HTTP_BUFFER_H_
+
+#include <functional>
+#include <optional>
+#include <vector>
+
+class BDataIO;
+class BString;
+
+
+namespace BPrivate {
+
+namespace Network {
+
+using HttpTransferFunction = std::function<size_t (const std::byte*, size_t)>;
+
+
+class HttpBuffer {
+public:
+							HttpBuffer(size_t capacity = 8*1024);
+
+	ssize_t					ReadFrom(BDataIO* source);
+	void					WriteExactlyTo(BDataIO* target);
+	void					WriteTo(HttpTransferFunction func,
+								std::optional<size_t> maxSize = std::nullopt);
+	std::optional<BString>	GetNextLine();
+
+	size_t					RemainingBytes() noexcept;
+
+	void					Flush() noexcept;
+	void					Clear() noexcept;
+
+private:
+	std::vector<std::byte>	fBuffer;
+	size_t					fCurrentOffset = 0;
+};
+
+
+} // namespace Network
+
+} // namespace BPrivate
+
+#endif // _B_HTTP_BUFFER_H_

src/kits/network/libnetservices2/HttpParser.cpp

@@ -0,0 +1,345 @@
+/*
+ * Copyright 2022 Haiku Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Niels Sascha Reedijk, [email protected]
+ */
+
+#include "HttpParser.h"
+
+#include <string>
+#include <stdexcept>
+
+#include <HttpFields.h>
+#include <NetServicesDefs.h>
+#include <ZlibCompressionAlgorithm.h>
+
+using namespace BPrivate::Network;
+
+
+/*!
+	\brief Parse the status from the \a buffer and store it in \a status.
+
+	\retval true The status was succesfully parsed
+	\retval false There is not enough data in the buffer for a full status.
+
+	\exception BNetworkRequestException The status does not conform to the HTTP spec.
+*/
+bool
+HttpParser::ParseStatus(HttpBuffer& buffer, BHttpStatus& status)
+{
+	auto statusLine = buffer.GetNextLine();
+	if (!statusLine)
+		return false;
+
+	auto codeStart = statusLine->FindFirst(' ') + 1;
+	if (codeStart < 0)
+		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
+
+	auto codeEnd = statusLine->FindFirst(' ', codeStart);
+
+	if (codeEnd < 0 || (codeEnd - codeStart) != 3)
+		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
+
+	std::string statusCodeString(statusLine->String() + codeStart, 3);
+
+	// build the output
+	try {
+		status.code = std::stol(statusCodeString);
+	} catch (...) {
+		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
+	}
+
+	status.text = std::move(statusLine.value());
+	return true;
+}
+
+
+/*!
+	\brief Parse the fields from the \a buffer and store it in \a fields.
+
+	The fields are parsed incrementally, meaning that even if the full header is not yet in the
+	\a buffer, it will still parse all complete fields and store them in the \a fields.
+
+	\retval true All fields were succesfully parsed
+	\retval false There is not enough data in the buffer to complete parsing of fields.
+
+	\exception BNetworkRequestException The fields not conform to the HTTP spec.
+*/
+bool
+HttpParser::ParseFields(HttpBuffer& buffer, BHttpFields& fields)
+{
+	auto fieldLine = buffer.GetNextLine();
+	
+	while (fieldLine && !fieldLine.value().IsEmpty()){
+		// Parse next header line
+		fields.AddField(fieldLine.value());
+		fieldLine = buffer.GetNextLine();
+	}
+
+	if (fieldLine && fieldLine.value().IsEmpty()){
+		// end of the header section of the message
+		return true;
+	} else {
+		// there is more to parse
+		return false;
+	}
+}
+
+
+/*!
+	\brief Parse the Gzip compression from the body.
+
+	\exception std::bad_alloc in case there is an error allocating memory.
+*/
+void
+HttpParser::SetGzipCompression(bool compression)
+{
+	if (compression) {
+		fDecompressorStorage = std::make_unique<BMallocIO>();
+
+		BDataIO* stream = nullptr;
+		auto result = BZlibCompressionAlgorithm()
+			.CreateDecompressingOutputStream(fDecompressorStorage.get(), nullptr, stream);
+		
+		if (result != B_OK) {
+			throw BNetworkRequestError(
+				"BZlibCompressionAlgorithm().CreateCompressingOutputStream",
+				BNetworkRequestError::SystemError, result);
+		}
+
+		fDecompressingStream = std::unique_ptr<BDataIO>(stream);
+	} else {
+		fDecompressingStream = nullptr;
+		fDecompressorStorage = nullptr;
+	}
+}
+
+
+/*!
+	\brief Set the content length of the body.
+
+	If a content length is set, the body will not be handled as a chunked transfer.
+*/
+void
+HttpParser::SetContentLength(std::optional<off_t> contentLength) noexcept
+{
+	fBodyBytesTotal = contentLength;
+}
+
+
+/*!
+	\brief Parse the body from the \a buffer and use \a writeToBody function to save.
+*/
+size_t
+HttpParser::ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody)
+{
+	if (fBodyBytesTotal.has_value()) {
+		return _ParseBodyRaw(buffer, writeToBody);
+	} else {
+		return _ParseBodyChunked(buffer, writeToBody);
+	}
+}
+
+
+/*!
+	\brief Parse the body from the \a buffer and use \a writeToBody function to save.
+*/
+size_t
+HttpParser::_ParseBodyRaw(HttpBuffer& buffer, HttpTransferFunction writeToBody)
+{
+	if (fBodyBytesTotal && (fTransferredBodySize + static_cast<off_t>(buffer.RemainingBytes()))
+		> *fBodyBytesTotal)
+		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
+
+	auto bytesToRead = buffer.RemainingBytes();
+	auto readEnd = fBodyBytesTotal.value()
+		== (fTransferredBodySize + static_cast<off_t>(bytesToRead));
+
+	auto bytesRead = _ReadChunk(buffer, writeToBody, bytesToRead, readEnd);
+	fTransferredBodySize += bytesRead;
+	return bytesRead;
+}
+
+
+/*!
+	\brief Parse the body from the \a buffer and use \a writeToBody function to save.
+*/
+size_t
+HttpParser::_ParseBodyChunked(HttpBuffer& buffer, HttpTransferFunction writeToBody)
+{
+	size_t totalBytesRead = 0;
+	while (buffer.RemainingBytes() > 0) {
+		switch (fBodyState) {
+		case HttpBodyInputStreamState::ChunkSize:
+		{
+			// Read the next chunk size from the buffer; if unsuccesful wait for more data
+			auto chunkSizeString = buffer.GetNextLine();
+			if (!chunkSizeString)
+				return totalBytesRead;
+			auto chunkSizeStr = std::string(chunkSizeString.value().String());
+			try {
+				size_t pos = 0;
+				fRemainingChunkSize = std::stoll(chunkSizeStr, &pos, 16);
+				if (pos < chunkSizeStr.size() && chunkSizeStr[pos] != ';'){
+					throw BNetworkRequestError(__PRETTY_FUNCTION__,
+						BNetworkRequestError::ProtocolError);
+				}
+			} catch (const std::invalid_argument&) {
+				throw BNetworkRequestError(__PRETTY_FUNCTION__,
+					BNetworkRequestError::ProtocolError);
+			} catch (const std::out_of_range&) {
+				throw BNetworkRequestError(__PRETTY_FUNCTION__,
+					BNetworkRequestError::ProtocolError);
+			}
+
+			if (fRemainingChunkSize > 0)
+				fBodyState = HttpBodyInputStreamState::Chunk;
+			else
+				fBodyState = HttpBodyInputStreamState::Trailers;
+			break;
+		}
+
+		case HttpBodyInputStreamState::Chunk:
+		{
+			size_t bytesToRead;
+			bool readEnd = false;
+			if (fRemainingChunkSize > static_cast<off_t>(buffer.RemainingBytes()))
+				bytesToRead = buffer.RemainingBytes();
+			else {
+				readEnd = true;
+				bytesToRead = fRemainingChunkSize;
+			}
+
+			auto bytesRead = _ReadChunk(buffer, writeToBody, bytesToRead, readEnd);
+			fTransferredBodySize += bytesRead;
+			totalBytesRead += bytesRead;
+			fRemainingChunkSize -= bytesRead;
+			if (fRemainingChunkSize == 0)
+				fBodyState = HttpBodyInputStreamState::ChunkEnd;
+			break;
+		}
+
+		case HttpBodyInputStreamState::ChunkEnd:
+		{
+			if (buffer.RemainingBytes() < 2) {
+				// not enough data in the buffer to finish the chunk
+				return totalBytesRead;
+			}
+			auto chunkEndString = buffer.GetNextLine();
+			if (!chunkEndString || chunkEndString.value().Length() != 0) {
+				// There should have been an empty chunk
+				throw BNetworkRequestError(__PRETTY_FUNCTION__,
+					BNetworkRequestError::ProtocolError);
+			}
+
+			fBodyState = HttpBodyInputStreamState::ChunkSize;
+			break;
+		}
+
+		case HttpBodyInputStreamState::Trailers:
+		{
+			auto trailerString = buffer.GetNextLine();
+			if (!trailerString)  {
+				// More data to come
+				return totalBytesRead;
+			}
+
+			if (trailerString.value().Length() > 0) {
+				// Ignore empty trailers for now
+				// TODO: review if the API should support trailing headers
+			} else {
+				fBodyState = HttpBodyInputStreamState::Done;
+				return totalBytesRead;
+			}
+			break;
+		}
+
+		case HttpBodyInputStreamState::Done:
+			return totalBytesRead;
+		}
+	}
+	return totalBytesRead;
+}
+
+
+/*!
+	\brief Check if the body is fully parsed.
+*/
+bool
+HttpParser::Complete() const noexcept
+{
+	if (_IsChunked())
+		return fBodyState == HttpBodyInputStreamState::Done;
+
+	return fBodyBytesTotal.value() == fTransferredBodySize;
+}
+
+
+/*!
+	\brief Read a chunk of data from the buffer and write it to the output.
+
+	If there is a compression algorithm applied, then it passes through the compression first.
+
+	\param buffer The buffer to read from
+	\param writeToBody The function that can write data from the buffer to the body.
+	\param size The maximum size to read from the buffer. When larger than the buffer size, the
+		remaining bytes from the buffer are read.
+	\param flush Setting this parameter will force the decompression to write out all data, if
+		applicable. Set when all the data has been received.
+
+	\exception BNetworkRequestError When there was any error with any of the system cals.
+*/
+size_t
+HttpParser::_ReadChunk(HttpBuffer& buffer, HttpTransferFunction writeToBody, size_t size, bool flush)
+{
+	if (size == 0)
+		return 0;
+
+	if (size > buffer.RemainingBytes())
+		size = buffer.RemainingBytes();
+
+	if (fDecompressingStream) {
+		buffer.WriteTo([this](const std::byte* buffer, size_t bufferSize){
+			auto status = fDecompressingStream->WriteExactly(buffer, bufferSize);
+			if (status != B_OK) {
+				throw BNetworkRequestError("BDataIO::WriteExactly()",
+					BNetworkRequestError::SystemError, status);
+			}
+			return bufferSize;
+		}, size);
+
+		if (flush) {
+			// No more bytes expected so flush out the final bytes
+			if (auto status = fDecompressingStream->Flush(); status != B_OK)
+				throw BNetworkRequestError("BZlibDecompressionStream::Flush()",
+					BNetworkRequestError::SystemError, status);
+		}
+
+		if (auto bodySize = fDecompressorStorage->Position(); bodySize > 0) {
+			auto bytesWritten
+				= writeToBody(static_cast<const std::byte*>(fDecompressorStorage->Buffer()),
+					bodySize);
+			if (static_cast<off_t>(bytesWritten) != bodySize) {
+				throw BNetworkRequestError(__PRETTY_FUNCTION__,
+					BNetworkRequestError::SystemError, B_PARTIAL_WRITE);
+			}
+			fDecompressorStorage->Seek(0, SEEK_SET);
+		} 
+	} else {
+		// Write the body directly to the target
+		buffer.WriteTo(writeToBody, size);
+	}
+	return size;
+}
+
+
+/*!
+	\brief Internal helper to determine if the body is sent as a chunked transfer.
+*/
+bool
+HttpParser::_IsChunked() const noexcept
+{
+	return fBodyBytesTotal == std::nullopt;
+}

src/kits/network/libnetservices2/HttpParser.h

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2022 Haiku Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ */
+
+#ifndef _B_HTTP_PARSER_H_
+#define _B_HTTP_PARSER_H_
+
+
+#include <functional>
+#include <optional>
+
+#include <HttpResult.h>
+
+#include "HttpBuffer.h"
+
+class BMallocIO;
+
+namespace BPrivate {
+
+namespace Network {
+
+using HttpTransferFunction = std::function<size_t (const std::byte*, size_t)>;
+
+
+enum class HttpBodyInputStreamState {
+	ChunkSize,
+	ChunkEnd,
+	Chunk,
+	Trailers,
+	Done
+};
+
+
+class HttpParser {
+public:
+							HttpParser() {};
+
+	// HTTP Header
+	bool					ParseStatus(HttpBuffer& buffer, BHttpStatus& status);
+	bool					ParseFields(HttpBuffer& buffer, BHttpFields& fields);
+
+	// HTTP Body
+	void					SetGzipCompression(bool compression = true);
+	void					SetContentLength(std::optional<off_t> contentLength) noexcept;
+
+	size_t					ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody);
+	std::optional<off_t>	BodyBytesTotal() const noexcept { return fBodyBytesTotal; };
+	off_t					BodyBytesTransferred() const noexcept { return fTransferredBodySize; };
+	bool					Complete() const noexcept;
+
+private:
+	size_t					_ParseBodyRaw(HttpBuffer& buffer, HttpTransferFunction writeToBody);
+	size_t					_ParseBodyChunked(HttpBuffer& buffer, HttpTransferFunction writeToBody);
+	size_t					_ReadChunk(HttpBuffer& buffer, HttpTransferFunction writeToBody,
+								size_t maxSize, bool flush);
+	bool					_IsChunked() const noexcept;
+
+private:
+	off_t					fHeaderBytes = 0;
+
+	// Support for chunked transfers
+	HttpBodyInputStreamState fBodyState = HttpBodyInputStreamState::ChunkSize;
+	off_t					fRemainingChunkSize = 0;
+	bool					fLastChunk = false;
+
+	// Receive stats
+	std::optional<off_t>	fBodyBytesTotal = 0;
+	off_t					fTransferredBodySize = 0;
+
+	// Optional decompression
+	std::unique_ptr<BMallocIO>		fDecompressorStorage = nullptr;
+	std::unique_ptr<BDataIO>		fDecompressingStream = nullptr;
+
+};
+
+
+} // namespace Network
+
+} // namespace BPrivate
+
+#endif // _B_HTTP_PARSER_H_

src/kits/network/libnetservices2/HttpSession.cpp

@@ -32,6 +32,8 @@
 #include <Socket.h>
 #include <ZlibCompressionAlgorithm.h>
 
+#include "HttpBuffer.h"
+#include "HttpParser.h"
 #include "HttpResultPrivate.h"
 #include "NetServicesPrivate.h"
 

src/kits/network/libnetservices2/HttpStream.cpp

@@ -8,9 +8,7 @@
 
 #include <HttpStream.h>
 
-#include <algorithm>
 #include <optional>
-#include <stdexcept>
 #include <string>
 
 #include <DataIO.h>
@@ -18,7 +16,8 @@
 #include <HttpRequest.h>
 #include <HttpResult.h>
 #include <NetServicesDefs.h>
-#include <ZlibCompressionAlgorithm.h>
+
+#include "HttpBuffer.h"
 
 using namespace BPrivate::Network;
 
@@ -36,14 +35,6 @@ using namespace BPrivate::Network;
 static constexpr ssize_t kMaxBufferSize = 8192;
 
 
-/*!
-	\brief Newline sequence
-
-	As per the RFC, defined as \r\n
-*/
-static constexpr std::array<std::byte, 2> kNewLine = {std::byte('\r'), std::byte('\n')};
-
-
 // #pragma mark -- ByteIOHelper base class
 
 
@@ -218,486 +209,3 @@ BHttpRequestStream::Transfer(BDataIO* target)
 	auto complete = fRemainingHeaderSize == 0 && fTransferredBodySize == fTotalBodySize;
 	return TransferInfo{bytesWritten, fTransferredBodySize, fTotalBodySize, complete};
 }
-
-
-/*!
-	\brief Create a new HTTP buffer with \a capacity.
-*/
-HttpBuffer::HttpBuffer(size_t capacity)
-{
-	fBuffer.reserve(capacity);
-};
-
-
-/*!
-	\brief Load data from \a source into the spare capacity of this buffer.
-
-	\exception BNetworkRequestError When BDataIO::Read() returns any error other than B_WOULD_BLOCK
-
-	\retval B_WOULD_BLOCK The read call on the \a source was unsuccessful because it would block.
-	\retval >=0 The actual number of bytes read.
-*/
-ssize_t
-HttpBuffer::ReadFrom(BDataIO* source)
-{
-	// Remove any unused bytes at the beginning of the buffer
-	Flush();
-
-	auto currentSize = fBuffer.size();
-	auto remainingBufferSize = fBuffer.capacity() - currentSize;
-
-	// Adjust the buffer to the maximum size
-	fBuffer.resize(fBuffer.capacity());
-
-	ssize_t bytesRead = B_INTERRUPTED;
-	while (bytesRead == B_INTERRUPTED)
-		bytesRead = source->Read(fBuffer.data() + currentSize, remainingBufferSize);
-
-	if (bytesRead == B_WOULD_BLOCK || bytesRead == 0) {
-		fBuffer.resize(currentSize);
-		return bytesRead;
-	} else if (bytesRead < 0) {
-		throw BNetworkRequestError("BDataIO::Read()", BNetworkRequestError::NetworkError,
-			bytesRead);
-	}
-
-	// Adjust the buffer to the current size
-	fBuffer.resize(currentSize + bytesRead);
-
-	return bytesRead;
-}
-
-
-/*!
-	\brief Use BDataIO::WriteExactly() on target to write the contents of the buffer.
-*/
-void
-HttpBuffer::WriteExactlyTo(BDataIO* target)
-{
-	if (RemainingBytes() == 0)
-		return;
-
-	auto status = target->WriteExactly(fBuffer.data() + fCurrentOffset, RemainingBytes());
-	if (status != B_OK) {
-		throw BNetworkRequestError("BDataIO::WriteExactly()", BNetworkRequestError::SystemError,
-			status);
-	}
-
-	// Entire buffer is written; reset internal buffer
-	Clear();
-}
-
-
-/*!
-	\brief Write the contents of the buffer through the helper \a func.
-
-	\param func Handle the actual writing. The function accepts a pointer and a size as inputs
-		and should return the number of actual written bytes, which may be fewer than the number
-		of available bytes.
-*/
-void
-HttpBuffer::WriteTo(HttpTransferFunction func , std::optional<size_t> maxSize)
-{
-	if (RemainingBytes() == 0)
-		return;
-
-	auto size = RemainingBytes();
-	if (maxSize.has_value() && *maxSize < size)
-		size = *maxSize;
-
-	auto bytesWritten = func(fBuffer.data() + fCurrentOffset, size);
-	if (bytesWritten > size)
-		throw BRuntimeError(__PRETTY_FUNCTION__, "More bytes written than were made available");
-
-	fCurrentOffset += bytesWritten;
-}
-
-
-/*!
-	\brief Get the next line from this buffer.
-
-	This can be called iteratively until all lines in the current data are read. After using this
-	method, you should use Flush() to make sure that the read lines are cleared from the beginning
-	of the buffer.
-
-	\retval std::nullopt There are no more lines in the buffer.
-	\retval BString The next line.
-*/
-std::optional<BString>
-HttpBuffer::GetNextLine()
-{
-	auto offset = fBuffer.cbegin() + fCurrentOffset;
-	auto result = std::search(offset, fBuffer.cend(), kNewLine.cbegin(), kNewLine.cend());
-	if (result == fBuffer.cend())
-		return std::nullopt;
-
-	BString line(reinterpret_cast<const char*>(std::addressof(*offset)), std::distance(offset, result));
-	fCurrentOffset = std::distance(fBuffer.cbegin(), result) + 2;
-	return line;
-}
-
-
-/*!
-	\brief Get the number of remaining bytes in this buffer.
-*/
-size_t
-HttpBuffer::RemainingBytes() noexcept
-{
-	return fBuffer.size() - fCurrentOffset;
-}
-
-
-/*!
-	\brief Move data to the beginning of the buffer to clear at the back.
-
-	The GetNextLine() increases the offset of the internal buffer. This call moves remaining data
-	to the beginning of the buffer sets the correct size, making the remainder of the capacity
-	available for further reading.
-*/
-void
-HttpBuffer::Flush() noexcept
-{
-	if (fCurrentOffset > 0) {
-		auto end = fBuffer.cbegin() + fCurrentOffset;
-		fBuffer.erase(fBuffer.cbegin(), end);
-		fCurrentOffset = 0;
-	}
-}
-
-
-/*!
-	\brief Clear the internal buffer
-*/
-void
-HttpBuffer::Clear() noexcept
-{
-	fBuffer.clear();
-	fCurrentOffset = 0;
-}
-
-
-/*!
-	\brief Parse the status from the \a buffer and store it in \a status.
-
-	\retval true The status was succesfully parsed
-	\retval false There is not enough data in the buffer for a full status.
-
-	\exception BNetworkRequestException The status does not conform to the HTTP spec.
-*/
-bool
-HttpParser::ParseStatus(HttpBuffer& buffer, BHttpStatus& status)
-{
-	auto statusLine = buffer.GetNextLine();
-	if (!statusLine)
-		return false;
-
-	auto codeStart = statusLine->FindFirst(' ') + 1;
-	if (codeStart < 0)
-		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
-
-	auto codeEnd = statusLine->FindFirst(' ', codeStart);
-
-	if (codeEnd < 0 || (codeEnd - codeStart) != 3)
-		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
-
-	std::string statusCodeString(statusLine->String() + codeStart, 3);
-
-	// build the output
-	try {
-		status.code = std::stol(statusCodeString);
-	} catch (...) {
-		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
-	}
-
-	status.text = std::move(statusLine.value());
-	return true;
-}
-
-
-/*!
-	\brief Parse the fields from the \a buffer and store it in \a fields.
-
-	The fields are parsed incrementally, meaning that even if the full header is not yet in the
-	\a buffer, it will still parse all complete fields and store them in the \a fields.
-
-	\retval true All fields were succesfully parsed
-	\retval false There is not enough data in the buffer to complete parsing of fields.
-
-	\exception BNetworkRequestException The fields not conform to the HTTP spec.
-*/
-bool
-HttpParser::ParseFields(HttpBuffer& buffer, BHttpFields& fields)
-{
-	auto fieldLine = buffer.GetNextLine();
-	
-	while (fieldLine && !fieldLine.value().IsEmpty()){
-		// Parse next header line
-		fields.AddField(fieldLine.value());
-		fieldLine = buffer.GetNextLine();
-	}
-
-	if (fieldLine && fieldLine.value().IsEmpty()){
-		// end of the header section of the message
-		return true;
-	} else {
-		// there is more to parse
-		return false;
-	}
-}
-
-
-/*!
-	\brief Parse the Gzip compression from the body.
-
-	\exception std::bad_alloc in case there is an error allocating memory.
-*/
-void
-HttpParser::SetGzipCompression(bool compression)
-{
-	if (compression) {
-		fDecompressorStorage = std::make_unique<BMallocIO>();
-
-		BDataIO* stream = nullptr;
-		auto result = BZlibCompressionAlgorithm()
-			.CreateDecompressingOutputStream(fDecompressorStorage.get(), nullptr, stream);
-		
-		if (result != B_OK) {
-			throw BNetworkRequestError(
-				"BZlibCompressionAlgorithm().CreateCompressingOutputStream",
-				BNetworkRequestError::SystemError, result);
-		}
-
-		fDecompressingStream = std::unique_ptr<BDataIO>(stream);
-	} else {
-		fDecompressingStream = nullptr;
-		fDecompressorStorage = nullptr;
-	}
-}
-
-
-/*!
-	\brief Set the content length of the body.
-
-	If a content length is set, the body will not be handled as a chunked transfer.
-*/
-void
-HttpParser::SetContentLength(std::optional<off_t> contentLength) noexcept
-{
-	fBodyBytesTotal = contentLength;
-}
-
-
-/*!
-	\brief Parse the body from the \a buffer and use \a writeToBody function to save.
-*/
-size_t
-HttpParser::ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody)
-{
-	if (fBodyBytesTotal.has_value()) {
-		return _ParseBodyRaw(buffer, writeToBody);
-	} else {
-		return _ParseBodyChunked(buffer, writeToBody);
-	}
-}
-
-
-/*!
-	\brief Parse the body from the \a buffer and use \a writeToBody function to save.
-*/
-size_t
-HttpParser::_ParseBodyRaw(HttpBuffer& buffer, HttpTransferFunction writeToBody)
-{
-	if (fBodyBytesTotal && (fTransferredBodySize + static_cast<off_t>(buffer.RemainingBytes()))
-		> *fBodyBytesTotal)
-		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
-
-	auto bytesToRead = buffer.RemainingBytes();
-	auto readEnd = fBodyBytesTotal.value()
-		== (fTransferredBodySize + static_cast<off_t>(bytesToRead));
-
-	auto bytesRead = _ReadChunk(buffer, writeToBody, bytesToRead, readEnd);
-	fTransferredBodySize += bytesRead;
-	return bytesRead;
-}
-
-
-/*!
-	\brief Parse the body from the \a buffer and use \a writeToBody function to save.
-*/
-size_t
-HttpParser::_ParseBodyChunked(HttpBuffer& buffer, HttpTransferFunction writeToBody)
-{
-	size_t totalBytesRead = 0;
-	while (buffer.RemainingBytes() > 0) {
-		switch (fBodyState) {
-		case HttpBodyInputStreamState::ChunkSize:
-		{
-			// Read the next chunk size from the buffer; if unsuccesful wait for more data
-			auto chunkSizeString = buffer.GetNextLine();
-			if (!chunkSizeString)
-				return totalBytesRead;
-			auto chunkSizeStr = std::string(chunkSizeString.value().String());
-			try {
-				size_t pos = 0;
-				fRemainingChunkSize = std::stoll(chunkSizeStr, &pos, 16);
-				if (pos < chunkSizeStr.size() && chunkSizeStr[pos] != ';'){
-					throw BNetworkRequestError(__PRETTY_FUNCTION__,
-						BNetworkRequestError::ProtocolError);
-				}
-			} catch (const std::invalid_argument&) {
-				throw BNetworkRequestError(__PRETTY_FUNCTION__,
-					BNetworkRequestError::ProtocolError);
-			} catch (const std::out_of_range&) {
-				throw BNetworkRequestError(__PRETTY_FUNCTION__,
-					BNetworkRequestError::ProtocolError);
-			}
-
-			if (fRemainingChunkSize > 0)
-				fBodyState = HttpBodyInputStreamState::Chunk;
-			else
-				fBodyState = HttpBodyInputStreamState::Trailers;
-			break;
-		}
-
-		case HttpBodyInputStreamState::Chunk:
-		{
-			size_t bytesToRead;
-			bool readEnd = false;
-			if (fRemainingChunkSize > static_cast<off_t>(buffer.RemainingBytes()))
-				bytesToRead = buffer.RemainingBytes();
-			else {
-				readEnd = true;
-				bytesToRead = fRemainingChunkSize;
-			}
-
-			auto bytesRead = _ReadChunk(buffer, writeToBody, bytesToRead, readEnd);
-			fTransferredBodySize += bytesRead;
-			totalBytesRead += bytesRead;
-			fRemainingChunkSize -= bytesRead;
-			if (fRemainingChunkSize == 0)
-				fBodyState = HttpBodyInputStreamState::ChunkEnd;
-			break;
-		}
-
-		case HttpBodyInputStreamState::ChunkEnd:
-		{
-			if (buffer.RemainingBytes() < 2) {
-				// not enough data in the buffer to finish the chunk
-				return totalBytesRead;
-			}
-			auto chunkEndString = buffer.GetNextLine();
-			if (!chunkEndString || chunkEndString.value().Length() != 0) {
-				// There should have been an empty chunk
-				throw BNetworkRequestError(__PRETTY_FUNCTION__,
-					BNetworkRequestError::ProtocolError);
-			}
-
-			fBodyState = HttpBodyInputStreamState::ChunkSize;
-			break;
-		}
-
-		case HttpBodyInputStreamState::Trailers:
-		{
-			auto trailerString = buffer.GetNextLine();
-			if (!trailerString)  {
-				// More data to come
-				return totalBytesRead;
-			}
-
-			if (trailerString.value().Length() > 0) {
-				// Ignore empty trailers for now
-				// TODO: review if the API should support trailing headers
-			} else {
-				fBodyState = HttpBodyInputStreamState::Done;
-				return totalBytesRead;
-			}
-			break;
-		}
-
-		case HttpBodyInputStreamState::Done:
-			return totalBytesRead;
-		}
-	}
-	return totalBytesRead;
-}
-
-
-/*!
-	\brief Check if the body is fully parsed.
-*/
-bool
-HttpParser::Complete() const noexcept
-{
-	if (_IsChunked())
-		return fBodyState == HttpBodyInputStreamState::Done;
-
-	return fBodyBytesTotal.value() == fTransferredBodySize;
-}
-
-
-/*!
-	\brief Read a chunk of data from the buffer and write it to the output.
-
-	If there is a compression algorithm applied, then it passes through the compression first.
-
-	\param buffer The buffer to read from
-	\param writeToBody The function that can write data from the buffer to the body.
-	\param size The maximum size to read from the buffer. When larger than the buffer size, the
-		remaining bytes from the buffer are read.
-	\param flush Setting this parameter will force the decompression to write out all data, if
-		applicable. Set when all the data has been received.
-
-	\exception BNetworkRequestError When there was any error with any of the system cals.
-*/
-size_t
-HttpParser::_ReadChunk(HttpBuffer& buffer, HttpTransferFunction writeToBody, size_t size, bool flush)
-{
-	if (size == 0)
-		return 0;
-
-	if (size > buffer.RemainingBytes())
-		size = buffer.RemainingBytes();
-
-	if (fDecompressingStream) {
-		buffer.WriteTo([this](const std::byte* buffer, size_t bufferSize){
-			auto status = fDecompressingStream->WriteExactly(buffer, bufferSize);
-			if (status != B_OK) {
-				throw BNetworkRequestError("BDataIO::WriteExactly()",
-					BNetworkRequestError::SystemError, status);
-			}
-			return bufferSize;
-		}, size);
-
-		if (flush) {
-			// No more bytes expected so flush out the final bytes
-			if (auto status = fDecompressingStream->Flush(); status != B_OK)
-				throw BNetworkRequestError("BZlibDecompressionStream::Flush()",
-					BNetworkRequestError::SystemError, status);
-		}
-
-		if (auto bodySize = fDecompressorStorage->Position(); bodySize > 0) {
-			auto bytesWritten
-				= writeToBody(static_cast<const std::byte*>(fDecompressorStorage->Buffer()),
-					bodySize);
-			if (static_cast<off_t>(bytesWritten) != bodySize) {
-				throw BNetworkRequestError(__PRETTY_FUNCTION__,
-					BNetworkRequestError::SystemError, B_PARTIAL_WRITE);
-			}
-			fDecompressorStorage->Seek(0, SEEK_SET);
-		} 
-	} else {
-		// Write the body directly to the target
-		buffer.WriteTo(writeToBody, size);
-	}
-	return size;
-}
-
-
-/*!
-	\brief Internal helper to determine if the body is sent as a chunked transfer.
-*/
-bool
-HttpParser::_IsChunked() const noexcept
-{
-	return fBodyBytesTotal == std::nullopt;
-}

src/kits/network/libnetservices2/Jamfile

@@ -19,7 +19,9 @@ for architectureObject in [ MultiArchSubDirSetup ] {
 
 		StaticLibrary [ MultiArchDefaultGristFiles libnetservices2.a ] :
 			ErrorsExt.cpp
+			HttpBuffer.cpp
 			HttpFields.cpp
+			HttpParser.cpp
 			HttpRequest.cpp
 			HttpResult.cpp
 			HttpSession.cpp