NetServices: Refactor the parsing code into a HttpParser helper class

Change-Id: Ib03aacc36d46318c8870554f794ff66db4cce063

headers/private/netservices2/HttpStream.h

@@ -20,7 +20,9 @@ namespace BPrivate {
 
 namespace Network {
 
+class BHttpFields;
 class BHttpRequest;
+class BHttpStatus;
 
 
 class BAbstractDataStream {
@@ -57,16 +59,16 @@ private:
 };
 
 
-class HttpBuffer {
+using HttpTransferFunction = std::function<size_t (const std::byte*, size_t)>;
-public:
-	using WriteFunction = std::function<size_t (const std::byte*, size_t)>;
 
+
+class HttpBuffer {
 public:
 							HttpBuffer(size_t capacity = 8*1024);
 
 	ssize_t					ReadFrom(BDataIO* source);
 	void					WriteExactlyTo(BDataIO* target);
-	void					WriteTo(WriteFunction func);
+	void					WriteTo(HttpTransferFunction func);
 	std::optional<BString>	GetNextLine();
 
 	size_t					RemainingBytes() noexcept;
@@ -80,6 +82,46 @@ private:
 };
 
 
+class HttpParser {
+public:
+							HttpParser() {};
+
+	// HTTP Header
+	bool					ParseStatus(HttpBuffer& buffer, BHttpStatus& status);
+	bool					ParseFields(HttpBuffer& buffer, BHttpFields& fields);
+
+	// HTTP Body
+	void					SetGzipCompression(bool compression = true);
+	void					SetContentLength(std::optional<off_t> contentLength) noexcept;
+
+	size_t					ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody);
+	std::optional<off_t>	BodyBytesTotal() const noexcept { return fBodyBytesTotal; };
+	off_t					BodyBytesTransferred() const noexcept { return fTransferredBodySize; };
+	bool					Complete() const noexcept;
+
+private:
+	size_t					_ReadChunk(HttpBuffer& buffer, HttpTransferFunction writeToBody,
+								size_t maxSize, bool flush);
+	bool					_IsChunked() const noexcept;
+
+private:
+	off_t					fHeaderBytes = 0;
+
+	// Support for chunked transfers
+	std::optional<off_t>	fRemainingChunkSize;
+	bool					fChunkedTransferComplete = false;
+
+	// Receive stats
+	std::optional<off_t>	fBodyBytesTotal = 0;
+	off_t					fTransferredBodySize = 0;
+
+	// Optional decompression
+	std::unique_ptr<BMallocIO>		fDecompressorStorage = nullptr;
+	std::unique_ptr<BDataIO>		fDecompressingStream = nullptr;
+
+};
+
+
 } // namespace Network
 
 } // namespace BPrivate

src/kits/network/libnetservices2/HttpSession.cpp

@@ -102,9 +102,6 @@ public:
 	int32							Id() const noexcept { return fResult->id; }
 	bool							CanCancel() const noexcept { return fResult->CanCancel(); }
 
-private:
-	BHttpStatus						_ParseStatus(BString&& statusLine);
-
 private:
 	BHttpRequest					fRequest;
 
@@ -125,10 +122,9 @@ private:
 
 	// Receive buffers
 	HttpBuffer						fBuffer;
+	HttpParser						fParser;
 
 	// Receive state
-	off_t							fBodyBytesTotal = 0;
-	off_t							fBodyBytesReceived = 0;
 	BHttpFields						fFields;
 	bool							fNoContent = false;
 
@@ -784,14 +780,8 @@ BHttpSession::Request::ReceiveResult()
 			"Read function called for object that is not yet connected or sent");
 	case RequestSent:
 	{
-		auto statusLine = fBuffer.GetNextLine();
 		BHttpStatus status;
-
+		if (fParser.ParseStatus(fBuffer, status)) {
-		if (statusLine) {
-			std::cout << "statusLine: " << statusLine.value() << std::endl;
-			status = _ParseStatus(std::move(statusLine.value()));
-		}
-		if (status.code != 0) {
 			// the status headers are now received, decide what to do next
 
 			// Determine if we can handle redirects; else notify of receiving status
@@ -842,24 +832,12 @@ BHttpSession::Request::ReceiveResult()
 			// We do not have enough data for the status line yet, continue receiving data.
 			return false;
 		}
-
 		[[fallthrough]];
 	}
 	case StatusReceived:
 	{
-		auto fieldLine = fBuffer.GetNextLine();
+		if (!fParser.ParseFields(fBuffer, fFields)) {
-		while (fieldLine && !fieldLine.value().IsEmpty()){
+			// there may be more headers to receive.
-			std::cout << "ReceiveResult() [" << Id() << "] StatusReceived; adding header " << fieldLine.value() << std::endl;
-			// Parse next header line
-			fFields.AddField(fieldLine.value());
-			fieldLine = fBuffer.GetNextLine();
-		}
-
-		if (fieldLine && fieldLine.value().IsEmpty()){
-			std::cout << "ReceiveResult() [" << Id() << "] End of Header Block of Message" << std::endl;
-			// end of the header section of the message
-		} else {
-			// no more lines to process, and we are not done with receiving headers yet.
 			break;
 		}
 
@@ -901,10 +879,11 @@ BHttpSession::Request::ReceiveResult()
 		// TODO: Parse received cookies
 
 		// Handle Chunked Transfers
+		auto chunked = false;
 		auto header = fFields.FindField("Transfer-Encoding");
 		if (header != fFields.end() && header->Value() == "chunked") {
-			// TODO: Implement chunked transfers
+			fParser.SetContentLength(std::nullopt);
-			throw BRuntimeError(__PRETTY_FUNCTION__, "Chunked transfers are not supported");
+			chunked = true;
 		}
 
 		// Content-encoding
@@ -913,33 +892,27 @@ BHttpSession::Request::ReceiveResult()
 			&& (header->Value() == "gzip" || header->Value() == "deflate"))
 		{
 			std::cout << "ReceiveResult() [" << Id() << "] Content-Encoding has compression: " <<  header->Value() << std::endl;
-
+			fParser.SetGzipCompression(true);
-			fDecompressorStorage = std::make_unique<BMallocIO>();
-
-			BDataIO* stream = nullptr;
-			auto result = BZlibCompressionAlgorithm()
-				.CreateDecompressingOutputStream(fDecompressorStorage.get(), nullptr, stream);
-			
-			if (result != B_OK) {
-				throw BNetworkRequestError(
-					"BZlibCompressionAlgorithm().CreateCompressingOutputStream",
-					BNetworkRequestError::SystemError, result);
-			}
-
-			fDecompressingStream = std::unique_ptr<BDataIO>(stream);
 		}
 
 		// Content-length
-		header = fFields.FindField("Content-Length");
+		if (!chunked && !fNoContent && fRequest.Method() != BHttpMethod::Head) {
-		if (header != fFields.end()) {
+			std::optional<off_t> bodyBytesTotal = std::nullopt;
-			try {
+			header = fFields.FindField("Content-Length");
-				auto contentLength = std::string(header->Value());
+			if (header != fFields.end()) {
-				fBodyBytesTotal = std::stol(contentLength);
+				try {
-				if (fBodyBytesTotal == 0)
+					auto contentLength = std::string(header->Value());
-					fNoContent = true;
+					bodyBytesTotal = std::stol(contentLength);
-			} catch (const std::logic_error& e) {
+					if (bodyBytesTotal.value() == 0)
-				throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
+						fNoContent = true;
+					fParser.SetContentLength(bodyBytesTotal);
+				} catch (const std::logic_error& e) {
+					throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
+				}
 			}
+
+			if (bodyBytesTotal  == std::nullopt)
+				throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
 		}
 
 		// TODO: move headers to the result and inform listener
@@ -957,73 +930,19 @@ BHttpSession::Request::ReceiveResult()
 	}
 	case HeadersReceived:
 	{
-		// TODO: handle chunked transfer
+		bytesRead = fParser.ParseBody(fBuffer, [this](const std::byte* buffer, size_t size) {
+			return fResult->WriteToBody(buffer, size);
+		});
 
-		bytesRead = fBuffer.RemainingBytes();
-		fBodyBytesReceived += bytesRead;
 		std::cout << "ReceiveResult() [" << Id() << "] body bytes current read/total received/total expected: " << 
-			bytesRead << "/" << fBodyBytesReceived << "/" << fBodyBytesTotal  << std::endl;
+			bytesRead << "/" << fParser.BodyBytesTransferred() << "/" << fParser.BodyBytesTotal().value_or(0) << std::endl;
 
-		// Normally, the request is done when the number of bytes received is the number of bytes expected.
+		if (fParser.Complete()) {
-		// The exceptions are:
+			std::cout << "ReceiveResult() [" << Id() << "] received all body bytes: " << fParser.BodyBytesTransferred() << std::endl;
-		//	For chunked transfers (with unknown total size)
+			fResult->SetBody();
-		//	HTTP HEAD requests (will never have a body)
+			return true;
-		if (fBodyBytesTotal > 0 && fBodyBytesReceived == fBodyBytesTotal) {
-			std::cout << "ReceiveResult() [" << Id() << "] received all body bytes: " << fBodyBytesTotal << std::endl;
-			receiveEnd = true;
-		} else if (fBodyBytesTotal > 0 && fBodyBytesReceived > fBodyBytesTotal) {
-			std::cout << "ReceiveResult() [" << Id() << "] received more body than expected: "
-				<< fBodyBytesReceived << "/" << fBodyBytesTotal << std::endl;
-			throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
 		}
 
-		// TODO: check for HEAD requests and chunked requests
-
-		// Process the incoming data and write to body
-		if (bytesRead > 0) {
-			if (fDecompressingStream) {
-				fBuffer.WriteExactlyTo(fDecompressingStream.get());
-
-				if (receiveEnd) {
-					// No more bytes expected so flush out the final bytes
-					if (auto status = fDecompressingStream->Flush(); status != B_OK)
-						throw BNetworkRequestError("BZlibDecompressionStream::Flush()",
-							BNetworkRequestError::SystemError, status);
-				}
-
-				if (auto bodySize = fDecompressorStorage->Position(); bodySize > 0) {
-					std::cout << "ReceiveResult() [" << Id() << "] Decompressed " << bodySize << " bytes and copying into target." << std::endl;
-					fResult->WriteToBody(fDecompressorStorage->Buffer(), bodySize);
-					fDecompressorStorage->Seek(0, SEEK_SET);
-				}
-			} else {
-				fBuffer.WriteTo([this](const std::byte* buffer, size_t size) {
-					return fResult->WriteToBody(buffer, size);
-				});
-			}
-		}
-
-		if (receiveEnd) {
-			// Normally, the request is done when the number of bytes received is the number of bytes expected.
-			// The exceptions are:
-			//	For chunked transfers (with unknown total size)
-			//	HTTP HEAD requests (will never have a body)
-			if (fBodyBytesTotal > 0) {
-				if(fBodyBytesReceived == fBodyBytesTotal) {
-					std::cout << "ReceiveResult() [" << Id() << "] received all body bytes: " << fBodyBytesTotal << std::endl;
-					fResult->SetBody();
-					return true;
-				} else {
-					throw BNetworkRequestError(__PRETTY_FUNCTION__,
-						BNetworkRequestError::ProtocolError);
-				}
-			} else {
-				// TODO: validate that HTTP HEAD requests are handled perfectly
-				// The expectation is that broken HTTP chunked requests would be noticed before here.
-				fResult->SetBody();
-				return true;
-			}
-		}
 		break;
 	}
 	default:
@@ -1046,41 +965,3 @@ BHttpSession::Request::Disconnect() noexcept
 
 	// TODO: inform listeners that the request has ended
 }
-
-
-/*!
-	\brief Parse a HTTP status line, and return a BHttpStatus object on success
-
-	\exception BNetworkRequestError If the status line does not follow protocol.
-*/
-BHttpStatus
-BHttpSession::Request::_ParseStatus(BString&& statusLine)
-{
-	// From the RFC:
-	//	status-line = HTTP-version SP status-code SP reason-phrase CRLF
-	//	note that the reason phrase may also contain spaces.
-
-	std::cout << "_ParseStatus() [" << Id() << "] status line: " << statusLine << std::endl;
-
-	auto codeStart = statusLine.FindFirst(' ') + 1;
-	if (codeStart < 0)
-		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
-
-	auto codeEnd = statusLine.FindFirst(' ', codeStart);
-
-	if (codeEnd < 0 || (codeEnd - codeStart) != 3)
-		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
-
-	std::string statusCodeString(statusLine.String() + codeStart, 3);
-	std::cout << "_ParseStatus() [" << Id() << "] status code string: " << statusCodeString << std::endl;
-
-	// build the output
-	BHttpStatus status = {0, std::move(statusLine)};
-	try {
-		status.code = std::stol(statusCodeString);
-	} catch (...) {
-		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
-	}
-
-	return status;
-}

src/kits/network/libnetservices2/HttpStream.cpp

@@ -10,10 +10,14 @@
 
 #include <algorithm>
 #include <optional>
+#include <string>
 
 #include <DataIO.h>
+#include <HttpFields.h>
 #include <HttpRequest.h>
+#include <HttpResult.h>
 #include <NetServicesDefs.h>
+#include <ZlibCompressionAlgorithm.h>
 
 using namespace BPrivate::Network;
 
@@ -291,7 +295,7 @@ HttpBuffer::WriteExactlyTo(BDataIO* target)
 		of available bytes.
 */
 void
-HttpBuffer::WriteTo(WriteFunction func)
+HttpBuffer::WriteTo(HttpTransferFunction func)
 {
 	if (RemainingBytes() == 0)
 		return;
@@ -365,3 +369,226 @@ HttpBuffer::Clear() noexcept
 	fBuffer.clear();
 	fCurrentOffset = 0;
 }
+
+
+/*!
+	\brief Parse the status from the \a buffer and store it in \a status.
+
+	\retval true The status was succesfully parsed
+	\retval false There is not enough data in the buffer for a full status.
+
+	\exception BNetworkRequestException The status does not conform to the HTTP spec.
+*/
+bool
+HttpParser::ParseStatus(HttpBuffer& buffer, BHttpStatus& status)
+{
+	auto statusLine = buffer.GetNextLine();
+	if (!statusLine)
+		return false;
+
+	auto codeStart = statusLine->FindFirst(' ') + 1;
+	if (codeStart < 0)
+		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
+
+	auto codeEnd = statusLine->FindFirst(' ', codeStart);
+
+	if (codeEnd < 0 || (codeEnd - codeStart) != 3)
+		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
+
+	std::string statusCodeString(statusLine->String() + codeStart, 3);
+
+	// build the output
+	try {
+		status.code = std::stol(statusCodeString);
+	} catch (...) {
+		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
+	}
+
+	status.text = std::move(statusLine.value());
+	return true;
+}
+
+
+/*!
+	\brief Parse the fields from the \a buffer and store it in \a fields.
+
+	The fields are parsed incrementally, meaning that even if the full header is not yet in the
+	\a buffer, it will still parse all complete fields and store them in the \a fields.
+
+	\retval true All fields were succesfully parsed
+	\retval false There is not enough data in the buffer to complete parsing of fields.
+
+	\exception BNetworkRequestException The fields not conform to the HTTP spec.
+*/
+bool
+HttpParser::ParseFields(HttpBuffer& buffer, BHttpFields& fields)
+{
+	auto fieldLine = buffer.GetNextLine();
+	
+	while (fieldLine && !fieldLine.value().IsEmpty()){
+		// Parse next header line
+		fields.AddField(fieldLine.value());
+		fieldLine = buffer.GetNextLine();
+	}
+
+	if (fieldLine && fieldLine.value().IsEmpty()){
+		// end of the header section of the message
+		return true;
+	} else {
+		// there is more to parse
+		return false;
+	}
+}
+
+
+/*!
+	\brief Parse the Gzip compression from the body.
+
+	\exception std::bad_alloc in case there is an error allocating memory.
+*/
+void
+HttpParser::SetGzipCompression(bool compression)
+{
+	if (compression) {
+		fDecompressorStorage = std::make_unique<BMallocIO>();
+
+		BDataIO* stream = nullptr;
+		auto result = BZlibCompressionAlgorithm()
+			.CreateDecompressingOutputStream(fDecompressorStorage.get(), nullptr, stream);
+		
+		if (result != B_OK) {
+			throw BNetworkRequestError(
+				"BZlibCompressionAlgorithm().CreateCompressingOutputStream",
+				BNetworkRequestError::SystemError, result);
+		}
+
+		fDecompressingStream = std::unique_ptr<BDataIO>(stream);
+	} else {
+		fDecompressingStream = nullptr;
+		fDecompressorStorage = nullptr;
+	}
+}
+
+
+/*!
+	\brief Set the content length of the body.
+
+	If a content length is set, the body will not be handled as a chunked transfer.
+*/
+void
+HttpParser::SetContentLength(std::optional<off_t> contentLength) noexcept
+{
+	fBodyBytesTotal = contentLength;
+}
+
+
+/*!
+	\brief Parse the body from the \a buffer and use \a writeToBody function to save.
+*/
+size_t
+HttpParser::ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody)
+{
+	if (fBodyBytesTotal && (fTransferredBodySize + buffer.RemainingBytes()) > fBodyBytesTotal)
+		throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError);
+
+	size_t bytesRead = 0;
+	size_t bytesToRead = 0;
+	bool readEnd = false;
+	while (buffer.RemainingBytes() > 0) {
+		if (_IsChunked()) {
+			bytesToRead = 100;
+			readEnd = false;
+		} else {
+			bytesToRead = buffer.RemainingBytes();
+			readEnd = fBodyBytesTotal.value()
+				== (fTransferredBodySize + static_cast<off_t>(bytesToRead));
+		}
+
+		bytesRead += _ReadChunk(buffer, writeToBody, bytesToRead, readEnd);
+	}
+	fTransferredBodySize += bytesRead;
+	return bytesRead;
+}
+
+
+/*!
+	\brief Check if the body is fully parsed.
+*/
+bool
+HttpParser::Complete() const noexcept
+{
+	if (_IsChunked())
+		return fChunkedTransferComplete;
+
+	return fBodyBytesTotal.value() == fTransferredBodySize;
+}
+
+
+/*!
+	\brief Read a chunk of data from the buffer and write it to the output.
+
+	If there is a compression algorithm applied, then it passes through the compression first.
+
+	\param buffer The buffer to read from
+	\param writeToBody The function that can write data from the buffer to the body.
+	\param size The maximum size to read from the buffer. When larger than the buffer size, the
+		remaining bytes from the buffer are read.
+	\param flush Setting this parameter will force the decompression to write out all data, if
+		applicable. Set when all the data has been received.
+
+	\exception BNetworkRequestError When there was any error with any of the system cals.
+*/
+size_t
+HttpParser::_ReadChunk(HttpBuffer& buffer, HttpTransferFunction writeToBody, size_t size, bool flush)
+{
+	if (size == 0)
+		return 0;
+
+	if (size > buffer.RemainingBytes())
+		size = buffer.RemainingBytes();
+
+	if (fDecompressingStream) {
+		buffer.WriteTo([this, &size](const std::byte* buffer, size_t bufferSize){
+			if (size < bufferSize)
+				bufferSize = size;
+			auto status = fDecompressingStream->WriteExactly(buffer, bufferSize);
+			if (status != B_OK) {
+				throw BNetworkRequestError("BDataIO::WriteExactly()",
+					BNetworkRequestError::SystemError, status);
+			}
+			return bufferSize;
+		});
+
+		if (flush) {
+			// No more bytes expected so flush out the final bytes
+			if (auto status = fDecompressingStream->Flush(); status != B_OK)
+				throw BNetworkRequestError("BZlibDecompressionStream::Flush()",
+					BNetworkRequestError::SystemError, status);
+		}
+
+		if (auto bodySize = fDecompressorStorage->Position(); bodySize > 0) {
+			auto bytesWritten
+				= writeToBody(static_cast<const std::byte*>(fDecompressorStorage->Buffer()),
+					bodySize);
+			if (static_cast<off_t>(bytesWritten) != bodySize) {
+				throw BNetworkRequestError(__PRETTY_FUNCTION__,
+					BNetworkRequestError::SystemError, B_PARTIAL_WRITE);
+			}
+			fDecompressorStorage->Seek(0, SEEK_SET);
+		} 
+	} else {
+		// Write the body directly to the target
+		buffer.WriteTo(writeToBody);
+	}
+	return size;
+}
+
+
+/*!
+	\brief Internal helper to determine if the body is sent as a chunked transfer.
+*/
+bool
+HttpParser::_IsChunked() const noexcept
+{
+	return fBodyBytesTotal == std::nullopt;
+}