diff --git a/headers/private/netservices2/HttpStream.h b/headers/private/netservices2/HttpStream.h index 77b6b81f6e..7464e61328 100644 --- a/headers/private/netservices2/HttpStream.h +++ b/headers/private/netservices2/HttpStream.h @@ -20,7 +20,9 @@ namespace BPrivate { namespace Network { +class BHttpFields; class BHttpRequest; +class BHttpStatus; class BAbstractDataStream { @@ -57,16 +59,16 @@ private: }; -class HttpBuffer { -public: - using WriteFunction = std::function; +using HttpTransferFunction = std::function; + +class HttpBuffer { public: HttpBuffer(size_t capacity = 8*1024); ssize_t ReadFrom(BDataIO* source); void WriteExactlyTo(BDataIO* target); - void WriteTo(WriteFunction func); + void WriteTo(HttpTransferFunction func); std::optional GetNextLine(); size_t RemainingBytes() noexcept; @@ -80,6 +82,46 @@ private: }; +class HttpParser { +public: + HttpParser() {}; + + // HTTP Header + bool ParseStatus(HttpBuffer& buffer, BHttpStatus& status); + bool ParseFields(HttpBuffer& buffer, BHttpFields& fields); + + // HTTP Body + void SetGzipCompression(bool compression = true); + void SetContentLength(std::optional contentLength) noexcept; + + size_t ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody); + std::optional BodyBytesTotal() const noexcept { return fBodyBytesTotal; }; + off_t BodyBytesTransferred() const noexcept { return fTransferredBodySize; }; + bool Complete() const noexcept; + +private: + size_t _ReadChunk(HttpBuffer& buffer, HttpTransferFunction writeToBody, + size_t maxSize, bool flush); + bool _IsChunked() const noexcept; + +private: + off_t fHeaderBytes = 0; + + // Support for chunked transfers + std::optional fRemainingChunkSize; + bool fChunkedTransferComplete = false; + + // Receive stats + std::optional fBodyBytesTotal = 0; + off_t fTransferredBodySize = 0; + + // Optional decompression + std::unique_ptr fDecompressorStorage = nullptr; + std::unique_ptr fDecompressingStream = nullptr; + +}; + + } // namespace Network } // namespace BPrivate diff --git a/src/kits/network/libnetservices2/HttpSession.cpp b/src/kits/network/libnetservices2/HttpSession.cpp index d622d686c7..2aa08f0172 100644 --- a/src/kits/network/libnetservices2/HttpSession.cpp +++ b/src/kits/network/libnetservices2/HttpSession.cpp @@ -101,9 +101,6 @@ public: int Socket() const noexcept { return fSocket->Socket(); } int32 Id() const noexcept { return fResult->id; } bool CanCancel() const noexcept { return fResult->CanCancel(); } - -private: - BHttpStatus _ParseStatus(BString&& statusLine); private: BHttpRequest fRequest; @@ -125,10 +122,9 @@ private: // Receive buffers HttpBuffer fBuffer; + HttpParser fParser; // Receive state - off_t fBodyBytesTotal = 0; - off_t fBodyBytesReceived = 0; BHttpFields fFields; bool fNoContent = false; @@ -784,14 +780,8 @@ BHttpSession::Request::ReceiveResult() "Read function called for object that is not yet connected or sent"); case RequestSent: { - auto statusLine = fBuffer.GetNextLine(); BHttpStatus status; - - if (statusLine) { - std::cout << "statusLine: " << statusLine.value() << std::endl; - status = _ParseStatus(std::move(statusLine.value())); - } - if (status.code != 0) { + if (fParser.ParseStatus(fBuffer, status)) { // the status headers are now received, decide what to do next // Determine if we can handle redirects; else notify of receiving status @@ -842,24 +832,12 @@ BHttpSession::Request::ReceiveResult() // We do not have enough data for the status line yet, continue receiving data. return false; } - [[fallthrough]]; } case StatusReceived: { - auto fieldLine = fBuffer.GetNextLine(); - while (fieldLine && !fieldLine.value().IsEmpty()){ - std::cout << "ReceiveResult() [" << Id() << "] StatusReceived; adding header " << fieldLine.value() << std::endl; - // Parse next header line - fFields.AddField(fieldLine.value()); - fieldLine = fBuffer.GetNextLine(); - } - - if (fieldLine && fieldLine.value().IsEmpty()){ - std::cout << "ReceiveResult() [" << Id() << "] End of Header Block of Message" << std::endl; - // end of the header section of the message - } else { - // no more lines to process, and we are not done with receiving headers yet. + if (!fParser.ParseFields(fBuffer, fFields)) { + // there may be more headers to receive. break; } @@ -901,10 +879,11 @@ BHttpSession::Request::ReceiveResult() // TODO: Parse received cookies // Handle Chunked Transfers + auto chunked = false; auto header = fFields.FindField("Transfer-Encoding"); if (header != fFields.end() && header->Value() == "chunked") { - // TODO: Implement chunked transfers - throw BRuntimeError(__PRETTY_FUNCTION__, "Chunked transfers are not supported"); + fParser.SetContentLength(std::nullopt); + chunked = true; } // Content-encoding @@ -913,33 +892,27 @@ BHttpSession::Request::ReceiveResult() && (header->Value() == "gzip" || header->Value() == "deflate")) { std::cout << "ReceiveResult() [" << Id() << "] Content-Encoding has compression: " << header->Value() << std::endl; - - fDecompressorStorage = std::make_unique(); - - BDataIO* stream = nullptr; - auto result = BZlibCompressionAlgorithm() - .CreateDecompressingOutputStream(fDecompressorStorage.get(), nullptr, stream); - - if (result != B_OK) { - throw BNetworkRequestError( - "BZlibCompressionAlgorithm().CreateCompressingOutputStream", - BNetworkRequestError::SystemError, result); - } - - fDecompressingStream = std::unique_ptr(stream); + fParser.SetGzipCompression(true); } // Content-length - header = fFields.FindField("Content-Length"); - if (header != fFields.end()) { - try { - auto contentLength = std::string(header->Value()); - fBodyBytesTotal = std::stol(contentLength); - if (fBodyBytesTotal == 0) - fNoContent = true; - } catch (const std::logic_error& e) { - throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError); + if (!chunked && !fNoContent && fRequest.Method() != BHttpMethod::Head) { + std::optional bodyBytesTotal = std::nullopt; + header = fFields.FindField("Content-Length"); + if (header != fFields.end()) { + try { + auto contentLength = std::string(header->Value()); + bodyBytesTotal = std::stol(contentLength); + if (bodyBytesTotal.value() == 0) + fNoContent = true; + fParser.SetContentLength(bodyBytesTotal); + } catch (const std::logic_error& e) { + throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError); + } } + + if (bodyBytesTotal == std::nullopt) + throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError); } // TODO: move headers to the result and inform listener @@ -957,73 +930,19 @@ BHttpSession::Request::ReceiveResult() } case HeadersReceived: { - // TODO: handle chunked transfer + bytesRead = fParser.ParseBody(fBuffer, [this](const std::byte* buffer, size_t size) { + return fResult->WriteToBody(buffer, size); + }); - bytesRead = fBuffer.RemainingBytes(); - fBodyBytesReceived += bytesRead; std::cout << "ReceiveResult() [" << Id() << "] body bytes current read/total received/total expected: " << - bytesRead << "/" << fBodyBytesReceived << "/" << fBodyBytesTotal << std::endl; + bytesRead << "/" << fParser.BodyBytesTransferred() << "/" << fParser.BodyBytesTotal().value_or(0) << std::endl; - // Normally, the request is done when the number of bytes received is the number of bytes expected. - // The exceptions are: - // For chunked transfers (with unknown total size) - // HTTP HEAD requests (will never have a body) - if (fBodyBytesTotal > 0 && fBodyBytesReceived == fBodyBytesTotal) { - std::cout << "ReceiveResult() [" << Id() << "] received all body bytes: " << fBodyBytesTotal << std::endl; - receiveEnd = true; - } else if (fBodyBytesTotal > 0 && fBodyBytesReceived > fBodyBytesTotal) { - std::cout << "ReceiveResult() [" << Id() << "] received more body than expected: " - << fBodyBytesReceived << "/" << fBodyBytesTotal << std::endl; - throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError); + if (fParser.Complete()) { + std::cout << "ReceiveResult() [" << Id() << "] received all body bytes: " << fParser.BodyBytesTransferred() << std::endl; + fResult->SetBody(); + return true; } - // TODO: check for HEAD requests and chunked requests - - // Process the incoming data and write to body - if (bytesRead > 0) { - if (fDecompressingStream) { - fBuffer.WriteExactlyTo(fDecompressingStream.get()); - - if (receiveEnd) { - // No more bytes expected so flush out the final bytes - if (auto status = fDecompressingStream->Flush(); status != B_OK) - throw BNetworkRequestError("BZlibDecompressionStream::Flush()", - BNetworkRequestError::SystemError, status); - } - - if (auto bodySize = fDecompressorStorage->Position(); bodySize > 0) { - std::cout << "ReceiveResult() [" << Id() << "] Decompressed " << bodySize << " bytes and copying into target." << std::endl; - fResult->WriteToBody(fDecompressorStorage->Buffer(), bodySize); - fDecompressorStorage->Seek(0, SEEK_SET); - } - } else { - fBuffer.WriteTo([this](const std::byte* buffer, size_t size) { - return fResult->WriteToBody(buffer, size); - }); - } - } - - if (receiveEnd) { - // Normally, the request is done when the number of bytes received is the number of bytes expected. - // The exceptions are: - // For chunked transfers (with unknown total size) - // HTTP HEAD requests (will never have a body) - if (fBodyBytesTotal > 0) { - if(fBodyBytesReceived == fBodyBytesTotal) { - std::cout << "ReceiveResult() [" << Id() << "] received all body bytes: " << fBodyBytesTotal << std::endl; - fResult->SetBody(); - return true; - } else { - throw BNetworkRequestError(__PRETTY_FUNCTION__, - BNetworkRequestError::ProtocolError); - } - } else { - // TODO: validate that HTTP HEAD requests are handled perfectly - // The expectation is that broken HTTP chunked requests would be noticed before here. - fResult->SetBody(); - return true; - } - } break; } default: @@ -1046,41 +965,3 @@ BHttpSession::Request::Disconnect() noexcept // TODO: inform listeners that the request has ended } - - -/*! - \brief Parse a HTTP status line, and return a BHttpStatus object on success - - \exception BNetworkRequestError If the status line does not follow protocol. -*/ -BHttpStatus -BHttpSession::Request::_ParseStatus(BString&& statusLine) -{ - // From the RFC: - // status-line = HTTP-version SP status-code SP reason-phrase CRLF - // note that the reason phrase may also contain spaces. - - std::cout << "_ParseStatus() [" << Id() << "] status line: " << statusLine << std::endl; - - auto codeStart = statusLine.FindFirst(' ') + 1; - if (codeStart < 0) - throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError); - - auto codeEnd = statusLine.FindFirst(' ', codeStart); - - if (codeEnd < 0 || (codeEnd - codeStart) != 3) - throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError); - - std::string statusCodeString(statusLine.String() + codeStart, 3); - std::cout << "_ParseStatus() [" << Id() << "] status code string: " << statusCodeString << std::endl; - - // build the output - BHttpStatus status = {0, std::move(statusLine)}; - try { - status.code = std::stol(statusCodeString); - } catch (...) { - throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError); - } - - return status; -} diff --git a/src/kits/network/libnetservices2/HttpStream.cpp b/src/kits/network/libnetservices2/HttpStream.cpp index d2b2ef393b..ef209ac2a3 100644 --- a/src/kits/network/libnetservices2/HttpStream.cpp +++ b/src/kits/network/libnetservices2/HttpStream.cpp @@ -10,10 +10,14 @@ #include #include +#include #include +#include #include +#include #include +#include using namespace BPrivate::Network; @@ -291,7 +295,7 @@ HttpBuffer::WriteExactlyTo(BDataIO* target) of available bytes. */ void -HttpBuffer::WriteTo(WriteFunction func) +HttpBuffer::WriteTo(HttpTransferFunction func) { if (RemainingBytes() == 0) return; @@ -365,3 +369,226 @@ HttpBuffer::Clear() noexcept fBuffer.clear(); fCurrentOffset = 0; } + + +/*! + \brief Parse the status from the \a buffer and store it in \a status. + + \retval true The status was succesfully parsed + \retval false There is not enough data in the buffer for a full status. + + \exception BNetworkRequestException The status does not conform to the HTTP spec. +*/ +bool +HttpParser::ParseStatus(HttpBuffer& buffer, BHttpStatus& status) +{ + auto statusLine = buffer.GetNextLine(); + if (!statusLine) + return false; + + auto codeStart = statusLine->FindFirst(' ') + 1; + if (codeStart < 0) + throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError); + + auto codeEnd = statusLine->FindFirst(' ', codeStart); + + if (codeEnd < 0 || (codeEnd - codeStart) != 3) + throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError); + + std::string statusCodeString(statusLine->String() + codeStart, 3); + + // build the output + try { + status.code = std::stol(statusCodeString); + } catch (...) { + throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError); + } + + status.text = std::move(statusLine.value()); + return true; +} + + +/*! + \brief Parse the fields from the \a buffer and store it in \a fields. + + The fields are parsed incrementally, meaning that even if the full header is not yet in the + \a buffer, it will still parse all complete fields and store them in the \a fields. + + \retval true All fields were succesfully parsed + \retval false There is not enough data in the buffer to complete parsing of fields. + + \exception BNetworkRequestException The fields not conform to the HTTP spec. +*/ +bool +HttpParser::ParseFields(HttpBuffer& buffer, BHttpFields& fields) +{ + auto fieldLine = buffer.GetNextLine(); + + while (fieldLine && !fieldLine.value().IsEmpty()){ + // Parse next header line + fields.AddField(fieldLine.value()); + fieldLine = buffer.GetNextLine(); + } + + if (fieldLine && fieldLine.value().IsEmpty()){ + // end of the header section of the message + return true; + } else { + // there is more to parse + return false; + } +} + + +/*! + \brief Parse the Gzip compression from the body. + + \exception std::bad_alloc in case there is an error allocating memory. +*/ +void +HttpParser::SetGzipCompression(bool compression) +{ + if (compression) { + fDecompressorStorage = std::make_unique(); + + BDataIO* stream = nullptr; + auto result = BZlibCompressionAlgorithm() + .CreateDecompressingOutputStream(fDecompressorStorage.get(), nullptr, stream); + + if (result != B_OK) { + throw BNetworkRequestError( + "BZlibCompressionAlgorithm().CreateCompressingOutputStream", + BNetworkRequestError::SystemError, result); + } + + fDecompressingStream = std::unique_ptr(stream); + } else { + fDecompressingStream = nullptr; + fDecompressorStorage = nullptr; + } +} + + +/*! + \brief Set the content length of the body. + + If a content length is set, the body will not be handled as a chunked transfer. +*/ +void +HttpParser::SetContentLength(std::optional contentLength) noexcept +{ + fBodyBytesTotal = contentLength; +} + + +/*! + \brief Parse the body from the \a buffer and use \a writeToBody function to save. +*/ +size_t +HttpParser::ParseBody(HttpBuffer& buffer, HttpTransferFunction writeToBody) +{ + if (fBodyBytesTotal && (fTransferredBodySize + buffer.RemainingBytes()) > fBodyBytesTotal) + throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::ProtocolError); + + size_t bytesRead = 0; + size_t bytesToRead = 0; + bool readEnd = false; + while (buffer.RemainingBytes() > 0) { + if (_IsChunked()) { + bytesToRead = 100; + readEnd = false; + } else { + bytesToRead = buffer.RemainingBytes(); + readEnd = fBodyBytesTotal.value() + == (fTransferredBodySize + static_cast(bytesToRead)); + } + + bytesRead += _ReadChunk(buffer, writeToBody, bytesToRead, readEnd); + } + fTransferredBodySize += bytesRead; + return bytesRead; +} + + +/*! + \brief Check if the body is fully parsed. +*/ +bool +HttpParser::Complete() const noexcept +{ + if (_IsChunked()) + return fChunkedTransferComplete; + + return fBodyBytesTotal.value() == fTransferredBodySize; +} + + +/*! + \brief Read a chunk of data from the buffer and write it to the output. + + If there is a compression algorithm applied, then it passes through the compression first. + + \param buffer The buffer to read from + \param writeToBody The function that can write data from the buffer to the body. + \param size The maximum size to read from the buffer. When larger than the buffer size, the + remaining bytes from the buffer are read. + \param flush Setting this parameter will force the decompression to write out all data, if + applicable. Set when all the data has been received. + + \exception BNetworkRequestError When there was any error with any of the system cals. +*/ +size_t +HttpParser::_ReadChunk(HttpBuffer& buffer, HttpTransferFunction writeToBody, size_t size, bool flush) +{ + if (size == 0) + return 0; + + if (size > buffer.RemainingBytes()) + size = buffer.RemainingBytes(); + + if (fDecompressingStream) { + buffer.WriteTo([this, &size](const std::byte* buffer, size_t bufferSize){ + if (size < bufferSize) + bufferSize = size; + auto status = fDecompressingStream->WriteExactly(buffer, bufferSize); + if (status != B_OK) { + throw BNetworkRequestError("BDataIO::WriteExactly()", + BNetworkRequestError::SystemError, status); + } + return bufferSize; + }); + + if (flush) { + // No more bytes expected so flush out the final bytes + if (auto status = fDecompressingStream->Flush(); status != B_OK) + throw BNetworkRequestError("BZlibDecompressionStream::Flush()", + BNetworkRequestError::SystemError, status); + } + + if (auto bodySize = fDecompressorStorage->Position(); bodySize > 0) { + auto bytesWritten + = writeToBody(static_cast(fDecompressorStorage->Buffer()), + bodySize); + if (static_cast(bytesWritten) != bodySize) { + throw BNetworkRequestError(__PRETTY_FUNCTION__, + BNetworkRequestError::SystemError, B_PARTIAL_WRITE); + } + fDecompressorStorage->Seek(0, SEEK_SET); + } + } else { + // Write the body directly to the target + buffer.WriteTo(writeToBody); + } + return size; +} + + +/*! + \brief Internal helper to determine if the body is sent as a chunked transfer. +*/ +bool +HttpParser::_IsChunked() const noexcept +{ + return fBodyBytesTotal == std::nullopt; +}