From 3b172a3dc62c3b02138cf7e30d20ec8f404100d4 Mon Sep 17 00:00:00 2001 From: Niels Sascha Reedijk Date: Mon, 7 Mar 2022 08:03:10 +0000 Subject: [PATCH] NetServices: implement hostname resolution and connection for BHttpRequest BHttpSession::Execute() moves the request into the session, and returns a future BHttpResponse object. Currently implemented are resolving the hostname, and opening the connection. There is some scaffolding for the actual data transfer. Change-Id: I5a8a7a7f8680036b91cdba4beee140bbed6bfd5a --- docs/user/netservices/HttpSession.dox | 30 ++ headers/private/netservices2/HttpRequest.h | 5 +- headers/private/netservices2/HttpSession.h | 14 +- .../network/libnetservices2/HttpPrivate.h | 3 + .../network/libnetservices2/HttpRequest.cpp | 21 +- .../network/libnetservices2/HttpSession.cpp | 266 +++++++++++++++++- src/kits/network/libnetservices2/Jamfile | 3 + .../libnetservices2/NetServicesMisc.cpp | 23 +- .../libnetservices2/NetServicesPrivate.h | 25 ++ .../net/netservices2/HttpProtocolTest.cpp | 54 ++++ .../kits/net/netservices2/HttpProtocolTest.h | 1 + 11 files changed, 421 insertions(+), 24 deletions(-) create mode 100644 src/kits/network/libnetservices2/NetServicesPrivate.h diff --git a/docs/user/netservices/HttpSession.dox b/docs/user/netservices/HttpSession.dox index 1a9271cc78..c07b4bcd63 100644 --- a/docs/user/netservices/HttpSession.dox +++ b/docs/user/netservices/HttpSession.dox @@ -13,6 +13,16 @@ #if __cplusplus >= 201703L + +/*! + \file HttpSession.h + \ingroup netservices + \brief Provides classes and tools to schedule and execute HTTP requests. + + \since Haiku R1 +*/ + + namespace BPrivate { namespace Network { @@ -153,6 +163,26 @@ namespace Network { */ +/*! + \fn BHttpResult BHttpSession::Execute(BHttpRequest &&request, + std::unique_ptr< BDataIO > target=nullptr, BMessenger observer=BMessenger()) + \brief Schedule and execute a \a request. + + \param request The (valid) request to move from. + \param target An optional data buffer to write the incoming body of the request to. This can be + \c nullptr if you want to use the default internal storage. If you provide a buffer, it + must be wrapped in a \c std::unique_ptr. This means that you transfer ownership to the + session. After the request is finished, you can regain ownership. + \param observer An optional observer that will receive the progress and status messages for + this request. + + \return The \ref BHttpResult object that corresponds to this request, and that can be used to + monitor the progress. + + \since Haiku R1 +*/ + + } // namespace Network } // namespace BPrivate diff --git a/headers/private/netservices2/HttpRequest.h b/headers/private/netservices2/HttpRequest.h index e66e96b642..8fea24a677 100644 --- a/headers/private/netservices2/HttpRequest.h +++ b/headers/private/netservices2/HttpRequest.h @@ -88,8 +88,9 @@ public: void SetUrl(const BUrl& url); private: - struct Impl; - std::unique_ptr fData; + friend class BHttpSession; + struct Data; + std::unique_ptr fData; }; diff --git a/headers/private/netservices2/HttpSession.h b/headers/private/netservices2/HttpSession.h index 245eb3e7a7..bcea957741 100644 --- a/headers/private/netservices2/HttpSession.h +++ b/headers/private/netservices2/HttpSession.h @@ -1,5 +1,5 @@ /* - * Copyright 2021 Haiku Inc. All rights reserved. + * Copyright 2022 Haiku Inc. All rights reserved. * Distributed under the terms of the MIT License. */ @@ -8,12 +8,18 @@ #include +#include + class BUrl; + namespace BPrivate { namespace Network { +class BHttpRequest; +class BHttpResult; + class BHttpSession { public: @@ -27,6 +33,12 @@ public: BHttpSession& operator=(const BHttpSession&) noexcept; BHttpSession& operator=(BHttpSession&&) noexcept = delete; + + // Requests + BHttpResult Execute(BHttpRequest&& request, + std::unique_ptr target = nullptr, + BMessenger observer = BMessenger()); + private: class Request; class Impl; diff --git a/src/kits/network/libnetservices2/HttpPrivate.h b/src/kits/network/libnetservices2/HttpPrivate.h index 7d8335f0f9..8756f025cf 100644 --- a/src/kits/network/libnetservices2/HttpPrivate.h +++ b/src/kits/network/libnetservices2/HttpPrivate.h @@ -8,6 +8,9 @@ #include +#include +#include + namespace BPrivate { diff --git a/src/kits/network/libnetservices2/HttpRequest.cpp b/src/kits/network/libnetservices2/HttpRequest.cpp index dd80172b4e..feeb465b8f 100644 --- a/src/kits/network/libnetservices2/HttpRequest.cpp +++ b/src/kits/network/libnetservices2/HttpRequest.cpp @@ -130,15 +130,14 @@ BHttpMethod::Method() const noexcept } -// #pragma mark -- BHttpRequest::Impl +// #pragma mark -- BHttpRequest::Data static const BUrl kDefaultUrl = BUrl(); static const BHttpMethod kDefaultMethod = BHttpMethod::Get; -struct BHttpRequest::Impl { - BUrl url; +struct BHttpRequest::Data { + BUrl url = kDefaultUrl; BHttpMethod method = kDefaultMethod; - bool ssl = false; }; @@ -146,14 +145,14 @@ struct BHttpRequest::Impl { BHttpRequest::BHttpRequest() - : fData(std::make_unique()) + : fData(std::make_unique()) { } BHttpRequest::BHttpRequest(const BUrl& url) - : fData(std::make_unique()) + : fData(std::make_unique()) { SetUrl(url); } @@ -198,7 +197,7 @@ void BHttpRequest::SetMethod(const BHttpMethod& method) { if (!fData) - fData = std::make_unique(); + fData = std::make_unique(); fData->method = method; } @@ -207,15 +206,11 @@ void BHttpRequest::SetUrl(const BUrl& url) { if (!fData) - fData = std::make_unique(); + fData = std::make_unique(); if (!url.IsValid()) throw BInvalidUrl(__PRETTY_FUNCTION__, BUrl(url)); - if (url.Protocol() == "http") - fData->ssl = false; - else if (url.Protocol() == "https") - fData->ssl = true; - else { + if (url.Protocol() != "http" && url.Protocol() != "https") { // TODO: optimize BStringList with modern language features BStringList list; list.Add("http"); diff --git a/src/kits/network/libnetservices2/HttpSession.cpp b/src/kits/network/libnetservices2/HttpSession.cpp index 60c9443647..84efbfc1b3 100644 --- a/src/kits/network/libnetservices2/HttpSession.cpp +++ b/src/kits/network/libnetservices2/HttpSession.cpp @@ -10,15 +10,86 @@ #include #include +#include +#include #include +#include +#include +#include #include #include +#include +#include +#include +#include #include +#include +#include +#include +#include + +#include "HttpResultPrivate.h" +#include "NetServicesPrivate.h" using namespace BPrivate::Network; class BHttpSession::Request { +public: + Request(BHttpRequest&& request, + std::unique_ptr target, + BMessenger observer); + + // States + enum RequestState { + InitialState, + Connected, + StatusReceived, + HeadersReceived, + ContentReceived, + TrailingHeadersReceived + }; + RequestState State() const { return fRequestStatus; } + + // Result Helpers + std::shared_ptr + Result() { return fResult; } + void SetError(std::exception_ptr e) { fResult->SetError(e); } + + // Operational methods + void ResolveHostName(); + void OpenConnection(); + + // +private: + BHttpRequest fRequest; + + // Request state/events + RequestState fRequestStatus = InitialState; + + // Communication + BMessenger fObserver; + std::shared_ptr fResult; + + // Connection + BNetworkAddress fRemoteAddress; + std::unique_ptr fSocket; + + // Receive state +/* bool receiveEnd = false; + bool parseEnd = false; + BNetBuffer inputBuffer; + size_t previousBufferSize = 0; + off_t bytesReceived = 0; + off_t bytesTotal = 0; + BHttpFields headers; + bool readByChunks = false; + bool decompress = false; + DynamicBuffer decompressorStorage; + std::unique_ptr decompressingStream = nullptr; + std::vector inputTempBuffer = std::vector(4096); + BHttpStatus status; */ + // TODO: reset method to reset Connection and Receive State when redirected }; @@ -28,6 +99,10 @@ public: Impl(); ~Impl() noexcept; + BHttpResult Execute(BHttpRequest&& request, + std::unique_ptr target, + BMessenger observer); + private: // Thread functions static status_t ControlThreadFunc(void* arg); @@ -55,6 +130,9 @@ private: }; +// #pragma mark -- BHttpSession::Impl + + BHttpSession::Impl::Impl() : fControlQueueSem(create_sem(0, "http:control")), @@ -92,15 +170,112 @@ BHttpSession::Impl::~Impl() noexcept } +BHttpResult +BHttpSession::Impl::Execute(BHttpRequest&& request, std::unique_ptr target, + BMessenger observer) +{ + auto wRequest = Request(std::move(request), std::move(target), observer); + + auto retval = BHttpResult(wRequest.Result()); + auto lock = AutoLocker(fLock); + fControlQueue.push_back(std::move(wRequest)); + release_sem(fControlQueueSem); + return retval; +} + + /*static*/ status_t BHttpSession::Impl::ControlThreadFunc(void* arg) { BHttpSession::Impl* impl = static_cast(arg); + // Outer loop to use the fControlQueueSem when new items have entered the queue + while (true) { + if (auto status = acquire_sem(impl->fControlQueueSem); status == B_INTERRUPTED) + continue; + else if (status != B_OK) { + // Most likely B_BAD_SEM_ID indicating that the sem was deleted; go to cleanup + break; + } + + // Inner loop to process items on the queue + while (true) { + impl->fLock.Lock(); + if (impl->fControlQueue.empty() || atomic_get(&impl->fQuitting) == 1) { + impl->fLock.Unlock(); + break; + } + auto request = std::move(impl->fControlQueue.front()); + impl->fControlQueue.pop_front(); + impl->fLock.Unlock(); + + switch (request.State()) { + case Request::InitialState: + { + bool hasError = false; + try { + request.ResolveHostName(); + request.OpenConnection(); + } catch (...) { + request.SetError(std::current_exception()); + hasError = true; + } + + if (hasError) { + // Do not add the request back to the queue + break; + } + + // TODO: temporary end of the line here, as data thread not implemented + try { + throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::Canceled); + } catch (...) { + request.SetError(std::current_exception()); + break; + } + impl->fLock.Lock(); + impl->fDataQueue.push_back(std::move(request)); + impl->fLock.Unlock(); + release_sem(impl->fDataQueueSem); + break; + } + default: + { + // not handled at this stage + break; + } + } + } + } + + // Clean up and make sure we are quitting + if (atomic_get(&impl->fQuitting) == 1) { + // First wait for the data thread to complete + status_t threadResult; + wait_for_thread(impl->fDataThread, &threadResult); + // Cancel all requests + for (auto& request: impl->fControlQueue) { + try { + throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::Canceled); + } catch (...) { + request.SetError(std::current_exception()); + } +/* TODO + if (request.observer.IsValid()) { + BMessage msg(UrlEvent::RequestCompleted); + msg.AddInt32(UrlEventData::Id, request.result->id); + msg.AddBool(UrlEventData::Success, false); + request.observer.SendMessage(&msg); + } +*/ + } + } else { + throw BRuntimeError(__PRETTY_FUNCTION__, + "Unknown reason that the controlQueueSem is deleted"); + } + // Cleanup: wait for data thread - status_t threadResult; - wait_for_thread(impl->fDataThread, &threadResult); - return threadResult; + return B_OK; } @@ -112,16 +287,16 @@ BHttpSession::Impl::DataThreadFunc(void* arg) } +// #pragma mark -- BHttpSession (public interface) + + BHttpSession::BHttpSession() { fImpl = std::make_shared(); } -BHttpSession::~BHttpSession() -{ - -} +BHttpSession::~BHttpSession() = default; BHttpSession::BHttpSession(const BHttpSession&) noexcept = default; @@ -129,3 +304,80 @@ BHttpSession::BHttpSession(const BHttpSession&) noexcept = default; BHttpSession& BHttpSession::operator=(const BHttpSession&) noexcept = default; + + +BHttpResult +BHttpSession::Execute(BHttpRequest&& request, std::unique_ptr target, BMessenger observer) +{ + return fImpl->Execute(std::move(request), std::move(target), observer); +} + + +// #pragma mark -- BHttpSession::Request (helpers) + +BHttpSession::Request::Request(BHttpRequest&& request, std::unique_ptr target, + BMessenger observer) + : fRequest(std::move(request)), fObserver(observer) +{ + auto identifier = get_netservices_request_identifier(); + + // create shared data + fResult = std::make_shared(identifier); + fResult->owned_body = std::move(target); +} + + +/*! + \brief Resolve the hostname for a request +*/ +void +BHttpSession::Request::ResolveHostName() +{ + int port; + if (fRequest.Url().HasPort()) + port = fRequest.Url().Port(); + else if (fRequest.Url().Protocol() == "https") + port = 443; + else + port = 80; + + // TODO: proxy + if (auto status = fRemoteAddress.SetTo(fRequest.Url().Host(), port); status != B_OK) { + throw BNetworkRequestError("BNetworkAddress::SetTo()", + BNetworkRequestError::HostnameError, status); + } +} + + +/*! + \brief Open the connection and make the socket non-blocking after opening it +*/ +void +BHttpSession::Request::OpenConnection() +{ + // Set up the socket + if (fRequest.Url().Protocol() == "https") { + // To do: secure socket with callbacks to check certificates + fSocket = std::make_unique(); + } else { + fSocket = std::make_unique(); + } + + // Open connection + if (auto status = fSocket->Connect(fRemoteAddress); status != B_OK) { + // TODO: inform listeners that the connection failed + throw BNetworkRequestError("BSocket::Connect()", + BNetworkRequestError::NetworkError, status); + } + + // Make the rest of the interaction non-blocking + auto flags = fcntl(fSocket->Socket(), F_GETFL, 0); + if (flags == -1) + throw BRuntimeError("fcntl()", "Error getting socket flags"); + if (fcntl(fSocket->Socket(), F_SETFL, flags | O_NONBLOCK) != 0) + throw BRuntimeError("fcntl()", "Error setting non-blocking flag on socket"); + + // TODO: inform the listeners that the connection was opened. + + fRequestStatus = Connected; +} diff --git a/src/kits/network/libnetservices2/Jamfile b/src/kits/network/libnetservices2/Jamfile index 8264adeb52..2824bb589f 100644 --- a/src/kits/network/libnetservices2/Jamfile +++ b/src/kits/network/libnetservices2/Jamfile @@ -1,6 +1,9 @@ SubDir HAIKU_TOP src kits network libnetservices2 ; +UsePrivateHeaders net ; UsePrivateHeaders netservices2 ; +UsePrivateHeaders support ; +UsePrivateHeaders shared ; local architectureObject ; for architectureObject in [ MultiArchSubDirSetup ] { diff --git a/src/kits/network/libnetservices2/NetServicesMisc.cpp b/src/kits/network/libnetservices2/NetServicesMisc.cpp index b023125112..4c37776a07 100644 --- a/src/kits/network/libnetservices2/NetServicesMisc.cpp +++ b/src/kits/network/libnetservices2/NetServicesMisc.cpp @@ -8,7 +8,10 @@ #include -using namespace BPrivate::Network; + +namespace BPrivate { + +namespace Network { // #pragma mark -- BUnsupportedProtocol @@ -147,3 +150,21 @@ BNetworkRequestError::ErrorCode() const noexcept { return fErrorCode; } + + +// #pragma mark -- Private functions and data + + +static int32 gRequestIdentifier = 1; + + +int32 +get_netservices_request_identifier() +{ + return atomic_add(&gRequestIdentifier, 1); +} + + +} // namespace Network + +} // namespace BPrivate diff --git a/src/kits/network/libnetservices2/NetServicesPrivate.h b/src/kits/network/libnetservices2/NetServicesPrivate.h new file mode 100644 index 0000000000..2aefb7da78 --- /dev/null +++ b/src/kits/network/libnetservices2/NetServicesPrivate.h @@ -0,0 +1,25 @@ +/* + * Copyright 2022 Haiku Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Niels Sascha Reedijk, niels.reedijk@gmail.com + */ + +#ifndef _NET_SERVICES_PRIVATE_H_ +#define _NET_SERVICES_PRIVATE_H_ + + +namespace BPrivate { + +namespace Network { + + +int32 get_netservices_request_identifier(); + + +} // namespace Network + +} // namespace BPrivate + +#endif // _NET_SERVICES_PRIVATE_H diff --git a/src/tests/kits/net/netservices2/HttpProtocolTest.cpp b/src/tests/kits/net/netservices2/HttpProtocolTest.cpp index e88cc35ca5..ab1e21407b 100644 --- a/src/tests/kits/net/netservices2/HttpProtocolTest.cpp +++ b/src/tests/kits/net/netservices2/HttpProtocolTest.cpp @@ -14,12 +14,16 @@ #include #include +#include +#include #include using BPrivate::Network::BHttpFields; using BPrivate::Network::BHttpMethod; using BPrivate::Network::BHttpRequest; using BPrivate::Network::BHttpSession; +using BPrivate::Network::BHttpResult; +using BPrivate::Network::BNetworkRequestError; HttpProtocolTest::HttpProtocolTest() @@ -233,6 +237,54 @@ HttpProtocolTest::HttpRequestTest() } +void +HttpProtocolTest::HttpIntegrationTest() +{ + // Test hostname resolution fail + { + auto request = BHttpRequest(BUrl("http://doesnotexist/")); + auto result = fSession.Execute(std::move(request)); + try { + result.Status(); + CPPUNIT_FAIL("Expecting exception when trying to connect to invalid hostname"); + } catch (const BNetworkRequestError& e) { + CPPUNIT_ASSERT_EQUAL(BNetworkRequestError::HostnameError, e.Type()); + } catch (...) { + CPPUNIT_FAIL("Unknown exception raised when getting invalid hostname"); + } + } + + // Test connection error fail + { + // FIXME: find a better way to get an unused local port, instead of hardcoding one + auto request = BHttpRequest(BUrl("http://localhost:59445/")); + auto result = fSession.Execute(std::move(request)); + try { + result.Status(); + CPPUNIT_FAIL("Expecting exception when trying to connect to invalid hostname"); + } catch (const BNetworkRequestError& e) { + CPPUNIT_ASSERT_EQUAL(BNetworkRequestError::NetworkError, e.Type()); + } catch (...) { + CPPUNIT_FAIL("Unknown exception raised when getting invalid hostname"); + } + } + + // Succesful connection (fails as canceled right now) + { + auto request = BHttpRequest(BUrl("https://www.haiku-os.org/")); + auto result = fSession.Execute(std::move(request)); + try { + result.Status(); + CPPUNIT_FAIL("Expecting exception"); + } catch (const BNetworkRequestError& e) { + CPPUNIT_ASSERT_EQUAL(BNetworkRequestError::Canceled, e.Type()); + } catch (...) { + CPPUNIT_FAIL("Unknown exception raised when executing request"); + } + } +} + + /* static */ void HttpProtocolTest::AddTests(BTestSuite& parent) { @@ -244,6 +296,8 @@ HttpProtocolTest::AddTests(BTestSuite& parent) "HttpProtocolTest::HttpMethodTest", &HttpProtocolTest::HttpMethodTest)); suite.addTest(new CppUnit::TestCaller( "HttpProtocolTest::HttpRequestTest", &HttpProtocolTest::HttpRequestTest)); + suite.addTest(new CppUnit::TestCaller( + "HttpProtocolTest::HttpIntegrationTest", &HttpProtocolTest::HttpIntegrationTest)); parent.addTest("HttpProtocolTest", &suite); } diff --git a/src/tests/kits/net/netservices2/HttpProtocolTest.h b/src/tests/kits/net/netservices2/HttpProtocolTest.h index 9ed347c2dd..4b9df5f26f 100644 --- a/src/tests/kits/net/netservices2/HttpProtocolTest.h +++ b/src/tests/kits/net/netservices2/HttpProtocolTest.h @@ -20,6 +20,7 @@ public: void HttpFieldsTest(); void HttpMethodTest(); void HttpRequestTest(); + void HttpIntegrationTest(); static void AddTests(BTestSuite& suite);