NetServices: implement hostname resolution and connection for BHttpRequest

BHttpSession::Execute() moves the request into the session, and returns a future BHttpResponse
object. Currently implemented are resolving the hostname, and opening the connection.

There is some scaffolding for the actual data transfer.

Change-Id: I5a8a7a7f8680036b91cdba4beee140bbed6bfd5a

docs/user/netservices/HttpSession.dox

@@ -13,6 +13,16 @@
 
 #if __cplusplus >= 201703L
 
+
+/*!
+	\file HttpSession.h
+	\ingroup netservices
+	\brief Provides classes and tools to schedule and execute HTTP requests.
+
+	\since Haiku R1
+*/
+
+
 namespace BPrivate {
 
 namespace Network {
@@ -153,6 +163,26 @@ namespace Network {
 */
 
 
+/*!
+	\fn BHttpResult BHttpSession::Execute(BHttpRequest &&request,
+		std::unique_ptr< BDataIO > target=nullptr, BMessenger observer=BMessenger())
+	\brief Schedule and execute a \a request.
+
+	\param request The (valid) request to move from.
+	\param target An optional data buffer to write the incoming body of the request to. This can be
+		\c nullptr if you want to use the default internal storage. If you provide a buffer, it
+		must be wrapped in a \c std::unique_ptr. This means that you transfer ownership to the
+		session. After the request is finished, you can regain ownership.
+	\param observer An optional observer that will receive the progress and status messages for
+		this request.
+
+	\return The \ref BHttpResult object that corresponds to this request, and that can be used to
+		monitor the progress.
+
+	\since Haiku R1
+*/
+
+
 } // namespace Network
 
 } // namespace BPrivate

headers/private/netservices2/HttpRequest.h

@@ -88,8 +88,9 @@ public:
 			void			SetUrl(const BUrl& url);
 
 private:
-	struct Impl;
+	friend class BHttpSession;
-	std::unique_ptr<Impl>	fData;
+	struct Data;
+	std::unique_ptr<Data>	fData;
 };
 
 

headers/private/netservices2/HttpSession.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2021 Haiku Inc. All rights reserved.
+ * Copyright 2022 Haiku Inc. All rights reserved.
  * Distributed under the terms of the MIT License.
  */
 
@@ -8,12 +8,18 @@
 
 #include <memory>
 
+#include <Messenger.h>
+
 class BUrl;
 
+
 namespace BPrivate {
 
 namespace Network {
 
+class BHttpRequest;
+class BHttpResult;
+
 
 class BHttpSession {
 public:
@@ -27,6 +33,12 @@ public:
 	BHttpSession&			operator=(const BHttpSession&) noexcept;
 	BHttpSession&			operator=(BHttpSession&&) noexcept = delete;
 
+
+	// Requests
+	BHttpResult				Execute(BHttpRequest&& request,
+								std::unique_ptr<BDataIO> target = nullptr,
+								BMessenger observer = BMessenger());
+
 private:
 	class Request;
 	class Impl;

src/kits/network/libnetservices2/HttpPrivate.h

@@ -8,6 +8,9 @@
 
 #include <string_view>
 
+#include <HttpRequest.h>
+#include <Url.h>
+
 
 namespace BPrivate {
 

src/kits/network/libnetservices2/HttpRequest.cpp

@@ -130,15 +130,14 @@ BHttpMethod::Method() const noexcept
 }
 
 
-// #pragma mark -- BHttpRequest::Impl
+// #pragma mark -- BHttpRequest::Data
 static const BUrl kDefaultUrl = BUrl();
 static const BHttpMethod kDefaultMethod = BHttpMethod::Get;
 
 
-struct BHttpRequest::Impl {
+struct BHttpRequest::Data {
-	BUrl		url;
+	BUrl		url = kDefaultUrl;
 	BHttpMethod	method	= kDefaultMethod;
-	bool		ssl		= false;
 };
 
 
@@ -146,14 +145,14 @@ struct BHttpRequest::Impl {
 
 
 BHttpRequest::BHttpRequest()
-	: fData(std::make_unique<Impl>())
+	: fData(std::make_unique<Data>())
 {
 
 }
 
 
 BHttpRequest::BHttpRequest(const BUrl& url)
-	: fData(std::make_unique<Impl>())
+	: fData(std::make_unique<Data>())
 {
 	SetUrl(url);
 }
@@ -198,7 +197,7 @@ void
 BHttpRequest::SetMethod(const BHttpMethod& method)
 {
 	if (!fData)
-		fData = std::make_unique<Impl>();
+		fData = std::make_unique<Data>();
 	fData->method = method;
 }
 
@@ -207,15 +206,11 @@ void
 BHttpRequest::SetUrl(const BUrl& url)
 {
 	if (!fData)
-		fData = std::make_unique<Impl>();
+		fData = std::make_unique<Data>();
 
 	if (!url.IsValid())
 		throw BInvalidUrl(__PRETTY_FUNCTION__, BUrl(url));
-	if (url.Protocol() == "http")
+	if (url.Protocol() != "http" && url.Protocol() != "https") {
-		fData->ssl = false;
-	else if (url.Protocol() == "https")
-		fData->ssl = true;
-	else {
 		// TODO: optimize BStringList with modern language features
 		BStringList list;
 		list.Add("http");

src/kits/network/libnetservices2/HttpSession.cpp

@@ -10,15 +10,86 @@
 #include <map>
 #include <vector>
 
+#include <AutoLocker.h>
+#include <DynamicBuffer.h>
 #include <ErrorsExt.h>
+#include <HttpFields.h>
+#include <HttpRequest.h>
+#include <HttpResult.h>
 #include <HttpSession.h>
 #include <Locker.h>
+#include <Messenger.h>
+#include <NetBuffer.h>
+#include <NetServicesDefs.h>
+#include <NetworkAddress.h>
 #include <OS.h>
+#include <SecureSocket.h>
+#include <Socket.h>
+#include <StackOrHeapArray.h>
+#include <ZlibCompressionAlgorithm.h>
+
+#include "HttpResultPrivate.h"
+#include "NetServicesPrivate.h"
 
 using namespace BPrivate::Network;
 
 
 class BHttpSession::Request {
+public:
+									Request(BHttpRequest&& request,
+													std::unique_ptr<BDataIO> target,
+													BMessenger observer);
+
+	// States
+	enum RequestState {
+		InitialState,
+		Connected,
+		StatusReceived,
+		HeadersReceived,
+		ContentReceived,
+		TrailingHeadersReceived
+	};
+	RequestState					State() const { return fRequestStatus; }
+
+	// Result Helpers
+	std::shared_ptr<HttpResultPrivate>
+									Result() { return fResult; }
+	void							SetError(std::exception_ptr e) { fResult->SetError(e); }
+
+	// Operational methods
+	void							ResolveHostName();
+	void							OpenConnection();
+
+	// 
+private:
+	BHttpRequest					fRequest;
+
+	// Request state/events
+	RequestState					fRequestStatus = InitialState;
+
+	// Communication
+	BMessenger						fObserver;
+	std::shared_ptr<HttpResultPrivate> fResult;
+
+	// Connection
+	BNetworkAddress					fRemoteAddress;
+	std::unique_ptr<BSocket>		fSocket;
+
+	// Receive state
+/*	bool							receiveEnd = false;
+	bool							parseEnd = false;
+	BNetBuffer						inputBuffer;
+	size_t							previousBufferSize = 0;
+	off_t							bytesReceived = 0;
+	off_t							bytesTotal = 0;
+	BHttpFields						headers;
+	bool							readByChunks = false;
+	bool							decompress = false;
+	DynamicBuffer					decompressorStorage;
+	std::unique_ptr<BDataIO>		decompressingStream = nullptr;
+	std::vector<char>				inputTempBuffer = std::vector<char>(4096);
+	BHttpStatus						status; */
+	// TODO: reset method to reset Connection and Receive State when redirected
 
 };
 
@@ -28,6 +99,10 @@ public:
 												Impl();
 												~Impl() noexcept;
 
+			BHttpResult							Execute(BHttpRequest&& request,
+													std::unique_ptr<BDataIO> target,
+													BMessenger observer);
+
 private:
 	// Thread functions
 	static	status_t							ControlThreadFunc(void* arg);
@@ -55,6 +130,9 @@ private:
 };
 
 
+// #pragma mark -- BHttpSession::Impl
+
+
 BHttpSession::Impl::Impl()
 	:
 	fControlQueueSem(create_sem(0, "http:control")),
@@ -92,15 +170,112 @@ BHttpSession::Impl::~Impl() noexcept
 }
 
 
+BHttpResult
+BHttpSession::Impl::Execute(BHttpRequest&& request, std::unique_ptr<BDataIO> target,
+	BMessenger observer)
+{
+	auto wRequest = Request(std::move(request), std::move(target), observer);
+
+	auto retval = BHttpResult(wRequest.Result());
+	auto lock = AutoLocker<BLocker>(fLock);
+	fControlQueue.push_back(std::move(wRequest));
+	release_sem(fControlQueueSem);
+	return retval;
+}
+
+
 /*static*/ status_t
 BHttpSession::Impl::ControlThreadFunc(void* arg)
 {
 	BHttpSession::Impl* impl = static_cast<BHttpSession::Impl*>(arg);
 
+	// Outer loop to use the fControlQueueSem when new items have entered the queue
+	while (true) {
+		if (auto status = acquire_sem(impl->fControlQueueSem); status == B_INTERRUPTED)
+			continue;
+		else if (status != B_OK) {
+			// Most likely B_BAD_SEM_ID indicating that the sem was deleted; go to cleanup
+			break;
+		}
+
+		// Inner loop to process items on the queue
+		while (true) {
+			impl->fLock.Lock();
+			if (impl->fControlQueue.empty() || atomic_get(&impl->fQuitting) == 1) {
+				impl->fLock.Unlock();
+				break;
+			}
+			auto request = std::move(impl->fControlQueue.front());
+			impl->fControlQueue.pop_front();
+			impl->fLock.Unlock();
+
+			switch (request.State()) {
+				case Request::InitialState:
+				{
+					bool hasError = false;
+					try {
+						request.ResolveHostName();
+						request.OpenConnection();
+					} catch (...) {
+						request.SetError(std::current_exception());
+						hasError = true;
+					}
+
+					if (hasError) {
+						// Do not add the request back to the queue
+						break;
+					}
+
+					// TODO: temporary end of the line here, as data thread not implemented
+					try {
+						throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::Canceled);
+					} catch (...) {
+						request.SetError(std::current_exception());
+						break;
+					}
+					impl->fLock.Lock();
+					impl->fDataQueue.push_back(std::move(request));
+					impl->fLock.Unlock();
+					release_sem(impl->fDataQueueSem);
+					break;
+				}
+				default:
+				{
+					// not handled at this stage
+					break;
+				}
+			}
+		}
+	}
+
+	// Clean up and make sure we are quitting
+	if (atomic_get(&impl->fQuitting) == 1) {
+		// First wait for the data thread to complete
+		status_t threadResult;
+		wait_for_thread(impl->fDataThread, &threadResult);
+		// Cancel all requests
+		for (auto& request: impl->fControlQueue) {
+			try {
+				throw BNetworkRequestError(__PRETTY_FUNCTION__, BNetworkRequestError::Canceled);
+			} catch (...) {
+				request.SetError(std::current_exception());
+			}
+/*	TODO
+			if (request.observer.IsValid()) {
+				BMessage msg(UrlEvent::RequestCompleted);
+				msg.AddInt32(UrlEventData::Id, request.result->id);
+				msg.AddBool(UrlEventData::Success, false);
+				request.observer.SendMessage(&msg);
+			}
+*/
+		}
+	} else {
+		throw BRuntimeError(__PRETTY_FUNCTION__,
+			"Unknown reason that the controlQueueSem is deleted");
+	}
+
 	// Cleanup: wait for data thread
-	status_t threadResult;
+	return B_OK;
-	wait_for_thread(impl->fDataThread, &threadResult);
-	return threadResult;
 }
 
 
@@ -112,16 +287,16 @@ BHttpSession::Impl::DataThreadFunc(void* arg)
 }
 
 
+// #pragma mark -- BHttpSession (public interface)
+
+
 BHttpSession::BHttpSession()
 {
 	fImpl = std::make_shared<BHttpSession::Impl>();
 }
 
 
-BHttpSession::~BHttpSession()
+BHttpSession::~BHttpSession() = default;
-{
-
-}
 
 
 BHttpSession::BHttpSession(const BHttpSession&) noexcept = default;
@@ -129,3 +304,80 @@ BHttpSession::BHttpSession(const BHttpSession&) noexcept = default;
 
 BHttpSession&
 BHttpSession::operator=(const BHttpSession&) noexcept = default;
+
+
+BHttpResult
+BHttpSession::Execute(BHttpRequest&& request, std::unique_ptr<BDataIO> target, BMessenger observer)
+{
+	return fImpl->Execute(std::move(request), std::move(target), observer);
+}
+
+
+// #pragma mark -- BHttpSession::Request (helpers)
+
+BHttpSession::Request::Request(BHttpRequest&& request, std::unique_ptr<BDataIO> target,
+		BMessenger observer)
+	: fRequest(std::move(request)), fObserver(observer)
+{
+	auto identifier = get_netservices_request_identifier();
+
+	// create shared data
+	fResult = std::make_shared<HttpResultPrivate>(identifier);
+	fResult->owned_body = std::move(target);
+}
+
+
+/*!
+	\brief Resolve the hostname for a request
+*/
+void
+BHttpSession::Request::ResolveHostName()
+{
+	int port;
+	if (fRequest.Url().HasPort())
+		port = fRequest.Url().Port();
+	else if (fRequest.Url().Protocol() == "https")
+		port = 443;
+	else
+		port = 80;
+
+	// TODO: proxy
+	if (auto status = fRemoteAddress.SetTo(fRequest.Url().Host(), port); status != B_OK) {
+		throw BNetworkRequestError("BNetworkAddress::SetTo()",
+			BNetworkRequestError::HostnameError, status);
+	}
+}
+
+
+/*!
+	\brief Open the connection and make the socket non-blocking after opening it
+*/
+void
+BHttpSession::Request::OpenConnection()
+{
+	// Set up the socket
+	if (fRequest.Url().Protocol() == "https") {
+		// To do: secure socket with callbacks to check certificates
+		fSocket = std::make_unique<BSecureSocket>();
+	} else {
+		fSocket = std::make_unique<BSocket>();
+	}
+
+	// Open connection
+	if (auto status = fSocket->Connect(fRemoteAddress); status != B_OK) {
+		// TODO: inform listeners that the connection failed
+		throw BNetworkRequestError("BSocket::Connect()",
+			BNetworkRequestError::NetworkError, status);
+	}
+
+	// Make the rest of the interaction non-blocking
+	auto flags = fcntl(fSocket->Socket(), F_GETFL, 0);
+	if (flags == -1)
+		throw BRuntimeError("fcntl()", "Error getting socket flags");
+	if (fcntl(fSocket->Socket(), F_SETFL, flags | O_NONBLOCK) != 0)
+		throw BRuntimeError("fcntl()", "Error setting non-blocking flag on socket");
+
+	// TODO: inform the listeners that the connection was opened.
+
+	fRequestStatus = Connected;
+}

src/kits/network/libnetservices2/Jamfile

@@ -1,6 +1,9 @@
 SubDir HAIKU_TOP src kits network libnetservices2 ;
 
+UsePrivateHeaders net ;
 UsePrivateHeaders netservices2 ;
+UsePrivateHeaders support ;
+UsePrivateHeaders shared ;
 
 local architectureObject ;
 for architectureObject in [ MultiArchSubDirSetup ] {

src/kits/network/libnetservices2/NetServicesMisc.cpp

@@ -8,7 +8,10 @@
 
 #include <NetServicesDefs.h>
 
-using namespace BPrivate::Network;
+
+namespace BPrivate {
+
+namespace Network {
 
 
 // #pragma mark -- BUnsupportedProtocol
@@ -147,3 +150,21 @@ BNetworkRequestError::ErrorCode() const noexcept
 {
 	return fErrorCode;
 }
+
+
+// #pragma mark -- Private functions and data
+
+
+static int32 gRequestIdentifier = 1;
+
+
+int32
+get_netservices_request_identifier()
+{
+	return atomic_add(&gRequestIdentifier, 1);
+}
+
+
+} // namespace Network
+
+} // namespace BPrivate

src/kits/network/libnetservices2/NetServicesPrivate.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2022 Haiku Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Niels Sascha Reedijk, [email protected]
+ */
+
+#ifndef _NET_SERVICES_PRIVATE_H_
+#define _NET_SERVICES_PRIVATE_H_
+
+
+namespace BPrivate {
+
+namespace Network {
+
+
+int32 get_netservices_request_identifier();
+
+
+} // namespace Network
+
+} // namespace BPrivate
+
+#endif // _NET_SERVICES_PRIVATE_H

src/tests/kits/net/netservices2/HttpProtocolTest.cpp

@@ -14,12 +14,16 @@
 
 #include <HttpFields.h>
 #include <HttpRequest.h>
+#include <HttpResult.h>
+#include <NetServicesDefs.h>
 #include <Url.h>
 
 using BPrivate::Network::BHttpFields;
 using BPrivate::Network::BHttpMethod;
 using BPrivate::Network::BHttpRequest;
 using BPrivate::Network::BHttpSession;
+using BPrivate::Network::BHttpResult;
+using BPrivate::Network::BNetworkRequestError;
 
 
 HttpProtocolTest::HttpProtocolTest()
@@ -233,6 +237,54 @@ HttpProtocolTest::HttpRequestTest()
 }
 
 
+void
+HttpProtocolTest::HttpIntegrationTest()
+{
+	// Test hostname resolution fail
+	{
+		auto request = BHttpRequest(BUrl("http://doesnotexist/"));
+		auto result = fSession.Execute(std::move(request));
+		try {
+			result.Status();
+			CPPUNIT_FAIL("Expecting exception when trying to connect to invalid hostname");
+		} catch (const BNetworkRequestError& e) {
+			CPPUNIT_ASSERT_EQUAL(BNetworkRequestError::HostnameError, e.Type());
+		} catch (...) {
+			CPPUNIT_FAIL("Unknown exception raised when getting invalid hostname");
+		}
+	}
+
+	// Test connection error fail
+	{
+		// FIXME: find a better way to get an unused local port, instead of hardcoding one
+		auto request = BHttpRequest(BUrl("http://localhost:59445/"));
+		auto result = fSession.Execute(std::move(request));
+		try {
+			result.Status();
+			CPPUNIT_FAIL("Expecting exception when trying to connect to invalid hostname");
+		} catch (const BNetworkRequestError& e) {
+			CPPUNIT_ASSERT_EQUAL(BNetworkRequestError::NetworkError, e.Type());
+		} catch (...) {
+			CPPUNIT_FAIL("Unknown exception raised when getting invalid hostname");
+		}
+	}
+
+	// Succesful connection (fails as canceled right now)
+	{
+		auto request = BHttpRequest(BUrl("https://www.haiku-os.org/"));
+		auto result = fSession.Execute(std::move(request));
+		try {
+			result.Status();
+			CPPUNIT_FAIL("Expecting exception");
+		} catch (const BNetworkRequestError& e) {
+			CPPUNIT_ASSERT_EQUAL(BNetworkRequestError::Canceled, e.Type());
+		} catch (...) {
+			CPPUNIT_FAIL("Unknown exception raised when executing request");
+		}
+	}
+}
+
+
 /* static */ void
 HttpProtocolTest::AddTests(BTestSuite& parent)
 {
@@ -244,6 +296,8 @@ HttpProtocolTest::AddTests(BTestSuite& parent)
 		"HttpProtocolTest::HttpMethodTest", &HttpProtocolTest::HttpMethodTest));
 	suite.addTest(new CppUnit::TestCaller<HttpProtocolTest>(
 		"HttpProtocolTest::HttpRequestTest", &HttpProtocolTest::HttpRequestTest));
+	suite.addTest(new CppUnit::TestCaller<HttpProtocolTest>(
+		"HttpProtocolTest::HttpIntegrationTest", &HttpProtocolTest::HttpIntegrationTest));
 
 	parent.addTest("HttpProtocolTest", &suite);
 }

src/tests/kits/net/netservices2/HttpProtocolTest.h

@@ -20,6 +20,7 @@ public:
 			void	HttpFieldsTest();
 			void	HttpMethodTest();
 			void	HttpRequestTest();
+			void	HttpIntegrationTest();
 
 	static	void	AddTests(BTestSuite& suite);