NetServices: implement support for HEAD requests and 204 responses.

These particular responses will not have a body. This is now handled by the
BHttpSession object. There is also a minor fix in here that prevents a crash
when multiple requests are handled by the DataThread at the same time, and not
all of the requests have events.

Change-Id: I7f47d8b3cd8491c8193275be4b3fc1080780fa20

docs/user/netservices/HttpRequest.dox

@@ -42,7 +42,7 @@ namespace Network {
 	use the implicit constructors while interacting with the \ref BHttpRequest class.
 
 	\code
-	auto url = BUrl2("https://www.haiku-os.org/");
+	auto url = BUrl("https://www.haiku-os.org/");
 	// implicitly construct a standard get request
 	auto standard = BHttpRequest(url, BHttpMethod::Get);
 	// implicitly construct a nonstandard patch request
@@ -214,6 +214,19 @@ namespace Network {
 */
 
 
+/*!
+	\fn bool BHttpMethod::operator==(const Verb &other) const noexcept
+	\brief Comparison operator.
+
+	\param other The verb to compare to.
+
+	\retval true This method is equal to \a other.
+	\retval false This method is different from \a other.
+
+	\since Haiku R1
+*/
+
+
 /*!
 	\fn const std::string_view BHttpMethod::Method() const noexcept
 	\brief Get a string representation of the method.

docs/user/netservices/HttpResult.dox

@@ -74,6 +74,40 @@ namespace Network {
 */
 
 
+/*!
+	\struct BHttpBody
+	\ingroup netservices
+	\brief Represents a HTTP response body.
+	
+	The HTTP response body is captured in this object. The body is either stored into a
+	\ref target, or into a \a text variable, depending on how you called the
+	\ref BHttpSession::Execute() method. If there is a \a target, the body will be empty,
+	and vice versa.
+
+	You will usually get a reference to this object through the \ref BHttpResult::Body() method.
+	If you want to keep the contents of the body beyond the lifetime of the BHttpResult object,
+	you should move the data into owned objects of your own.
+
+	\since Haiku R1
+*/
+
+
+/*!
+	\var std::unique_ptr<BDataIO> BHttpBody::target
+	\brief An owned pointer to where the body has been written.
+
+	\since Haiku R1
+*/
+
+
+/*!
+	\var BString BHttpBody::text
+	\brief A string containing the body of the HTTP request.
+
+	\since Haiku R1
+*/
+
+
 /*!
 	\class BHttpResult
 	\ingroup netservices
@@ -211,11 +245,11 @@ namespace Network {
 
 
 /*!
-	\fn bool BPrivate::Network::BHttpResult::HasHeaders() const
+	\fn bool BPrivate::Network::BHttpResult::HasFields() const
-	\brief Check if the headers are available.
+	\brief Check if the header fields are available.
 
-	\retval true The headers of the response is available using the \ref Headers() method.
+	\retval true The header fields of the response is available using the \ref Fields() method.
-	\retval false They are not yet available. Any call to \ref Headers() will block.
+	\retval false They are not yet available. Any call to \ref Fields() will block.
 
 	\exception BRuntimeException This exception is raised when the object has been moved from and
 		is thus no longer valid.
@@ -282,6 +316,46 @@ namespace Network {
 */
 
 
+/*!
+	\fn const BHttpFields& BHttpResult::Fields() const
+	\brief Retrieve the header fields of the HTTP response.
+
+	If the header fields are not yet available, then this method call will block until it is. You
+	can use the \ref HasFields() method to do a non-blocking check if the fields are available.
+
+	\returns A const reference to the \ref BHttpFields object that describes the header fields of
+		the response.
+
+	\exception BRuntimeException This exception is raised when the object has been moved from and
+		is thus no longer valid.
+	\exception BNetworkRequestError This exception is raised when there was an error that prevented
+		completely retrieving and parsing the HTTP response.
+
+	\since Haiku R1
+*/
+
+
+/*!
+	\fn BHttpBody& BHttpResult::Body() const
+	\brief Retrieve the body of the HTTP response.
+
+	If the body is not yet available, then this method call will block until it is. You can
+	use the \ref HasBody() method to do a non-blocking check if the status is available.
+
+	The lifetime of the body is tied to the lifetime of this response result object. If you want to
+	keep the body beyond that time, you can copy or move the data from the \ref BHttpBody object.
+
+	\returns A reference to the \ref BHttpBody object that contains the body.
+
+	\exception BRuntimeException This exception is raised when the object has been moved from and
+		is thus no longer valid.
+	\exception BNetworkRequestError This exception is raised when there was an error that prevented
+		completely retrieving and parsing the HTTP response.
+
+	\since Haiku R1
+*/
+
+
 //! @}
 
 
@@ -290,4 +364,3 @@ namespace Network {
 } // namespace Network
 
 #endif
-

headers/private/netservices2/HttpRequest.h

@@ -59,6 +59,9 @@ public:
 			BHttpMethod&		operator=(const BHttpMethod& other);
 			BHttpMethod&		operator=(BHttpMethod&& other) noexcept;
 
+	// Comparison
+			bool				operator==(const Verb& other) const noexcept;
+
 	// Get the method as a string
 	const	std::string_view	Method() const noexcept;
 

src/kits/network/libnetservices2/HttpRequest.cpp

@@ -100,6 +100,19 @@ BHttpMethod::operator=(BHttpMethod&& other) noexcept
 }
 
 
+bool
+BHttpMethod::operator==(const BHttpMethod::Verb& other) const noexcept
+{
+	if (std::holds_alternative<Verb>(fMethod)) {
+		return std::get<Verb>(fMethod) == other;
+	} else {
+		BHttpMethod otherMethod(other);
+		auto otherMethodSv = otherMethod.Method();
+		return std::get<BString>(fMethod).Compare(otherMethodSv.data(), otherMethodSv.size()) == 0;
+	}
+}
+
+
 const std::string_view
 BHttpMethod::Method() const noexcept
 {

src/kits/network/libnetservices2/HttpResult.cpp

@@ -37,7 +37,7 @@ BHttpResult::~BHttpResult()
 BHttpResult&
 BHttpResult::operator=(BHttpResult&& other) noexcept = default;
 
-#include <iostream>
+
 const BHttpStatus&
 BHttpResult::Status() const
 {
@@ -46,7 +46,6 @@ BHttpResult::Status() const
 	status_t status = B_OK;
 	while (status == B_INTERRUPTED || status == B_OK) {
 		auto dataStatus = fData->GetStatusAtomic();
-		std::cout << "BHttpResult::Status() dataStatus " << dataStatus << std::endl;
 		if (dataStatus == HttpResultPrivate::kError)
 			std::rethrow_exception(*(fData->error));
 

src/kits/network/libnetservices2/HttpSession.cpp

@@ -129,6 +129,7 @@ private:
 	off_t							fBodyBytesTotal = 0;
 	off_t							fBodyBytesReceived = 0;
 	BHttpFields						fFields;
+	bool							fNoContent = false;
 
 	// Optional decompression
 	std::unique_ptr<BMallocIO>		fDecompressorStorage = nullptr;
@@ -478,8 +479,13 @@ BHttpSession::Impl::DataThreadFunc(void* arg)
 				}*/
 				data->connectionMap.erase(item.object);
 				resizeObjectList = true;
+			} else if (item.events == 0) {
+				// No events for this item, skip
+				continue;
 			} else {
 				// Likely to be B_EVENT_INVALID. This should not happen
+				auto& request = data->connectionMap.find(item.object)->second;
+				std::cout << "DataThreadFunc() [" << request.Id() << "] other event " << item.events << std::endl;
 				throw BRuntimeError(__PRETTY_FUNCTION__, "Socket was deleted at an unexpected time");
 			}
 		}
@@ -493,8 +499,10 @@ BHttpSession::Impl::DataThreadFunc(void* arg)
 		auto i = 1;
 		for (auto it = data->connectionMap.cbegin(); it != data->connectionMap.cend(); it++) {
 			data->objectList[i].object = it->first;
-			if (it->second.State() == Request::InitialState)
+			if (it->second.State() == Request::InitialState) {
+				std::cout << "DataThreadFunc() [" << it->second.Id() << "] in Request::InitialState" << std::endl;
 				throw BRuntimeError(__PRETTY_FUNCTION__, "Invalid state of request");
+			}
 			else if (it->second.State() == Request::Connected) {
 				data->objectList[i].events = B_EVENT_WRITE | B_EVENT_DISCONNECTED;
 				std::cout << "DataThreadFunc() [ " << it->second.Id() << "] wait for B_EVENT_WRITE" << std::endl;
@@ -523,6 +531,7 @@ BHttpSession::Impl::DataThreadFunc(void* arg)
 			}*/
 	 	}
 	} else {
+		std::cout << "DataThreadFunc(): Unknown reason that the dataQueueSem is deleted" << std::endl;
 		throw BRuntimeError(__PRETTY_FUNCTION__,
 			"Unknown reason that the dataQueueSem is deleted");
 	}
@@ -577,6 +586,7 @@ BHttpSession::Request::Request(BHttpRequest&& request, std::unique_ptr<BDataIO>
 void
 BHttpSession::Request::ResolveHostName()
 {
+	std::cout << "BHttpSession::Request::ResolveHostName() [" << Id() << "] for URL: " << fRequest.Url().UrlString() << std::endl;
 	int port;
 	if (fRequest.Url().HasPort())
 		port = fRequest.Url().Port();
@@ -721,6 +731,9 @@ BHttpSession::Request::ReceiveResult()
 
 			// TODO: handle the case where we have an error code and we want to stop on error
 
+			if (status.code == 204)
+				fNoContent = true;
+
 			fResult->SetStatus(std::move(status));
 
 			// TODO: inform listeners of receiving the status code
@@ -797,12 +810,17 @@ BHttpSession::Request::ReceiveResult()
 			}
 		}
 
-		// TODO: check if we are head only or if there is no content
-
 		// TODO: move headers to the result and inform listener
 		fResult->SetFields(std::move(fFields));
-
 		fRequestStatus = HeadersReceived;
+
+		if (fRequest.Method() == BHttpMethod::Head || fNoContent) {
+			// HEAD requests and requests with status 204 (No content) are finished
+			std::cout << "ReceiveResult() [" << Id() << "] Request is completing without content" << std::endl;
+			fResult->SetBody();
+			fRequestStatus = ContentReceived;
+			return true;
+		}
 		[[fallthrough]];
 	}
 	case HeadersReceived:

src/tests/kits/net/netservices2/HttpProtocolTest.cpp

@@ -307,7 +307,8 @@ public:
 		ssize_t bytesWritten = (size < 8) ? size : 8;
 		CPPUNIT_ASSERT_MESSAGE("RequestStreamTestIO: bytes written larger than expected output",
 			fExpectedOutput.size() >= (fPos + bytesWritten));
-		CPPUNIT_ASSERT(fExpectedOutput.substr(fPos, bytesWritten) == std::string_view(static_cast<const char*>(buffer), bytesWritten));
+		CPPUNIT_ASSERT(fExpectedOutput.substr(fPos, bytesWritten)
+			== std::string_view(static_cast<const char*>(buffer), bytesWritten));
 		fPos += bytesWritten;
 		return bytesWritten;
 	};
@@ -338,7 +339,8 @@ HttpProtocolTest::HttpRequestStreamTest()
 		expectedTotalBytesWritten = expectedTotalSize;
 	}
 	while (!finished) {
-		auto [currentBytesWritten, totalBytesWritten, totalSize, complete] = requestStream.Transfer(&testIO);
+		auto [currentBytesWritten, totalBytesWritten, totalSize, complete]
+			= requestStream.Transfer(&testIO);
 		CPPUNIT_ASSERT_EQUAL(expectedBytesWritten, currentBytesWritten);
 		CPPUNIT_ASSERT_EQUAL(expectedTotalBytesWritten, totalBytesWritten);
 		CPPUNIT_ASSERT_EQUAL(expectedTotalSize, totalSize);
@@ -409,8 +411,11 @@ HttpIntegrationTest::AddTests(BTestSuite& parent)
 			= new BThreadedTestCaller<HttpIntegrationTest>("HttpTest::", httpIntegrationTest);
 
 		// HTTP
-		testCaller->addThread("HostAndNetworkFailTest", &HttpIntegrationTest::HostAndNetworkFailTest);
+		testCaller->addThread("HostAndNetworkFailTest",
+			&HttpIntegrationTest::HostAndNetworkFailTest);
 		testCaller->addThread("GetTest", &HttpIntegrationTest::GetTest);
+		testCaller->addThread("HeadTest", &HttpIntegrationTest::HeadTest);
+		testCaller->addThread("NoContentTest", &HttpIntegrationTest::NoContentTest);
 
 		suite.addTest(testCaller);
 		parent.addTest("HttpIntegrationTest", &suite);
@@ -425,8 +430,11 @@ HttpIntegrationTest::AddTests(BTestSuite& parent)
 			= new BThreadedTestCaller<HttpIntegrationTest>("HttpsTest::", httpsIntegrationTest);
 
 		// HTTP
-		testCaller->addThread("HostAndNetworkFailTest", &HttpIntegrationTest::HostAndNetworkFailTest);
+		testCaller->addThread("HostAndNetworkFailTest",
+			&HttpIntegrationTest::HostAndNetworkFailTest);
 		testCaller->addThread("GetTest", &HttpIntegrationTest::GetTest);
+		testCaller->addThread("HeadTest", &HttpIntegrationTest::HeadTest);
+		testCaller->addThread("NoContentTest", &HttpIntegrationTest::NoContentTest);
 
 		suite.addTest(testCaller);
 		parent.addTest("HttpsIntegrationTest", &suite);
@@ -466,8 +474,7 @@ HttpIntegrationTest::HostAndNetworkFailTest()
 
 static const BHttpFields kExpectedGetFields = {
 	{"Server"sv, "Test HTTP Server for Haiku"sv},
-	{"Date"sv, "bogus date"sv},
+	{"Date"sv, "Sun, 09 Feb 2020 19:32:42 GMT"sv},
-		// Dynamic content
 	{"Content-Type"sv, "text/plain"sv},
 	{"Content-Length"sv, "110"sv},
 	{"Content-Encoding"sv, "gzip"sv},
@@ -494,12 +501,9 @@ HttpIntegrationTest::GetTest()
 	try {
 		auto receivedFields = result.Fields();
 
-		CPPUNIT_ASSERT_EQUAL_MESSAGE("Mismatch in number of headers", kExpectedGetFields.CountFields(), receivedFields.CountFields());
+		CPPUNIT_ASSERT_EQUAL_MESSAGE("Mismatch in number of headers",
+			kExpectedGetFields.CountFields(), receivedFields.CountFields());
 		for (auto& field: receivedFields) {
-			if (field.Name() == "Date"sv) {
-				// Field with dynamic content; skip
-				continue;
-			}
 			auto expectedField = kExpectedGetFields.FindField(field.Name());
 			if (expectedField == kExpectedGetFields.end())
 				CPPUNIT_FAIL("Could not find expected field in response headers");
@@ -512,3 +516,63 @@ HttpIntegrationTest::GetTest()
 		CPPUNIT_FAIL(e.DebugMessage().String());
 	}
 }
+
+
+void
+HttpIntegrationTest::HeadTest()
+{
+	auto request = BHttpRequest(BUrl(fTestServer.BaseUrl(), "/"));
+	request.SetMethod(BHttpMethod::Head);
+	auto result = fSession.Execute(std::move(request));
+	try {
+		auto receivedFields = result.Fields();
+		CPPUNIT_ASSERT_EQUAL_MESSAGE("Mismatch in number of headers",
+			kExpectedGetFields.CountFields(), receivedFields.CountFields());
+		for (auto& field: receivedFields) {
+			auto expectedField = kExpectedGetFields.FindField(field.Name());
+			if (expectedField == kExpectedGetFields.end())
+				CPPUNIT_FAIL("Could not find expected field in response headers");
+
+			CPPUNIT_ASSERT_EQUAL(field.Value(), (*expectedField).Value());
+		}
+
+		auto receivedBody = result.Body().text;
+		CPPUNIT_ASSERT_EQUAL(receivedBody.Length(), 0);
+	} catch (const BPrivate::Network::BError& e) {
+		CPPUNIT_FAIL(e.DebugMessage().String());
+	}
+}
+
+
+static const BHttpFields kExpectedNoContentFields = {
+	{"Server"sv, "Test HTTP Server for Haiku"sv},
+	{"Date"sv, "Sun, 09 Feb 2020 19:32:42 GMT"sv},
+};
+
+
+void
+HttpIntegrationTest::NoContentTest()
+{
+	auto request = BHttpRequest(BUrl(fTestServer.BaseUrl(), "/204"));
+	auto result = fSession.Execute(std::move(request));
+	try {
+		auto receivedStatus = result.Status();
+		CPPUNIT_ASSERT_EQUAL(204, receivedStatus.code);
+
+		auto receivedFields = result.Fields();
+		CPPUNIT_ASSERT_EQUAL_MESSAGE("Mismatch in number of headers",
+			kExpectedNoContentFields.CountFields(), receivedFields.CountFields());
+		for (auto& field: receivedFields) {
+			auto expectedField = kExpectedNoContentFields.FindField(field.Name());
+			if (expectedField == kExpectedNoContentFields.end())
+				CPPUNIT_FAIL("Could not find expected field in response headers");
+
+			CPPUNIT_ASSERT_EQUAL(field.Value(), (*expectedField).Value());
+		}
+
+		auto receivedBody = result.Body().text;
+		CPPUNIT_ASSERT_EQUAL(receivedBody.Length(), 0);
+	} catch (const BPrivate::Network::BError& e) {
+		CPPUNIT_FAIL(e.DebugMessage().String());
+	}
+}

src/tests/kits/net/netservices2/HttpProtocolTest.h

@@ -39,6 +39,8 @@ public:
 
 			void	HostAndNetworkFailTest();
 			void	GetTest();
+			void	HeadTest();
+			void	NoContentTest();
 
 	static	void	AddTests(BTestSuite& suite);