diff --git a/src/tests/kits/net/service/HttpTest.cpp b/src/tests/kits/net/service/HttpTest.cpp index 0676740f02..76cf4e5b5b 100644 --- a/src/tests/kits/net/service/HttpTest.cpp +++ b/src/tests/kits/net/service/HttpTest.cpp @@ -1,29 +1,131 @@ /* * Copyright 2010, Christophe Huriaux - * Copyright 2014, Haiku, inc. + * Copyright 2014-2020, Haiku, inc. * Distributed under the terms of the MIT licence */ #include "HttpTest.h" - +#include +#include #include #include -#include +#include +#include +#include -#include #include +#include +#include #include - -static const int kHeaderCountInTrivialRequest = 7; - // FIXME This is too strict and not very useful. +#include "TestServer.h" -HttpTest::HttpTest() - : fBaseUrl("http://httpbin.org/") +namespace { + +typedef std::map HttpHeaderMap; + + +class TestListener : public BUrlProtocolListener { +public: + TestListener(const std::string& expectedResponseBody, + const HttpHeaderMap& expectedResponseHeaders) + : + fExpectedResponseBody(expectedResponseBody), + fExpectedResponseHeaders(expectedResponseHeaders) + { + } + + virtual void DataReceived( + BUrlRequest *caller, + const char *data, + off_t position, + ssize_t size) + { + std::copy_n( + data + position, + size, + std::back_inserter(fActualResponseBody)); + } + + virtual void HeadersReceived( + BUrlRequest* caller, + const BUrlResult& result) + { + const BHttpResult& http_result + = dynamic_cast(result); + const BHttpHeaders& headers = http_result.Headers(); + + for (int32 i = 0; i < headers.CountHeaders(); ++i) { + const BHttpHeader& header = headers.HeaderAt(i); + fActualResponseHeaders[std::string(header.Name())] + = std::string(header.Value()); + } + } + + void Verify() + { + CPPUNIT_ASSERT_EQUAL(fExpectedResponseBody, fActualResponseBody); + + for (HttpHeaderMap::iterator iter = fActualResponseHeaders.begin(); + iter != fActualResponseHeaders.end(); + ++iter) + { + CPPUNIT_ASSERT_EQUAL_MESSAGE( + "(header " + iter->first + ")", + fExpectedResponseHeaders[iter->first], + iter->second); + } + CPPUNIT_ASSERT_EQUAL( + fExpectedResponseHeaders.size(), + fActualResponseHeaders.size()); + } + +private: + std::string fExpectedResponseBody; + std::string fActualResponseBody; + + HttpHeaderMap fExpectedResponseHeaders; + HttpHeaderMap fActualResponseHeaders; +}; + + +void SendAuthenticatedRequest( + BUrlContext &context, + BUrl &testUrl, + const std::string& expectedResponseBody, + const HttpHeaderMap &expectedResponseHeaders) +{ + TestListener listener(expectedResponseBody, expectedResponseHeaders); + + BHttpRequest request(testUrl, false, "HTTP", &listener, &context); + request.SetUserName("walter"); + request.SetPassword("secret"); + + CPPUNIT_ASSERT(request.Run()); + + while (request.IsRunning()) + snooze(1000); + + CPPUNIT_ASSERT_EQUAL(B_OK, request.Status()); + + const BHttpResult &result = + dynamic_cast(request.Result()); + CPPUNIT_ASSERT_EQUAL(200, result.StatusCode()); + CPPUNIT_ASSERT_EQUAL(BString("OK"), result.StatusText()); + + listener.Verify(); +} + +} + + +HttpTest::HttpTest(TestServerMode mode) + : + fTestServer(mode) { } @@ -33,41 +135,68 @@ HttpTest::~HttpTest() } +void +HttpTest::setUp() +{ + CPPUNIT_ASSERT_EQUAL_MESSAGE( + "Starting up test server", + B_OK, + fTestServer.StartIfNotRunning()); +} + + void HttpTest::GetTest() { - BUrl testUrl(fBaseUrl, "/user-agent"); - BUrlContext* c = new BUrlContext(); - c->AcquireReference(); - BHttpRequest t(testUrl); + BUrl testUrl(fTestServer.BaseUrl(), "/"); + BUrlContext* context = new BUrlContext(); + context->AcquireReference(); - t.SetContext(c); + std::string expectedResponseBody( + "Path: /\r\n" + "\r\n" + "Headers:\r\n" + "--------\r\n" + "Host: 127.0.0.1:PORT\r\n" + "Accept: */*\r\n" + "Accept-Encoding: gzip\r\n" + "Connection: close\r\n" + "User-Agent: Services Kit (Haiku)\r\n"); + HttpHeaderMap expectedResponseHeaders; + expectedResponseHeaders["Content-Encoding"] = "gzip"; + expectedResponseHeaders["Content-Length"] = "144"; + expectedResponseHeaders["Content-Type"] = "text/plain"; + expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT"; + expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku"; + TestListener listener(expectedResponseBody, expectedResponseHeaders); - CPPUNIT_ASSERT(t.Run()); - - while (t.IsRunning()) + BHttpRequest request(testUrl, false, "HTTP", &listener, context); + CPPUNIT_ASSERT(request.Run()); + while (request.IsRunning()) snooze(1000); - CPPUNIT_ASSERT_EQUAL(B_OK, t.Status()); + CPPUNIT_ASSERT_EQUAL(B_OK, request.Status()); - const BHttpResult& r = dynamic_cast(t.Result()); - CPPUNIT_ASSERT_EQUAL(200, r.StatusCode()); - CPPUNIT_ASSERT_EQUAL(BString("OK"), r.StatusText()); - CPPUNIT_ASSERT_EQUAL(kHeaderCountInTrivialRequest, - r.Headers().CountHeaders()); - CPPUNIT_ASSERT_EQUAL(42, r.Length()); - // Fixed size as we know the response format. - CPPUNIT_ASSERT(!c->GetCookieJar().GetIterator().HasNext()); + const BHttpResult& result + = dynamic_cast(request.Result()); + CPPUNIT_ASSERT_EQUAL(200, result.StatusCode()); + CPPUNIT_ASSERT_EQUAL(BString("OK"), result.StatusText()); + + CPPUNIT_ASSERT_EQUAL(144, result.Length()); + + listener.Verify(); + + CPPUNIT_ASSERT(!context->GetCookieJar().GetIterator().HasNext()); // This page should not set cookies - c->ReleaseReference(); + context->ReleaseReference(); } void HttpTest::ProxyTest() { - BUrl testUrl(fBaseUrl, "/user-agent"); + BUrl testUrl(fTestServer.BaseUrl(), "/user-agent"); BUrlContext* c = new BUrlContext(); c->AcquireReference(); @@ -87,9 +216,6 @@ HttpTest::ProxyTest() CPPUNIT_ASSERT_EQUAL(B_OK, t.Status()); const BHttpResult& r = dynamic_cast(t.Result()); - -printf("%s\n", r.StatusText().String()); - CPPUNIT_ASSERT_EQUAL(200, r.StatusCode()); CPPUNIT_ASSERT_EQUAL(BString("OK"), r.StatusText()); CPPUNIT_ASSERT_EQUAL(42, r.Length()); @@ -101,119 +227,182 @@ printf("%s\n", r.StatusText().String()); } -class PortTestListener: public BUrlProtocolListener -{ -public: - virtual ~PortTestListener() {}; - - void DataReceived(BUrlRequest*, const char* data, off_t, - ssize_t size) - { - fResult.Append(data, size); - } - - BString fResult; -}; - - -void -HttpTest::PortTest() -{ - BUrl testUrl("http://portquiz.net:4242"); - BHttpRequest t(testUrl); - - // portquiz returns more easily parseable results when UA is Wget... - t.SetUserAgent("Wget/1.15 (haiku testsuite)"); - - PortTestListener listener; - t.SetListener(&listener); - - CPPUNIT_ASSERT(t.Run()); - - while (t.IsRunning()) - snooze(1000); - - CPPUNIT_ASSERT_EQUAL(B_OK, t.Status()); - - const BHttpResult& r = dynamic_cast(t.Result()); - CPPUNIT_ASSERT_EQUAL(200, r.StatusCode()); - - CPPUNIT_ASSERT(listener.fResult.StartsWith("Port 4242 test successful!")); -} - - void HttpTest::UploadTest() { - BUrl testUrl(fBaseUrl, "/post"); - BUrlContext c; - BHttpRequest t(testUrl); + // The test server will echo the POST body back to us in the HTTP response, + // so here we load it into memory so that we can compare to make sure that + // the server received it. + std::string fileContents; + { + std::ifstream inputStream("/system/data/licenses/MIT"); + CPPUNIT_ASSERT(inputStream.is_open()); + fileContents = std::string( + std::istreambuf_iterator(inputStream), + std::istreambuf_iterator()); + CPPUNIT_ASSERT(!fileContents.empty()); + } - t.SetContext(&c); + std::string expectedResponseBody( + "Path: /post\r\n" + "\r\n" + "Headers:\r\n" + "--------\r\n" + "Host: 127.0.0.1:PORT\r\n" + "Accept: */*\r\n" + "Accept-Encoding: gzip\r\n" + "Connection: close\r\n" + "User-Agent: Services Kit (Haiku)\r\n" + "Content-Type: multipart/form-data; boundary=<>\r\n" + "Content-Length: 1381\r\n" + "\r\n" + "Request body:\r\n" + "-------------\r\n" + "--<>\r\n" + "Content-Disposition: form-data; name=\"_uploadfile\";" + " filename=\"MIT\"\r\n" + "Content-Type: text/plain\r\n" + "\r\n" + + fileContents + + "\r\n" + "--<>\r\n" + "Content-Disposition: form-data; name=\"hello\"\r\n" + "\r\n" + "world\r\n" + "--<>--\r\n" + "\r\n"); + HttpHeaderMap expectedResponseHeaders; + expectedResponseHeaders["Content-Encoding"] = "gzip"; + expectedResponseHeaders["Content-Length"] = "900"; + expectedResponseHeaders["Content-Type"] = "text/plain"; + expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT"; + expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku"; + TestListener listener(expectedResponseBody, expectedResponseHeaders); - BHttpForm f; - f.AddString("hello", "world"); - CPPUNIT_ASSERT(f.AddFile("_uploadfile", BPath("/system/data/licenses/MIT")) - == B_OK); + BUrl testUrl(fTestServer.BaseUrl(), "/post"); - t.SetPostFields(f); + BUrlContext context; + BHttpRequest request(testUrl, false, "HTTP", &listener, &context); - CPPUNIT_ASSERT(t.Run()); + BHttpForm form; + form.AddString("hello", "world"); + CPPUNIT_ASSERT_EQUAL( + B_OK, + form.AddFile("_uploadfile", BPath("/system/data/licenses/MIT"))); - while (t.IsRunning()) + request.SetPostFields(form); + + CPPUNIT_ASSERT(request.Run()); + + while (request.IsRunning()) snooze(1000); - CPPUNIT_ASSERT_EQUAL(B_OK, t.Status()); + CPPUNIT_ASSERT_EQUAL(B_OK, request.Status()); - const BHttpResult& r = dynamic_cast(t.Result()); - CPPUNIT_ASSERT_EQUAL(200, r.StatusCode()); - CPPUNIT_ASSERT_EQUAL(BString("OK"), r.StatusText()); - CPPUNIT_ASSERT_EQUAL(466, r.Length()); - // Fixed size as we know the response format. + const BHttpResult &result = + dynamic_cast(request.Result()); + CPPUNIT_ASSERT_EQUAL(200, result.StatusCode()); + CPPUNIT_ASSERT_EQUAL(BString("OK"), result.StatusText()); + CPPUNIT_ASSERT_EQUAL(900, result.Length()); + + listener.Verify(); } void HttpTest::AuthBasicTest() { - BUrl testUrl(fBaseUrl, "/basic-auth/walter/secret"); - _AuthTest(testUrl); + BUrlContext context; + + BUrl testUrl(fTestServer.BaseUrl(), "/auth/basic/walter/secret"); + + std::string expectedResponseBody( + "Path: /auth/basic/walter/secret\r\n" + "\r\n" + "Headers:\r\n" + "--------\r\n" + "Host: 127.0.0.1:PORT\r\n" + "Accept: */*\r\n" + "Accept-Encoding: gzip\r\n" + "Connection: close\r\n" + "User-Agent: Services Kit (Haiku)\r\n" + "Referer: http://127.0.0.1:PORT/auth/basic/walter/secret\r\n" + "Authorization: Basic d2FsdGVyOnNlY3JldA==\r\n"); + + HttpHeaderMap expectedResponseHeaders; + expectedResponseHeaders["Content-Encoding"] = "gzip"; + expectedResponseHeaders["Content-Length"] = "210"; + expectedResponseHeaders["Content-Type"] = "text/plain"; + expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT"; + expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku"; + expectedResponseHeaders["Www-Authenticate"] = "Basic realm=\"Fake Realm\""; + + SendAuthenticatedRequest(context, testUrl, expectedResponseBody, + expectedResponseHeaders); + + CPPUNIT_ASSERT(!context.GetCookieJar().GetIterator().HasNext()); + // This page should not set cookies } void HttpTest::AuthDigestTest() { - BUrl testUrl(fBaseUrl, "/digest-auth/auth/walter/secret"); - _AuthTest(testUrl); -} + BUrlContext context; + BUrl testUrl(fTestServer.BaseUrl(), "/auth/digest/walter/secret"); -void -HttpTest::_AuthTest(BUrl& testUrl) -{ - BUrlContext c; - BHttpRequest t(testUrl); + std::string expectedResponseBody( + "Path: /auth/digest/walter/secret\r\n" + "\r\n" + "Headers:\r\n" + "--------\r\n" + "Host: 127.0.0.1:PORT\r\n" + "Accept: */*\r\n" + "Accept-Encoding: gzip\r\n" + "Connection: close\r\n" + "User-Agent: Services Kit (Haiku)\r\n" + "Referer: http://127.0.0.1:PORT/auth/digest/walter/secret\r\n" + "Authorization: Digest username=\"walter\"," + " realm=\"user@shredder\"," + " nonce=\"f3a95f20879dd891a5544bf96a3e5518\"," + " algorithm=MD5," + " opaque=\"f0bb55f1221a51b6d38117c331611799\"," + " uri=\"/auth/digest/walter/secret\"," + " qop=auth," + " cnonce=\"60a3d95d286a732374f0f35fb6d21e79\"," + " nc=00000001," + " response=\"f4264de468aa1a91d81ac40fa73445f3\"\r\n" + "Cookie: stale_after=never; fake=fake_value\r\n"); - t.SetContext(&c); + HttpHeaderMap expectedResponseHeaders; + expectedResponseHeaders["Content-Encoding"] = "gzip"; + expectedResponseHeaders["Content-Length"] = "401"; + expectedResponseHeaders["Content-Type"] = "text/plain"; + expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT"; + expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku"; + expectedResponseHeaders["Set-Cookie"] = "fake=fake_value; Path=/"; + expectedResponseHeaders["Www-Authenticate"] + = "Digest realm=\"user@shredder\", " + "nonce=\"f3a95f20879dd891a5544bf96a3e5518\", " + "qop=\"auth\", " + "opaque=f0bb55f1221a51b6d38117c331611799, " + "algorithm=MD5, " + "stale=FALSE"; - t.SetUserName("walter"); - t.SetPassword("secret"); + SendAuthenticatedRequest(context, testUrl, expectedResponseBody, + expectedResponseHeaders); - CPPUNIT_ASSERT(t.Run()); - - while (t.IsRunning()) - snooze(1000); - - CPPUNIT_ASSERT_EQUAL(B_OK, t.Status()); - - const BHttpResult& r = dynamic_cast(t.Result()); - CPPUNIT_ASSERT_EQUAL(200, r.StatusCode()); - CPPUNIT_ASSERT_EQUAL(BString("OK"), r.StatusText()); - CPPUNIT_ASSERT_EQUAL(kHeaderCountInTrivialRequest, - r.Headers().CountHeaders()); - CPPUNIT_ASSERT_EQUAL(48, r.Length()); - // Fixed size as we know the response format. + std::map cookies; + BNetworkCookieJar::Iterator iter + = context.GetCookieJar().GetIterator(); + while (iter.HasNext()) { + const BNetworkCookie* cookie = iter.Next(); + cookies[cookie->Name()] = cookie->Value(); + } + CPPUNIT_ASSERT_EQUAL(2, cookies.size()); + CPPUNIT_ASSERT_EQUAL(BString("fake_value"), cookies["fake"]); + CPPUNIT_ASSERT_EQUAL(BString("never"), cookies["stale_after"]); } @@ -249,10 +438,6 @@ HttpTest::AddTests(BTestSuite& parent) // HTTP + HTTPs _AddCommonTests("HttpTest::", suite); - // HTTP-only - suite.addTest(new CppUnit::TestCaller( - "HttpTest::PortTest", &HttpTest::PortTest)); - // TODO: reaches out to some mysterious IP 120.203.214.182 which does // not respond anymore? //suite.addTest(new CppUnit::TestCaller("HttpTest::ProxyTest", @@ -261,6 +446,9 @@ HttpTest::AddTests(BTestSuite& parent) parent.addTest("HttpTest", &suite); } + // The HTTPS tests are disabled for now until --use-tls is implemented + // in testserver.py. +#if 0 { CppUnit::TestSuite& suite = *new CppUnit::TestSuite("HttpsTest"); @@ -269,6 +457,7 @@ HttpTest::AddTests(BTestSuite& parent) parent.addTest("HttpsTest", &suite); } +#endif } @@ -276,7 +465,7 @@ HttpTest::AddTests(BTestSuite& parent) HttpsTest::HttpsTest() - : HttpTest() + : + HttpTest(TEST_SERVER_MODE_HTTPS) { - fBaseUrl.SetProtocol("https"); } diff --git a/src/tests/kits/net/service/HttpTest.h b/src/tests/kits/net/service/HttpTest.h index 7401920dad..6fe3ddc521 100644 --- a/src/tests/kits/net/service/HttpTest.h +++ b/src/tests/kits/net/service/HttpTest.h @@ -1,5 +1,5 @@ /* - * Copyright 2014 Haiku, inc. + * Copyright 2014-2020 Haiku, inc. * Distributed under the terms of the MIT License. */ #ifndef HTTP_TEST_H @@ -13,29 +13,30 @@ #include +#include "TestServer.h" + class HttpTest: public BTestCase { public: - HttpTest(); - virtual ~HttpTest(); + HttpTest(TestServerMode mode + = TEST_SERVER_MODE_HTTP); + virtual ~HttpTest(); - void GetTest(); - void PortTest(); - void UploadTest(); - void AuthBasicTest(); - void AuthDigestTest(); - void ProxyTest(); + virtual void setUp(); - static void AddTests(BTestSuite& suite); + void GetTest(); + void UploadTest(); + void AuthBasicTest(); + void AuthDigestTest(); + void ProxyTest(); + + static void AddTests(BTestSuite& suite); private: - void _AuthTest(BUrl& url); + template static void _AddCommonTests(BString prefix, + CppUnit::TestSuite& suite); - template static void _AddCommonTests(BString prefix, - CppUnit::TestSuite& suite); - -protected: - BUrl fBaseUrl; + TestServer fTestServer; }; diff --git a/src/tests/kits/net/service/Jamfile b/src/tests/kits/net/service/Jamfile index 567e717b65..de793aa8ea 100644 --- a/src/tests/kits/net/service/Jamfile +++ b/src/tests/kits/net/service/Jamfile @@ -9,6 +9,7 @@ UnitTestLib servicekittest.so : DataTest.cpp HttpTest.cpp UrlTest.cpp + TestServer.cpp : be $(TARGET_NETWORK_LIBS) $(HAIKU_NETAPI_LIB) [ TargetLibstdc++ ] ; diff --git a/src/tests/kits/net/service/TestServer.cpp b/src/tests/kits/net/service/TestServer.cpp new file mode 100644 index 0000000000..f02277715b --- /dev/null +++ b/src/tests/kits/net/service/TestServer.cpp @@ -0,0 +1,229 @@ +/* + * Copyright 2020 Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Kyle Ambroff-Kao, kyle@ambroffkao.com + */ +#include "TestServer.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace { + +template +std::string to_string(T value) +{ + std::ostringstream s; + s << value; + return s.str(); +} + + +void exec(const std::vector& args) +{ + const char** argv = new const char*[args.size() + 1]; + ArrayDeleter _(argv); + + for (size_t i = 0; i < args.size(); ++i) { + argv[i] = args[i].c_str(); + } + argv[args.size()] = NULL; + + execv(args[0].c_str(), const_cast(argv)); +} + +} + + +TestServer::TestServer(TestServerMode mode) + : + fMode(mode), + fRunning(false), + fChildPid(-1), + fSocketFd(-1), + fServerPort(0) +{ +} + + +TestServer::~TestServer() +{ + if (fChildPid != -1) { + ::kill(fChildPid, SIGTERM); + + pid_t result = -1; + while (result != fChildPid) { + result = ::waitpid(fChildPid, NULL, 0); + } + } + + if (fSocketFd != -1) { + ::close(fSocketFd); + fSocketFd = -1; + } +} + + +// The job of this method is to spawn a child process that will later be killed +// by the destructor. The steps are roughly: +// +// 1. If the child server process is already running, return early +// 2. Choose a random TCP port by binding to the loopback interface. +// 3. Spawn a child Python process to run testserver.py. +// 4. Return immediately allowing the tests to be performed by the caller of +// TestServer::StartIfNotRunning(). We don't have to wait for the child +// process to start up because the socket has already been created. The +// tests will block until accept() is called in the child. +status_t TestServer::StartIfNotRunning() +{ + if (fRunning == true) { + return B_OK; + } + + // Bind to a random unused TCP port. + { + // Create socket with port 0 to get an unused one selected by the + // kernel. + int socket_fd = ::socket(AF_INET, SOCK_STREAM, 0); + if (socket_fd == -1) { + fprintf( + stderr, + "ERROR: Unable to create socket: %s\n", + strerror(errno)); + return B_ERROR; + } + + fSocketFd = socket_fd; + + // We may quickly reclaim the same socket between test runs, so allow + // for reuse. + { + int reuse = 1; + int result = ::setsockopt( + socket_fd, + SOL_SOCKET, + SO_REUSEPORT, + &reuse, + sizeof(reuse)); + if (result == -1) { + fprintf( + stderr, + "ERROR: Unable to set socket options on fd %d: %s\n", + socket_fd, + strerror(errno)); + return B_ERROR; + } + } + + // Bind to loopback 127.0.0.1 + struct sockaddr_in server_address; + server_address.sin_family = AF_INET; + server_address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + int bind_result = ::bind( + socket_fd, + reinterpret_cast(&server_address), + sizeof(server_address)); + if (bind_result == -1) { + fprintf( + stderr, + "ERROR: Unable to bind to loopback interface: %s\n", + strerror(errno)); + return B_ERROR; + } + + // Listen is apparently required before getsockname will work. + if (::listen(socket_fd, 32) == -1) { + fprintf(stderr, "ERROR: listen() failed: %s\n", strerror(errno)); + return B_ERROR; + } + + // Now get the port from the socket. + socklen_t server_address_length = sizeof(server_address); + ::getsockname( + socket_fd, + reinterpret_cast(&server_address), + &server_address_length); + fServerPort = ntohs(server_address.sin_port); + } + + fprintf(stderr, "Binding to port %d for test server\n", fServerPort); + + pid_t child = ::fork(); + if (child < 0) + return B_ERROR; + + if (child > 0) { + // The child process has started. It may take a short amount of time + // before the child process is ready to call accept(), but that's OK. + // + // Since the socket has already been created above, the tests will not + // get ECONNREFUSED and will block until the child process calls + // accept(). So we don't have to busy loop here waiting for a + // connection to the child. + fRunning = true; + fChildPid = child; + return B_OK; + } + + // This is the child process. We can exec the server process. + char* testFileSource = strdup(__FILE__); + MemoryDeleter _(testFileSource); + + std::string testSrcDir(dirname(testFileSource)); + std::string testServerScript = testSrcDir + "/" + "testserver.py"; + + std::string socket_fd_string = to_string(fSocketFd); + std::string server_port_string = to_string(fServerPort); + + std::vector child_process_args; + child_process_args.push_back("/bin/python3"); + child_process_args.push_back(testServerScript); + child_process_args.push_back("--port"); + child_process_args.push_back(server_port_string); + child_process_args.push_back("--fd"); + child_process_args.push_back(socket_fd_string); + + if (fMode == TEST_SERVER_MODE_HTTPS) { + child_process_args.push_back("--use-tls"); + } + + exec(child_process_args); + + // If we reach this point we failed to load the Python image. + fprintf( + stderr, + "Unable to spawn %s: %s\n", + testServerScript.c_str(), + strerror(errno)); + exit(1); +} + + +BUrl TestServer::BaseUrl() const +{ + std::string scheme; + switch(fMode) { + case TEST_SERVER_MODE_HTTP: + scheme = "http://"; + break; + + case TEST_SERVER_MODE_HTTPS: + scheme = "https://"; + break; + } + + std::string baseUrl = scheme + "127.0.0.1:" + to_string(fServerPort) + "/"; + return BUrl(baseUrl.c_str()); +} diff --git a/src/tests/kits/net/service/TestServer.h b/src/tests/kits/net/service/TestServer.h new file mode 100644 index 0000000000..54be0472e9 --- /dev/null +++ b/src/tests/kits/net/service/TestServer.h @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Kyle Ambroff-Kao, kyle@ambroffkao.com + */ +#ifndef TEST_SERVER_H +#define TEST_SERVER_H + +#include +#include + + +enum TestServerMode { + TEST_SERVER_MODE_HTTP, + TEST_SERVER_MODE_HTTPS, +}; + + +class TestServer { +public: + TestServer(TestServerMode mode); + ~TestServer(); + + status_t StartIfNotRunning(); + BUrl BaseUrl() const; + +private: + TestServerMode fMode; + bool fRunning; + pid_t fChildPid; + int fSocketFd; + uint16_t fServerPort; +}; + + +#endif // TEST_SERVER_H diff --git a/src/tests/kits/net/service/testserver.py b/src/tests/kits/net/service/testserver.py new file mode 100644 index 0000000000..bb21955491 --- /dev/null +++ b/src/tests/kits/net/service/testserver.py @@ -0,0 +1,488 @@ +# +# Copyright 2020 Haiku, Inc. All rights reserved. +# Distributed under the terms of the MIT License. +# +# Authors: +# Kyle Ambroff-Kao, kyle@ambroffkao.com +# + +""" +HTTP(S) server used for integration testing of ServicesKit. + +This service receives HTTP requests and just echos them back in the response. + +This is intentionally not using any fancy frameworks or libraries so as to not +require any dependencies, and also to allow for adding endpoints to replicate +behavior of other servers in the future. +""" + +import abc +import base64 +import gzip +import hashlib +import http.server +import io +import optparse +import os +import re +import socket +import sys +import zlib + + +MULTIPART_FORM_BOUNDARY_RE = re.compile( + r'^multipart/form-data; boundary=(----------------------------\d+)$') +AUTH_PATH_RE = re.compile( + r'^/auth/(?P(basic|digest))' + '/(?P[a-z0-9]+)/(?P[a-z0-9]+)', + re.IGNORECASE) + + +class RequestHandler(http.server.BaseHTTPRequestHandler): + """ + Any GET or POST request just gets echoed back to the sender. If the path + ends with a numeric component like "/404" or "/500", then that value will + be set as the status code in the response. + + Note that this isn't meant to replicate expected functionality exactly. + Rather than implementing all of these status codes as expected per RFC, + such as having an empty response body for 201 response, only the + functionality that is required to handle requests from HttpTests is + implemented. + + There can also be endpoints here that are intentionally non-compliant in + order to exercize the HTTP client's behavior when a server is badly + behaved. + """ + def do_GET(self, write_response=True): + authorized, extra_headers = self._authorize() + if not authorized: + return + + encoding, response_body = self._build_response_body() + + self.send_response( + extract_desired_status_code_from_path(self.path, 200)) + self.send_header('Content-Type', 'text/plain') + self.send_header('Content-Length', str(len(response_body))) + if encoding: + self.send_header('Content-Encoding', encoding) + for header_name, header_value in extra_headers: + self.send_header(header_name, header_value) + self.end_headers() + + if write_response: + self.wfile.write(response_body) + + def do_HEAD(self): + self.do_GET(False) + + def do_POST(self): + authorized, extra_headers = self._authorize() + if not authorized: + return + + encoding, response_body = self._build_response_body() + self.send_response( + extract_desired_status_code_from_path(self.path, 200)) + self.send_header('Content-Type', 'text/plain') + self.send_header('Content-Length', str(len(response_body))) + if encoding: + self.send_header('Content-Encoding', encoding) + for header_name, header_value in extra_headers: + self.send_header(header_name, header_value) + + self.end_headers() + self.wfile.write(response_body) + + def do_DELETE(self): + self._not_supported() + + def do_PATCH(self): + self._not_supported() + + def do_OPTIONS(self): + self._not_supported() + + def send_response(self, code, message=None): + self.log_request(code) + self.send_response_only(code, message) + self.send_header('Server', 'Test HTTP Server for Haiku') + self.send_header('Date', 'Sun, 09 Feb 2020 19:32:42 GMT') + + def _build_response_body(self): + # The post-body may be multi-part/form-data, in which case the client + # will have generated some random identifier to identify the boundary. + # If that's the case, we'll replace it here in order to allow the test + # client to validate the response data without needing to predict the + # boundary identifier. This makes the response body deterministic even + # though the boundary will change with every request, and lets the + # tests in HttpTests hard-code the entire expected response body for + # validation. + boundary_id_value = None + + supported_encodings = [ + e.strip() + for e in self.headers.get('Accept-Encoding', '').split(',') + if e.strip()] + if 'gzip' in supported_encodings: + encoding = 'gzip' + output_stream = GzipResponseBodyBuilder() + elif 'deflate' in supported_encodings: + encoding = 'deflate' + output_stream = DeflateResponseBodyBuilder() + else: + encoding = None + output_stream = RawResponseBodyBuilder() + + output_stream.write( + 'Path: {}\r\n\r\n'.format(self.path).encode('utf-8')) + output_stream.write(b'Headers:\r\n') + output_stream.write(b'--------\r\n') + for header in self.headers: + for header_value in self.headers.get_all(header): + if header == 'Host' or header == 'Referer': + # The server port can change between runs which will change + # the size and contents of the response body. To make tests + # that verify the contents of the response body easier the + # server port will be stripped from these headers when + # echoed to the response body. + header_value = re.sub(r':[0-9]+', ':PORT', header_value) + if header == 'Content-Type': + match = MULTIPART_FORM_BOUNDARY_RE.match( + self.headers.get('Content-Type', 'text/plain')) + if match is not None: + boundary_id_value = match.group(1) + header_value = header_value.replace( + boundary_id_value, + '<>') + output_stream.write( + '{}: {}\r\n'.format(header, header_value).encode('utf-8')) + + content_length = int(self.headers.get('Content-Length', 0)) + if content_length > 0: + output_stream.write(b'\r\n') + output_stream.write(b'Request body:\r\n') + output_stream.write(b'-------------\r\n') + + body_bytes = self.rfile.read(content_length).decode('utf-8') + if boundary_id_value: + body_bytes = body_bytes.replace( + boundary_id_value, '<>') + + output_stream.write(body_bytes.encode('utf-8')) + output_stream.write(b'\r\n') + + return encoding, output_stream.get_bytes() + + def _not_supported(self): + self.send_response(405, '{} not supported'.format(self.command)) + self.end_headers() + self.wfile.write( + '{} not supported\r\n'.format(self.command).encode('utf-8')) + + def _authorize(self): + """ + Authorizes the request. If True is returned that means that the + request was not authorized and the 4xx response has been send to the + client. + """ + # We only authorize paths like + # /auth/// + match = AUTH_PATH_RE.match(self.path) + if match is None: + return True, [] + + strategy = match.group('strategy') + expected_username = match.group('username') + expected_password = match.group('password') + + if strategy == 'basic': + return self._handle_basic_auth( + expected_username, + expected_password) + elif strategy == 'digest': + return self._handle_digest_auth( + expected_username, + expected_password) + else: + raise NotImplementedError( + 'Unimplemented authorization strategy ' + strategy) + + def _handle_basic_auth(self, expected_username, expected_password): + authorization = self.headers.get('Authorization', None) + auth_type = None + encoded_credentials = None + username = None + password = None + + if authorization: + auth_type, encoded_credentials = authorization.split() + + if encoded_credentials is not None: + decoded = base64.decodebytes(encoded_credentials.encode('utf-8')) + username, password = decoded.decode('utf-8').split(':') + + if authorization is None or auth_type != 'Basic' \ + or encoded_credentials is None \ + or username != expected_username \ + or password != expected_password: + self.send_response(401, 'Not authorized') + self.send_header('Www-Authenticate', 'Basic realm="Fake Realm"') + self.end_headers() + return False, [] + + return True, [('Www-Authenticate', 'Basic realm="Fake Realm"')] + + def _handle_digest_auth(self, expected_username, expected_password): + """ + Implement enough of the digest auth RFC to make tests pass. + """ + # Note: These values will always be the same because we want the + # response to be deterministic for testing purposes. + NONCE = 'f3a95f20879dd891a5544bf96a3e5518' + OPAQUE = 'f0bb55f1221a51b6d38117c331611799' + + extra_headers = [] + authorization = self.headers.get('Authorization', None) + credentials = None + auth_type = None + if authorization is not None: + auth_type, fields = authorization.split(maxsplit=1) + if auth_type == 'Digest': + credentials = parse_kv_pair_header(fields) + + expected_response_hash = None + if credentials: + expected_response_hash = compute_digest_challenge_response_hash( + self.command, + self.path, + '', + credentials, + expected_password) + + if authorization is None or credentials is None \ + or auth_type != 'Digest' \ + or expected_response_hash != credentials.get('response'): + self.send_response(401, 'Not authorized') + self.send_header( + 'Www-Authenticate', + 'Digest realm="user@shredder",' + ' nonce="{}",' + ' qop="auth",' + ' opaque={},' + ' algorithm=MD5,' + ' stale=FALSE'.format(NONCE, OPAQUE)) + self.send_header('Set-Cookie', 'stale_after=never; Path=/') + self.send_header('Set-Cookie', 'fake=fake_value; Path=/') + self.end_headers() + return False, extra_headers + + return True, extra_headers + + +class ResponseBodyBuilder(object): + __meta__ = abc.ABCMeta + + @abc.abstractmethod + def write(self, bytes): + raise NotImplementedError() + + @abc.abstractmethod + def get_bytes(self): + raise NotImplementedError() + + +class RawResponseBodyBuilder(ResponseBodyBuilder): + def __init__(self): + self.buf = io.BytesIO() + + def write(self, bytes): + self.buf.write(bytes) + + def get_bytes(self): + return self.buf.getvalue() + + +class GzipResponseBodyBuilder(ResponseBodyBuilder): + def __init__(self): + self.buf = io.BytesIO() + self.compressor = gzip.GzipFile( + mode='wb', + compresslevel=4, + fileobj=self.buf) + + def write(self, bytes): + self.compressor.write(bytes) + + def get_bytes(self): + self.compressor.close() + return self.buf.getvalue() + + +class DeflateResponseBodyBuilder(ResponseBodyBuilder): + def __init__(self): + self.raw = RawResponseBodyBuilder() + + def write(self, bytes): + self.raw.write(bytes) + + def get_bytes(self): + return zlib.compress(self.raw.get_bytes()) + + +def extract_desired_status_code_from_path(path, default=200): + status_code = default + path_parts = os.path.split(path) + try: + status_code = int(path_parts[-1]) + except ValueError: + pass + return status_code + + +def compute_digest_challenge_response_hash( + request_method, + request_uri, + request_body, + credentials, + expected_password): + """ + Compute hash as defined by RFC2069, although this isn't an attempt to be + perfect, just enough for basic integration tests in HttpTests to work. + + :param credentials: Map of values parsed from the Authorization header + from the client. + :param expected_password: The known correct password of the user + attempting to authenticate. + :return: None if a hash cannot be produced, otherwise the hash as defined + by RFC2069. + """ + algorithm = credentials.get('algorithm') + if algorithm == 'MD5': + hashfunc = hashlib.md5 + elif algorithm == 'SHA-256': + hashfunc = hashlib.sha256 + elif algorithm == 'SHA-512': + hashfunc = hashlib.sha512 + else: + return None + + realm = credentials.get('realm') + username = credentials.get('username') + + ha1 = hashfunc(':'.join([ + username, + realm, + expected_password]).encode('utf-8')).hexdigest() + + qop = credentials.get('qop') + if qop is None or qop == 'auth': + ha2 = hashfunc(':'.join([ + request_method, + request_uri]).encode('utf-8')).hexdigest() + elif qop == 'auth-int': + ha2 = hashfunc(':'.join([ + request_method, + request_uri, + request_body]).encode('utf-8')).hexdigest() + else: + ha2 = None + + if ha1 is None or ha2 is None: + return None + + if qop is None: + return hashfunc(':'.join([ + ha1, + credentials.get('nonce', ''), + ha2]).encode('utf-8')).hexdigest() + elif qop == 'auth' or qop == 'auth-int': + hash_components = [ + ha1, + credentials.get('nonce', ''), + credentials.get('nc', ''), + credentials.get('cnonce', ''), + qop, + ha2] + return hashfunc(':'.join(hash_components).encode('utf-8')).hexdigest() + + +def parse_kv_pair_header(header_value, sep=','): + d = {} + for kvpair in header_value.split(sep): + key, value = kvpair.strip().split('=') + d[key.strip()] = value.strip().strip('"') + return d + + +def main(): + options = parse_args(sys.argv) + + if options.use_tls: + # TODO: Generate a self-signed TLS cert to test HTTPS. + raise NotImplementedError() + + bind_addr = (options.bind_addr, options.port) + + if options.server_socket_fd: + server = http.server.HTTPServer( + bind_addr, + RequestHandler, + bind_and_activate=False) + server.socket = socket.fromfd( + options.server_socket_fd, + socket.AF_INET, + socket.SOCK_STREAM) + server.server_port = server.socket.getsockname()[1] + else: + # A socket hasn't been open for us already, so we'll just use + # a random port here. + server = http.server.HTTPServer(bind_addr, RequestHandler) + + try: + print( + 'Test server listening on port', + server.server_port, + file=sys.stderr) + server.serve_forever(0.01) + except KeyboardInterrupt: + server.server_close() + + +def parse_args(argv): + parser = optparse.OptionParser( + usage='Usage: %prog [OPTIONS]', + description=__doc__) + parser.add_option( + '--bind-addr', + default='127.0.0.1', + dest='bind_addr', + help='By default only bind to loopback') + parser.add_option( + '--use-tls', + dest='use_tls', + default=False, + action='store_true', + help='If set, a self-signed TLS certificate, key and CA will be' + ' generated for testing purposes.') + parser.add_option( + '--port', + dest='port', + default=0, + type='int', + help='If not specified a random port will be used.') + parser.add_option( + "--fd", + dest='server_socket_fd', + default=None, + type='int', + help='A socket FD to use for accept() instead of binding a new one.') + options, args = parser.parse_args(argv) + if len(args) > 1: + parser.error('Unexpected arguments: {}'.format(', '.join(args[1:]))) + return options + + +if __name__ == '__main__': + main()