From 762f26bac81672772c75ad192b53583cf77831d0 Mon Sep 17 00:00:00 2001 From: Kyle Ambroff-Kao Date: Sat, 15 Feb 2020 10:07:12 -0800 Subject: [PATCH] tests/net: Implement testserver.py TLS for HttpsTests This patch is a followup to 0dde5052b which added testserver.py, a HTTP echo server for the HttpTests and HttpsTests in the ServicesKit test suite. This patch implements `testserver.py --use-tls` which allows for re-enabling HttpsTests. If `--use-tls` is used, then a self-signed TLS certificate is generated in a temporary directory which is used by the test server. This option is used when running HttpsTests. There doesn't seem to be a good way to have these tests trust the certificate generated by this test at the moment. Until that API exists I've just made these tests ignore certificate validation. We'll want to resolve this and update these tests to actually verify that validation works as expected. Some minor tweaks had to be made to testserver.py to take care of differences in the response body when serving HTTP and HTTPS requests. Some additional changes: * Don't depend on any files outside of src/tests/kits/net/service for these tests. UploadTest was uploading a file from /boot, but I copied it into the test source directory to avoid having these tests break if someone makes an unrelated change. It doesn't really matter what the contents of this file is as long as it doesn't change. * Use BThreadedTestCase. This speeds up the tests considerably, mostly because it means that the different test cases can share the same HttpTest instance, which means there is only a single TestServer instance, and it takes around half a second to bootstrap the test server on my system, and even longer if --use-tls is used. Change-Id: I6d93d390ebd56115365a85109140d175085e1f01 Reviewed-on: https://review.haiku-os.org/c/haiku/+/2260 Reviewed-by: Adrien Destugues --- src/tests/kits/net/service/HttpTest.cpp | 107 +++++++++++++++-------- src/tests/kits/net/service/HttpTest.h | 3 +- src/tests/kits/net/service/testfile.txt | 21 +++++ src/tests/kits/net/service/testserver.py | 76 ++++++++++++---- 4 files changed, 156 insertions(+), 51 deletions(-) create mode 100644 src/tests/kits/net/service/testfile.txt diff --git a/src/tests/kits/net/service/HttpTest.cpp b/src/tests/kits/net/service/HttpTest.cpp index 76cf4e5b5b..928fa9538c 100644 --- a/src/tests/kits/net/service/HttpTest.cpp +++ b/src/tests/kits/net/service/HttpTest.cpp @@ -13,13 +13,15 @@ #include #include #include +#include #include +#include #include #include #include -#include +#include #include "TestServer.h" @@ -66,6 +68,28 @@ public: } } + + virtual bool CertificateVerificationFailed( + BUrlRequest* caller, + BCertificate& certificate, + const char* message) + { + // TODO: Add tests that exercize this behavior. + // + // At the moment there doesn't seem to be any public API for providing + // an alternate certificate authority, or for constructing a + // BCertificate to be sent to BUrlContext::AddCertificateException(). + // Once we have such a public API then it will be useful to create + // test scenarios that exercize the validation performed by the + // undrelying TLS implementaiton to verify that it is configured + // to do so. + // + // For now we just disable TLS certificate validation entirely because + // we are generating a self-signed TLS certificate for these tests. + return true; + } + + void Verify() { CPPUNIT_ASSERT_EQUAL(fExpectedResponseBody, fActualResponseBody); @@ -101,7 +125,10 @@ void SendAuthenticatedRequest( { TestListener listener(expectedResponseBody, expectedResponseHeaders); - BHttpRequest request(testUrl, false, "HTTP", &listener, &context); + BHttpRequest request(testUrl, testUrl.Protocol() == "https"); + request.SetContext(&context); + request.SetListener(&listener); + request.SetUserName("walter"); request.SetPassword("secret"); @@ -120,6 +147,18 @@ void SendAuthenticatedRequest( listener.Verify(); } + +// Return the path of a file path relative to this source file. +std::string TestFilePath(const std::string& relativePath) +{ + char *testFileSource = strdup(__FILE__); + MemoryDeleter _(testFileSource); + + std::string testSrcDir(::dirname(testFileSource)); + + return testSrcDir + "/" + relativePath; +} + } @@ -168,9 +207,13 @@ HttpTest::GetTest() expectedResponseHeaders["Content-Type"] = "text/plain"; expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT"; expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku"; + TestListener listener(expectedResponseBody, expectedResponseHeaders); - BHttpRequest request(testUrl, false, "HTTP", &listener, context); + BHttpRequest request(testUrl, testUrl.Protocol() == "https"); + request.SetContext(context); + request.SetListener(&listener); + CPPUNIT_ASSERT(request.Run()); while (request.IsRunning()) snooze(1000); @@ -202,7 +245,7 @@ HttpTest::ProxyTest() c->AcquireReference(); c->SetProxy("120.203.214.182", 83); - BHttpRequest t(testUrl); + BHttpRequest t(testUrl, testUrl.Protocol() == "https"); t.SetContext(c); BUrlProtocolListener l; @@ -230,12 +273,14 @@ HttpTest::ProxyTest() void HttpTest::UploadTest() { + std::string testFilePath = TestFilePath("testfile.txt"); + // The test server will echo the POST body back to us in the HTTP response, // so here we load it into memory so that we can compare to make sure that // the server received it. std::string fileContents; { - std::ifstream inputStream("/system/data/licenses/MIT"); + std::ifstream inputStream(testFilePath); CPPUNIT_ASSERT(inputStream.is_open()); fileContents = std::string( std::istreambuf_iterator(inputStream), @@ -254,14 +299,14 @@ HttpTest::UploadTest() "Connection: close\r\n" "User-Agent: Services Kit (Haiku)\r\n" "Content-Type: multipart/form-data; boundary=<>\r\n" - "Content-Length: 1381\r\n" + "Content-Length: 1404\r\n" "\r\n" "Request body:\r\n" "-------------\r\n" "--<>\r\n" "Content-Disposition: form-data; name=\"_uploadfile\";" - " filename=\"MIT\"\r\n" - "Content-Type: text/plain\r\n" + " filename=\"testfile.txt\"\r\n" + "Content-Type: application/octet-stream\r\n" "\r\n" + fileContents + "\r\n" @@ -273,7 +318,7 @@ HttpTest::UploadTest() "\r\n"); HttpHeaderMap expectedResponseHeaders; expectedResponseHeaders["Content-Encoding"] = "gzip"; - expectedResponseHeaders["Content-Length"] = "900"; + expectedResponseHeaders["Content-Length"] = "913"; expectedResponseHeaders["Content-Type"] = "text/plain"; expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT"; expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku"; @@ -282,13 +327,16 @@ HttpTest::UploadTest() BUrl testUrl(fTestServer.BaseUrl(), "/post"); BUrlContext context; - BHttpRequest request(testUrl, false, "HTTP", &listener, &context); + + BHttpRequest request(testUrl, testUrl.Protocol() == "https"); + request.SetContext(&context); + request.SetListener(&listener); BHttpForm form; form.AddString("hello", "world"); CPPUNIT_ASSERT_EQUAL( B_OK, - form.AddFile("_uploadfile", BPath("/system/data/licenses/MIT"))); + form.AddFile("_uploadfile", BPath(testFilePath.c_str()))); request.SetPostFields(form); @@ -303,7 +351,7 @@ HttpTest::UploadTest() dynamic_cast(request.Result()); CPPUNIT_ASSERT_EQUAL(200, result.StatusCode()); CPPUNIT_ASSERT_EQUAL(BString("OK"), result.StatusText()); - CPPUNIT_ASSERT_EQUAL(900, result.Length()); + CPPUNIT_ASSERT_EQUAL(913, result.Length()); listener.Verify(); } @@ -326,12 +374,12 @@ HttpTest::AuthBasicTest() "Accept-Encoding: gzip\r\n" "Connection: close\r\n" "User-Agent: Services Kit (Haiku)\r\n" - "Referer: http://127.0.0.1:PORT/auth/basic/walter/secret\r\n" + "Referer: SCHEME://127.0.0.1:PORT/auth/basic/walter/secret\r\n" "Authorization: Basic d2FsdGVyOnNlY3JldA==\r\n"); HttpHeaderMap expectedResponseHeaders; expectedResponseHeaders["Content-Encoding"] = "gzip"; - expectedResponseHeaders["Content-Length"] = "210"; + expectedResponseHeaders["Content-Length"] = "212"; expectedResponseHeaders["Content-Type"] = "text/plain"; expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT"; expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku"; @@ -362,7 +410,7 @@ HttpTest::AuthDigestTest() "Accept-Encoding: gzip\r\n" "Connection: close\r\n" "User-Agent: Services Kit (Haiku)\r\n" - "Referer: http://127.0.0.1:PORT/auth/digest/walter/secret\r\n" + "Referer: SCHEME://127.0.0.1:PORT/auth/digest/walter/secret\r\n" "Authorization: Digest username=\"walter\"," " realm=\"user@shredder\"," " nonce=\"f3a95f20879dd891a5544bf96a3e5518\"," @@ -377,7 +425,7 @@ HttpTest::AuthDigestTest() HttpHeaderMap expectedResponseHeaders; expectedResponseHeaders["Content-Encoding"] = "gzip"; - expectedResponseHeaders["Content-Length"] = "401"; + expectedResponseHeaders["Content-Length"] = "403"; expectedResponseHeaders["Content-Type"] = "text/plain"; expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT"; expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku"; @@ -409,23 +457,16 @@ HttpTest::AuthDigestTest() /* static */ template void HttpTest::_AddCommonTests(BString prefix, CppUnit::TestSuite& suite) { - BString name; + T* test = new T(); + BThreadedTestCaller* testCaller + = new BThreadedTestCaller(prefix.String(), test); - name = prefix; - name << "GetTest"; - suite.addTest(new CppUnit::TestCaller(name.String(), &T::GetTest)); + testCaller->addThread("GetTest", &T::GetTest); + testCaller->addThread("UploadTest", &T::UploadTest); + testCaller->addThread("BasicAuthTest", &T::AuthBasicTest); + testCaller->addThread("DigestAuthTest", &T::AuthDigestTest); - name = prefix; - name << "UploadTest"; - suite.addTest(new CppUnit::TestCaller(name.String(), &T::UploadTest)); - - name = prefix; - name << "AuthBasicTest"; - suite.addTest(new CppUnit::TestCaller(name.String(), &T::AuthBasicTest)); - - name = prefix; - name << "AuthDigestTest"; - suite.addTest(new CppUnit::TestCaller(name.String(), &T::AuthDigestTest)); + suite.addTest(testCaller); } @@ -446,9 +487,6 @@ HttpTest::AddTests(BTestSuite& parent) parent.addTest("HttpTest", &suite); } - // The HTTPS tests are disabled for now until --use-tls is implemented - // in testserver.py. -#if 0 { CppUnit::TestSuite& suite = *new CppUnit::TestSuite("HttpsTest"); @@ -457,7 +495,6 @@ HttpTest::AddTests(BTestSuite& parent) parent.addTest("HttpsTest", &suite); } -#endif } diff --git a/src/tests/kits/net/service/HttpTest.h b/src/tests/kits/net/service/HttpTest.h index 6fe3ddc521..57a58f6bb4 100644 --- a/src/tests/kits/net/service/HttpTest.h +++ b/src/tests/kits/net/service/HttpTest.h @@ -12,11 +12,12 @@ #include #include +#include #include "TestServer.h" -class HttpTest: public BTestCase { +class HttpTest: public BThreadedTestCase { public: HttpTest(TestServerMode mode = TEST_SERVER_MODE_HTTP); diff --git a/src/tests/kits/net/service/testfile.txt b/src/tests/kits/net/service/testfile.txt new file mode 100644 index 0000000000..70299256f2 --- /dev/null +++ b/src/tests/kits/net/service/testfile.txt @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/src/tests/kits/net/service/testserver.py b/src/tests/kits/net/service/testserver.py index bb21955491..f7a9cb0735 100644 --- a/src/tests/kits/net/service/testserver.py +++ b/src/tests/kits/net/service/testserver.py @@ -26,7 +26,10 @@ import optparse import os import re import socket +import ssl +import subprocess import sys +import tempfile import zlib @@ -148,6 +151,14 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): # server port will be stripped from these headers when # echoed to the response body. header_value = re.sub(r':[0-9]+', ':PORT', header_value) + + # The scheme will also be in this header value, and we want + # to return the same reguardless of whether http:// or + # https:// was used. + header_value = re.sub( + r'https?://', + 'SCHEME://', + header_value) if header == 'Content-Type': match = MULTIPART_FORM_BOUNDARY_RE.match( self.headers.get('Content-Type', 'text/plain')) @@ -341,6 +352,20 @@ def extract_desired_status_code_from_path(path, default=200): return status_code +def generate_self_signed_tls_cert(common_name, cert_path, key_path): + subprocess.check_call([ + 'openssl', + 'req', + '-x509', + '-nodes', + '-subj', '/CN={}'.format(common_name), + '-newkey', 'rsa:4096', + '-keyout', key_path, + '-out', cert_path, + '-days', '1' + ]) + + def compute_digest_challenge_response_hash( request_method, request_uri, @@ -419,33 +444,54 @@ def parse_kv_pair_header(header_value, sep=','): def main(): options = parse_args(sys.argv) - if options.use_tls: - # TODO: Generate a self-signed TLS cert to test HTTPS. - raise NotImplementedError() + bind_addr = ( + options.bind_addr, + 0 if options.port is None else options.port) - bind_addr = (options.bind_addr, options.port) + server = http.server.HTTPServer( + bind_addr, + RequestHandler, + bind_and_activate=False) + if options.port is None: + server.server_port = server.socket.getsockname()[1] + else: + server.server_port = options.port if options.server_socket_fd: - server = http.server.HTTPServer( - bind_addr, - RequestHandler, - bind_and_activate=False) server.socket = socket.fromfd( options.server_socket_fd, socket.AF_INET, socket.SOCK_STREAM) - server.server_port = server.socket.getsockname()[1] - else: - # A socket hasn't been open for us already, so we'll just use - # a random port here. - server = http.server.HTTPServer(bind_addr, RequestHandler) - try: + def run_server(): + if not options.server_socket_fd: + server.server_bind() + server.server_activate() print( 'Test server listening on port', server.server_port, file=sys.stderr) server.serve_forever(0.01) + + try: + if options.use_tls: + with tempfile.TemporaryDirectory() as temp_cert_dir: + common_name = options.bind_addr + ':' + str(options.port) + cert_file = os.path.join(temp_cert_dir, 'cert.pem') + key_file = os.path.join(temp_cert_dir, 'key.pem') + generate_self_signed_tls_cert( + common_name, + cert_file, + key_file) + server.socket = ssl.wrap_socket( + server.socket, + certfile=cert_file, + keyfile=key_file, + server_side=True, + do_handshake_on_connect=False) + run_server() + else: + run_server() except KeyboardInterrupt: server.server_close() @@ -469,7 +515,7 @@ def parse_args(argv): parser.add_option( '--port', dest='port', - default=0, + default=None, type='int', help='If not specified a random port will be used.') parser.add_option(