tests/net: HTTP proxy client test

With this patch, ProxyTest is implemented and all of the tests in
HttpTest are enabled.

Adding a transparent proxy server implementation proxy.py. Like
testserver.py, this can be provided a socket file descriptor and port
via command-line arguments.

TestServer was refactored to extract ChildProcess and
RandomTCPServerPort, which are now also used by TestProxyServer.

ProxyTest starts TestProxyServer and validates that the request is
sent to the proxy and is routed to the appropriate endpoint of the
downstream server.

The template which adds common tests between HttpTest and HttpsTest
was changed slightly to just take a BThreadedTestCaller<T>&, which
made it simpler to add additional test cases to one suite which are
not appropriate to the other. There wasn't much point in keeping that
template as a member function so I moved it into HttpTest.cpp as a
free function template.

Change-Id: Ied32d6e10bb195d111cae7bbcf0e93168118088b
Reviewed-on: https://review.haiku-os.org/c/haiku/+/2291
Reviewed-by: Adrien Destugues <[email protected]>

src/tests/kits/net/service/HttpTest.cpp

@@ -159,6 +159,16 @@ std::string TestFilePath(const std::string& relativePath)
 	return testSrcDir + "/" + relativePath;
 }
 
+
+template <typename T>
+void AddCommonTests(BThreadedTestCaller<T>& testCaller)
+{
+	testCaller.addThread("GetTest", &T::GetTest);
+	testCaller.addThread("UploadTest", &T::UploadTest);
+	testCaller.addThread("BasicAuthTest", &T::AuthBasicTest);
+	testCaller.addThread("DigestAuthTest", &T::AuthDigestTest);
+}
+
 }
 
 
@@ -180,7 +190,7 @@ HttpTest::setUp()
 	CPPUNIT_ASSERT_EQUAL_MESSAGE(
 		"Starting up test server",
 		B_OK,
-		fTestServer.StartIfNotRunning());
+		fTestServer.Start());
 }
 
 
@@ -239,34 +249,62 @@ HttpTest::GetTest()
 void
 HttpTest::ProxyTest()
 {
-	BUrl testUrl(fTestServer.BaseUrl(), "/user-agent");
+	BUrl testUrl(fTestServer.BaseUrl(), "/");
 
-	BUrlContext* c = new BUrlContext();
-	c->AcquireReference();
-	c->SetProxy("120.203.214.182", 83);
+	TestProxyServer proxy;
+	CPPUNIT_ASSERT_EQUAL_MESSAGE(
+		"Test proxy server startup",
+		B_OK,
+		proxy.Start());
 
-	BHttpRequest t(testUrl, testUrl.Protocol() == "https");
-	t.SetContext(c);
+	BUrlContext* context = new BUrlContext();
+	context->AcquireReference();
+	context->SetProxy("127.0.0.1", proxy.Port());
 
-	BUrlProtocolListener l;
-	t.SetListener(&l);
+	std::string expectedResponseBody(
+		"Path: /\r\n"
+		"\r\n"
+		"Headers:\r\n"
+		"--------\r\n"
+		"Host: 127.0.0.1:PORT\r\n"
+		"Content-Length: 0\r\n"
+		"Accept: */*\r\n"
+		"Accept-Encoding: gzip\r\n"
+		"Connection: close\r\n"
+		"User-Agent: Services Kit (Haiku)\r\n"
+		"X-Forwarded-For: 127.0.0.1:PORT\r\n");
+	HttpHeaderMap expectedResponseHeaders;
+	expectedResponseHeaders["Content-Encoding"] = "gzip";
+	expectedResponseHeaders["Content-Length"] = "169";
+	expectedResponseHeaders["Content-Type"] = "text/plain";
+	expectedResponseHeaders["Date"] = "Sun, 09 Feb 2020 19:32:42 GMT";
+	expectedResponseHeaders["Server"] = "Test HTTP Server for Haiku";
 
-	CPPUNIT_ASSERT(t.Run());
+	TestListener listener(expectedResponseBody, expectedResponseHeaders);
 
-	while (t.IsRunning())
+	BHttpRequest request(testUrl);
+	request.SetContext(context);
+	request.SetListener(&listener);
+
+	CPPUNIT_ASSERT(request.Run());
+
+	while (request.IsRunning())
 		snooze(1000);
 
-	CPPUNIT_ASSERT_EQUAL(B_OK, t.Status());
+	CPPUNIT_ASSERT_EQUAL(B_OK, request.Status());
 
-	const BHttpResult& r = dynamic_cast<const BHttpResult&>(t.Result());
-	CPPUNIT_ASSERT_EQUAL(200, r.StatusCode());
-	CPPUNIT_ASSERT_EQUAL(BString("OK"), r.StatusText());
-	CPPUNIT_ASSERT_EQUAL(42, r.Length());
+	const BHttpResult& response
+		= dynamic_cast<const BHttpResult&>(request.Result());
+	CPPUNIT_ASSERT_EQUAL(200, response.StatusCode());
+	CPPUNIT_ASSERT_EQUAL(BString("OK"), response.StatusText());
+	CPPUNIT_ASSERT_EQUAL(169, response.Length());
 		// Fixed size as we know the response format.
-	CPPUNIT_ASSERT(!c->GetCookieJar().GetIterator().HasNext());
+	CPPUNIT_ASSERT(!context->GetCookieJar().GetIterator().HasNext());
 		// This page should not set cookies
 
-	c->ReleaseReference();
+	listener.Verify();
+
+	context->ReleaseReference();
 }
 
 
@@ -454,45 +492,36 @@ HttpTest::AuthDigestTest()
 }
 
 
-/* static */ template<class T> void
-HttpTest::_AddCommonTests(BString prefix, CppUnit::TestSuite& suite)
-{
-	T* test = new T();
-	BThreadedTestCaller<T>* testCaller
-		= new BThreadedTestCaller<T>(prefix.String(), test);
-
-	testCaller->addThread("GetTest", &T::GetTest);
-	testCaller->addThread("UploadTest", &T::UploadTest);
-	testCaller->addThread("BasicAuthTest", &T::AuthBasicTest);
-	testCaller->addThread("DigestAuthTest", &T::AuthDigestTest);
-
-	suite.addTest(testCaller);
-}
-
-
 /* static */ void
 HttpTest::AddTests(BTestSuite& parent)
 {
 	{
 		CppUnit::TestSuite& suite = *new CppUnit::TestSuite("HttpTest");
 
+		HttpTest* httpTest = new HttpTest();
+		BThreadedTestCaller<HttpTest>* httpTestCaller
+			= new BThreadedTestCaller<HttpTest>("HttpTest::", httpTest);
+
 		// HTTP + HTTPs
-		_AddCommonTests<HttpTest>("HttpTest::", suite);
+		AddCommonTests<HttpTest>(*httpTestCaller);
 
-		// TODO: reaches out to some mysterious IP 120.203.214.182 which does
-		// not respond anymore?
-		//suite.addTest(new CppUnit::TestCaller<HttpTest>("HttpTest::ProxyTest",
-		//	&HttpTest::ProxyTest));
+		httpTestCaller->addThread("ProxyTest", &HttpTest::ProxyTest);
 
+		suite.addTest(httpTestCaller);
 		parent.addTest("HttpTest", &suite);
 	}
 
 	{
 		CppUnit::TestSuite& suite = *new CppUnit::TestSuite("HttpsTest");
 
-		// HTTP + HTTPs
-		_AddCommonTests<HttpsTest>("HttpsTest::", suite);
+		HttpsTest* httpsTest = new HttpsTest();
+		BThreadedTestCaller<HttpsTest>* httpsTestCaller
+			= new BThreadedTestCaller<HttpsTest>("HttpsTest::", httpsTest);
 
+		// HTTP + HTTPs
+		AddCommonTests<HttpsTest>(*httpsTestCaller);
+
+		suite.addTest(httpsTestCaller);
 		parent.addTest("HttpsTest", &suite);
 	}
 }

src/tests/kits/net/service/HttpTest.h

@@ -19,31 +19,27 @@
 
 class HttpTest: public BThreadedTestCase {
 public:
-											HttpTest(TestServerMode mode
-												= TEST_SERVER_MODE_HTTP);
-	virtual									~HttpTest();
+						HttpTest(TestServerMode mode = TEST_SERVER_MODE_HTTP);
+	virtual				~HttpTest();
 
-	virtual						void		setUp();
+	virtual	void		setUp();
 
-								void		GetTest();
-								void		UploadTest();
-								void		AuthBasicTest();
-								void		AuthDigestTest();
-								void		ProxyTest();
+			void		GetTest();
+			void		UploadTest();
+			void		AuthBasicTest();
+			void		AuthDigestTest();
+			void		ProxyTest();
 
-	static						void		AddTests(BTestSuite& suite);
+	static	void		AddTests(BTestSuite& suite);
 
 private:
-	template<class T> static	void		_AddCommonTests(BString prefix,
-												CppUnit::TestSuite& suite);
-
-								TestServer	fTestServer;
+			TestServer	fTestServer;
 };
 
 
 class HttpsTest: public HttpTest {
 public:
-								HttpsTest();
+						HttpsTest();
 };
 
 

src/tests/kits/net/service/TestServer.cpp

@@ -44,21 +44,135 @@ void exec(const std::vector<std::string>& args)
 	execv(args[0].c_str(), const_cast<char* const*>(argv));
 }
 
+
+// Return the path of a file path relative to this source file.
+std::string TestFilePath(const std::string& relativePath)
+{
+	char *testFileSource = strdup(__FILE__);
+	MemoryDeleter _(testFileSource);
+
+	std::string testSrcDir(::dirname(testFileSource));
+
+	return testSrcDir + "/" + relativePath;
+}
+
 }
 
 
-TestServer::TestServer(TestServerMode mode)
+RandomTCPServerPort::RandomTCPServerPort()
 	:
-	fMode(mode),
-	fRunning(false),
-	fChildPid(-1),
+	fInitStatus(B_NOT_INITIALIZED),
 	fSocketFd(-1),
 	fServerPort(0)
+{
+	// Create socket with port 0 to get an unused one selected by the
+	// kernel.
+	int socket_fd = ::socket(AF_INET, SOCK_STREAM, 0);
+	if (socket_fd == -1) {
+		fprintf(
+			stderr,
+			"ERROR: Unable to create socket: %s\n",
+			strerror(errno));
+		fInitStatus = B_ERROR;
+		return;
+	}
+
+	fSocketFd = socket_fd;
+
+	// We may quickly reclaim the same socket between test runs, so allow
+	// for reuse.
+	{
+		int reuse = 1;
+		int result = ::setsockopt(
+			socket_fd,
+			SOL_SOCKET,
+			SO_REUSEPORT,
+			&reuse,
+			sizeof(reuse));
+		if (result == -1) {
+			fInitStatus = errno;
+			fprintf(
+				stderr,
+				"ERROR: Unable to set socket options on fd %d: %s\n",
+				socket_fd,
+				strerror(fInitStatus));
+			return;
+		}
+	}
+
+	// Bind to loopback 127.0.0.1
+	struct sockaddr_in server_address;
+	server_address.sin_family = AF_INET;
+	server_address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+	int bind_result = ::bind(
+		socket_fd,
+		reinterpret_cast<struct sockaddr*>(&server_address),
+		sizeof(server_address));
+	if (bind_result == -1) {
+		fInitStatus = errno;
+		fprintf(
+			stderr,
+			"ERROR: Unable to bind to loopback interface: %s\n",
+			strerror(fInitStatus));
+		return;
+	}
+
+	// Listen is apparently required before getsockname will work.
+	if (::listen(socket_fd, 32) == -1) {
+		fInitStatus = errno;
+		fprintf(stderr, "ERROR: listen() failed: %s\n", strerror(fInitStatus));
+
+		return;
+	}
+
+	// Now get the port from the socket.
+	socklen_t server_address_length = sizeof(server_address);
+	::getsockname(
+		socket_fd,
+		reinterpret_cast<struct sockaddr*>(&server_address),
+		&server_address_length);
+	fServerPort = ntohs(server_address.sin_port);
+
+	fInitStatus = B_OK;
+}
+
+
+RandomTCPServerPort::~RandomTCPServerPort()
+{
+	if (fSocketFd != -1) {
+		::close(fSocketFd);
+		fSocketFd = -1;
+		fInitStatus = B_NOT_INITIALIZED;
+	}
+}
+
+
+status_t RandomTCPServerPort::InitCheck() const
+{
+	return fInitStatus;
+}
+
+
+int RandomTCPServerPort::FileDescriptor() const
+{
+	return fSocketFd;
+}
+
+
+uint16_t RandomTCPServerPort::Port() const
+{
+	return fServerPort;
+}
+
+
+ChildProcess::ChildProcess()
+	:
+	fChildPid(-1)
 {
 }
 
 
-TestServer::~TestServer()
+ChildProcess::~ChildProcess()
 {
 	if (fChildPid != -1) {
 		::kill(fChildPid, SIGTERM);
@@ -68,146 +182,83 @@ TestServer::~TestServer()
 			result = ::waitpid(fChildPid, NULL, 0);
 		}
 	}
-
-	if (fSocketFd != -1) {
-		::close(fSocketFd);
-		fSocketFd = -1;
-	}
 }
 
 
 // The job of this method is to spawn a child process that will later be killed
-// by the destructor. The steps are roughly:
-//
-// 1. If the child server process is already running, return early
-// 2. Choose a random TCP port by binding to the loopback interface.
-// 3. Spawn a child Python process to run testserver.py.
-// 4. Return immediately allowing the tests to be performed by the caller of
-//    TestServer::StartIfNotRunning(). We don't have to wait for the child
-//    process to start up because the socket has already been created. The
-//    tests will block until accept() is called in the child.
-status_t TestServer::StartIfNotRunning()
+// by the destructor.
+status_t ChildProcess::Start(const std::vector<std::string>& args)
 {
-	if (fRunning == true) {
-		return B_OK;
+	if (fChildPid != -1) {
+		return B_ALREADY_RUNNING;
 	}
 
-	// Bind to a random unused TCP port.
-	{
-		// Create socket with port 0 to get an unused one selected by the
-		// kernel.
-		int socket_fd = ::socket(AF_INET, SOCK_STREAM, 0);
-		if (socket_fd == -1) {
-			fprintf(
-				stderr,
-				"ERROR: Unable to create socket: %s\n",
-				strerror(errno));
-			return B_ERROR;
-		}
-
-		fSocketFd = socket_fd;
-
-		// We may quickly reclaim the same socket between test runs, so allow
-		// for reuse.
-		{
-			int reuse = 1;
-			int result = ::setsockopt(
-				socket_fd,
-				SOL_SOCKET,
-				SO_REUSEPORT,
-				&reuse,
-				sizeof(reuse));
-			if (result == -1) {
-				fprintf(
-					stderr,
-					"ERROR: Unable to set socket options on fd %d: %s\n",
-					socket_fd,
-					strerror(errno));
-				return B_ERROR;
-			}
-		}
-
-		// Bind to loopback 127.0.0.1
-		struct sockaddr_in server_address;
-		server_address.sin_family = AF_INET;
-		server_address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
-		int bind_result = ::bind(
-			socket_fd,
-			reinterpret_cast<struct sockaddr*>(&server_address),
-			sizeof(server_address));
-		if (bind_result == -1) {
-			fprintf(
-				stderr,
-				"ERROR: Unable to bind to loopback interface: %s\n",
-				strerror(errno));
-			return B_ERROR;
-		}
-
-		// Listen is apparently required before getsockname will work.
-		if (::listen(socket_fd, 32) == -1) {
-			fprintf(stderr, "ERROR: listen() failed: %s\n", strerror(errno));
-			return B_ERROR;
-		}
-
-		// Now get the port from the socket.
-		socklen_t server_address_length = sizeof(server_address);
-		::getsockname(
-			socket_fd,
-			reinterpret_cast<struct sockaddr*>(&server_address),
-			&server_address_length);
-		fServerPort = ntohs(server_address.sin_port);
-	}
-
-	fprintf(stderr, "Binding to port %d for test server\n", fServerPort);
-
 	pid_t child = ::fork();
 	if (child < 0)
 		return B_ERROR;
 
 	if (child > 0) {
-		// The child process has started. It may take a short amount of time
-		// before the child process is ready to call accept(), but that's OK.
-		//
-		// Since the socket has already been created above, the tests will not
-		// get ECONNREFUSED and will block until the child process calls
-		// accept(). So we don't have to busy loop here waiting for a
-		// connection to the child.
-		fRunning = true;
 		fChildPid = child;
 		return B_OK;
 	}
 
+	// This is the child process. We can exec image provided in args.
+	exec(args);
+
+	// If we reach this point we failed to load the Python image.
+	std::ostringstream ostr;
+
+	for (std::vector<std::string>::const_iterator iter = args.cbegin();
+		 iter != args.end();
+		 ++iter) {
+		ostr << " " << *iter;
+	}
+
+	fprintf(
+		stderr,
+		"Unable to spawn `%s': %s\n",
+		ostr.str().c_str(),
+		strerror(errno));
+	exit(1);
+}
+
+
+TestServer::TestServer(TestServerMode mode)
+	:
+	fMode(mode)
+{
+}
+
+
+// Start a child testserver.py process with the random TCP port chosen by
+// fPort.
+status_t TestServer::Start()
+{
+	if (fPort.InitCheck() != B_OK) {
+		return fPort.InitCheck();
+	}
+
 	// This is the child process. We can exec the server process.
-	char* testFileSource = strdup(__FILE__);
-	MemoryDeleter _(testFileSource);
-
-	std::string testSrcDir(dirname(testFileSource));
-	std::string testServerScript = testSrcDir + "/" + "testserver.py";
-
-	std::string socket_fd_string = to_string(fSocketFd);
-	std::string server_port_string = to_string(fServerPort);
-
 	std::vector<std::string> child_process_args;
 	child_process_args.push_back("/bin/python3");
-	child_process_args.push_back(testServerScript);
+	child_process_args.push_back(TestFilePath("testserver.py"));
 	child_process_args.push_back("--port");
-	child_process_args.push_back(server_port_string);
+	child_process_args.push_back(to_string(fPort.Port()));
 	child_process_args.push_back("--fd");
-	child_process_args.push_back(socket_fd_string);
+	child_process_args.push_back(to_string(fPort.FileDescriptor()));
 
 	if (fMode == TEST_SERVER_MODE_HTTPS) {
 		child_process_args.push_back("--use-tls");
 	}
 
-	exec(child_process_args);
-
-	// If we reach this point we failed to load the Python image.
-	fprintf(
-		stderr,
-		"Unable to spawn %s: %s\n",
-		testServerScript.c_str(),
-		strerror(errno));
-	exit(1);
+	// After this the child process has started. It may take a short amount of
+	// time before the child process is ready to call accept(), but that's OK.
+	//
+	// Since the socket has already been created above, the tests will not
+	// get ECONNREFUSED and will block until the child process calls
+	// accept(). So we don't have to busy loop here waiting for a
+	// connection to the child.
+	return fChildProcess.Start(child_process_args);
 }
 
 
@@ -224,6 +275,40 @@ BUrl TestServer::BaseUrl() const
 		break;
 	}
 
-	std::string baseUrl = scheme + "127.0.0.1:" + to_string(fServerPort) + "/";
+	std::string port_string = to_string(fPort.Port());
+
+	std::string baseUrl = scheme + "127.0.0.1:" + port_string + "/";
 	return BUrl(baseUrl.c_str());
 }
+
+
+// Start a child proxy.py process using the random TCP port chosen by fPort.
+status_t TestProxyServer::Start()
+{
+	if (fPort.InitCheck() != B_OK) {
+		return fPort.InitCheck();
+	}
+
+	std::vector<std::string> child_process_args;
+	child_process_args.push_back("/bin/python3");
+	child_process_args.push_back(TestFilePath("proxy.py"));
+	child_process_args.push_back("--port");
+	child_process_args.push_back(to_string(fPort.Port()));
+	child_process_args.push_back("--fd");
+	child_process_args.push_back(to_string(fPort.FileDescriptor()));
+
+	// After this the child process has started. It may take a short amount of
+	// time before the child process is ready to call accept(), but that's OK.
+	//
+	// Since the socket has already been created above, the tests will not
+	// get ECONNREFUSED and will block until the child process calls
+	// accept(). So we don't have to busy loop here waiting for a
+	// connection to the child.
+	return fChildProcess.Start(child_process_args);
+}
+
+
+uint16_t TestProxyServer::Port() const
+{
+	return fPort.Port();
+}

src/tests/kits/net/service/TestServer.h

@@ -8,10 +8,41 @@
 #ifndef TEST_SERVER_H
 #define TEST_SERVER_H
 
+#include <string>
+#include <vector>
+
 #include <os/support/SupportDefs.h>
 #include <os/support/Url.h>
 
 
+// Binds to a random unused TCP port.
+class RandomTCPServerPort {
+public:
+						RandomTCPServerPort();
+						~RandomTCPServerPort();
+
+	status_t			InitCheck()							const;
+	int					FileDescriptor()					const;
+	uint16_t			Port()								const;
+
+private:
+	status_t			fInitStatus;
+	int					fSocketFd;
+	uint16_t			fServerPort;
+};
+
+
+class ChildProcess {
+public:
+						ChildProcess();
+						~ChildProcess();
+
+	status_t			Start(const std::vector<std::string>& args);
+private:
+	pid_t				fChildPid;
+};
+
+
 enum TestServerMode {
 	TEST_SERVER_MODE_HTTP,
 	TEST_SERVER_MODE_HTTPS,
@@ -20,18 +51,26 @@ enum TestServerMode {
 
 class TestServer {
 public:
-	TestServer(TestServerMode mode);
-	~TestServer();
+						TestServer(TestServerMode mode);
 
-	status_t	StartIfNotRunning();
-	BUrl		BaseUrl()	const;
+	status_t			Start();
+	BUrl				BaseUrl()							const;
 
 private:
-	TestServerMode	fMode;
-	bool			fRunning;
-	pid_t			fChildPid;
-	int				fSocketFd;
-	uint16_t		fServerPort;
+	TestServerMode		fMode;
+	ChildProcess		fChildProcess;
+	RandomTCPServerPort fPort;
+};
+
+
+class TestProxyServer {
+public:
+	status_t			Start();
+	uint16_t			Port()								const;
+
+private:
+	ChildProcess		fChildProcess;
+	RandomTCPServerPort	fPort;
 };
 
 

src/tests/kits/net/service/proxy.py

@@ -0,0 +1,201 @@
+#
+# Copyright 2020 Haiku, Inc. All rights reserved.
+# Distributed under the terms of the MIT License.
+#
+# Authors:
+#  Kyle Ambroff-Kao, [email protected]
+#
+
+"""
+Transparent HTTP proxy.
+"""
+
+import http.client
+import http.server
+import optparse
+import socket
+import sys
+import urllib.parse
+
+
+class RequestHandler(http.server.BaseHTTPRequestHandler):
+    """
+    Implement the basic requirements for a transparent HTTP proxy as defined
+    by RFC 7230. Enough of the functionality is implemented to support the
+    integration tests in HttpTest that use the HTTP proxy feature.
+
+    There are many error conditions and failure modes which are not handled.
+    Those cases can be added as the test suite expands to handle more error
+    cases.
+    """
+    def __init__(self, *args, **kwargs):
+        # This is used to hold on to persistent connections to the downstream
+        # servers. This maps downstream_host:port => HTTPConnection
+        #
+        # This implementation is not thread safe, but that's OK we only have
+        # a single thread anyway.
+        self._connections = {}
+
+        super(RequestHandler, self).__init__(*args, **kwargs)
+
+    def _proxy_request(self):
+        # Extract the downstream server from the request path.
+        #
+        # Note that no attempt is made to prevent message forwarding loops
+        # here. This doesn't need to be a complete proxy implementation, just
+        # enough of one for integration tests. RFC 7230 section 5.7 says if
+        # this were a complete implementation, it would have to make sure that
+        # the target system was not this process to avoid a loop.
+        target = urllib.parse.urlparse(self.path)
+
+        # If Connection: close wasn't used, then we may still have a connection
+        # to this downstream server handy.
+        conn = self._connections.get(target.netloc, None)
+        if conn is None:
+            conn = http.client.HTTPConnection(target.netloc)
+
+        # Collect headers from client which will be sent to the downstream
+        # server.
+        client_headers = {}
+        for header_name in self.headers:
+            if header_name in ('Host', 'Content-Length'):
+                continue
+            for header_value in self.headers.get_all(header_name):
+                client_headers[header_name] = header_value
+
+        # Compute X-Forwarded-For header
+        client_address = '{}:{}'.format(*self.client_address)
+        x_forwarded_for_header = self.headers.get('X-Forwarded-For', None)
+        if x_forwarded_for_header is None:
+            client_headers['X-Forwarded-For'] = client_address
+        else:
+            client_headers['X-Forwarded-For'] = \
+                x_forwarded_for_header + ', ' + client_address
+
+        # Read the request body from client.
+        request_body_length = int(self.headers.get('Content-Length', '0'))
+        request_body = self.rfile.read(request_body_length)
+
+        # Send the request to the downstream server
+        if target.query:
+            target_path = target.path + '?' + target.query
+        else:
+            target_path = target.path
+        conn.request(self.command, target_path, request_body, client_headers)
+        response = conn.getresponse()
+
+        # Echo the response to the client.
+        self.send_response_only(response.status, response.reason)
+        for header_name, header_value in response.headers.items():
+            self.send_header(header_name, header_value)
+        self.end_headers()
+
+        # Read the response body from upstream and write it to downstream, if
+        # there is a response body at all.
+        response_content_length = \
+            int(response.headers.get('Content-Length', '0'))
+        if response_content_length > 0:
+            self.wfile.write(response.read(response_content_length))
+
+        # Cleanup, possibly hang on to persistent connection to target
+        # server.
+        connection_header_value = self.headers.get('Connection', None)
+        if response.will_close or connection_header_value == 'close':
+            conn.close()
+            self.close_connection = True
+        else:
+            # Hang on to this connection for future requests. This isn't
+            # really bulletproof but it's good enough for integration tests.
+            self._connections[target.netloc] = conn
+
+        self.log_message(
+            'Proxied request from %s to %s',
+            client_address,
+            self.path)
+
+    def do_GET(self):
+        self._proxy_request()
+
+    def do_HEAD(self):
+        self._proxy_request()
+
+    def do_POST(self):
+        self._proxy_request()
+
+    def do_PUT(self):
+        self._proxy_request()
+
+    def do_DELETE(self):
+        self._proxy_request()
+
+    def do_PATCH(self):
+        self._proxy_request()
+
+    def do_OPTIONS(self):
+        self._proxy_request()
+
+
+def main():
+    options = parse_args(sys.argv)
+
+    bind_addr = (
+        options.bind_addr,
+        0 if options.port is None else options.port)
+
+    server = http.server.HTTPServer(
+        bind_addr,
+        RequestHandler,
+        bind_and_activate=False)
+    if options.port is None:
+        server.server_port = server.socket.getsockname()[1]
+    else:
+        server.server_port = options.port
+
+    if options.server_socket_fd:
+        server.socket = socket.fromfd(
+            options.server_socket_fd,
+            socket.AF_INET,
+            socket.SOCK_STREAM)
+    else:
+        server.server_bind()
+        server.server_activate()
+
+    print(
+        'Transparent HTTP proxy listening on port',
+        server.server_port,
+        file=sys.stderr)
+    try:
+        server.serve_forever(0.01)
+    except KeyboardInterrupt:
+        server.server_close()
+
+
+def parse_args(argv):
+    parser = optparse.OptionParser(
+        usage='Usage: %prog [OPTIONS]',
+        description=__doc__)
+    parser.add_option(
+        '--bind-addr',
+        default='127.0.0.1',
+        dest='bind_addr',
+        help='By default only bind to loopback')
+    parser.add_option(
+        '--port',
+        dest='port',
+        default=None,
+        type='int',
+        help='If not specified a random port will be used.')
+    parser.add_option(
+        "--fd",
+        dest='server_socket_fd',
+        default=None,
+        type='int',
+        help='A socket FD to use for accept() instead of binding a new one.')
+    options, args = parser.parse_args(argv)
+    if len(args) > 1:
+        parser.error('Unexpected arguments: {}'.format(', '.join(args[1:])))
+    return options
+
+
+if __name__ == '__main__':
+    main()

src/tests/kits/net/service/testserver.py

@@ -144,7 +144,7 @@ class RequestHandler(http.server.BaseHTTPRequestHandler):
         output_stream.write(b'--------\r\n')
         for header in self.headers:
             for header_value in self.headers.get_all(header):
-                if header == 'Host' or header == 'Referer':
+                if header in ('Host', 'Referer', 'X-Forwarded-For'):
                     # The server port can change between runs which will change
                     # the size and contents of the response body. To make tests
                     # that verify the contents of the response body easier the