libbnetapi: BUrlRequest now outputs to BDataIO

Previously, BUrlRequest returns data received via a callback that can't
return any value. This approach have several issues:

- It's not possible to signify failures to the request.
- Users have to implement custom listeners just to handle the common
  case of outputting to a buffer/file/etc.
- The received data has to be serialized into BMessage when
  BUrlProtocolDispatchingListener is employed. This can cause a
  noticible slowdown in real-world scenarios as evident by #10748.

With this change, BUrlRequest will output directly into a BDataIO, which
exposes a richer API for request handlers to work with (for example a
BitTorrent client can request a BPositionIO for non-linear data
delivery), as well as simplifying common cases for users.

The adaptation only requires one additional API:
BHttpRequest::SetStopOnError(). This API simply instructs the HTTP
request handler to cancel the request if an HTTP error is occurred.

Change-Id: I4160884d77bff0e7678e0a623e2587987704443a
Reviewed-on: https://review.haiku-os.org/c/haiku/+/3084
Reviewed-by: Adrien Destugues <[email protected]>

docs/user/netservices/UrlProtocolListener.dox

@@ -113,16 +113,18 @@
 */
 
 /*!
-	\fn virtual void BUrlProtocolListener::DataReceived(BUrlRequest* caller,
-		const char* data, off_t position, size_t size)
-	\brief Called each time a block of data is received.
+	\fn virtual void BUrlProtocolListener::BytesWritten(BUrlRequest* caller,
+		size_t size)
+	\brief Called each time a block of data is written.
+
+	This callback is called whenever a block of data is written to the
+	BDataIO associated with the request. If no BDataIO is associated, the
+	callback will not be invoked.
 
 	\b Frequency: Zero or more
 
 	\param caller The BUrlRequest that invoked this callback.
-	\param data Pointer to the data block in memory.
-	\param position Offset of the data in the stream.
-	\param size Size of the data block.
+	\param size Size of the written data block.
 */
 
 /*!
@@ -130,15 +132,17 @@
 		off_t bytesReceived, off_t bytesTotal)
 	\brief Called each time a block of data is downloaded.
 
-	This callback will usually be called after DataReceived().
+	This callback might still be invoked even when no BDataIO is associated
+	with the request, as data can still be downloaded and discarded so that the
+	request can be completed and the socket can be reused.
 
-	\b Frequency: Once or more
+	\b Frequency: Zero or more
 
 	\param caller The BUrlRequest that invoked this callback.
 	\param bytesReceived Number of data bytes received. This is the number of
 		bytes received prior to any processing and can be smaller than the
-		size of the data block sent to DataReceived() as the transport might
-		be compressed.
+		size of the data block written to the output BDataIO as the transport
+		might be compressed.
 	\param bytesTotal Total number of data bytes expected. \c 0 will be passed
 		if the total number of data bytes is not available.
 */

docs/user/netservices/UrlProtocolRoster.dox

@@ -27,10 +27,12 @@
 
 /*!
 	\fn static BUrlRequest* BUrlProtocolRoster::MakeRequest(const BUrl& url,
-		BUrlProtocolListener* listener = NULL, BUrlContext* context = NULL)
+		BDataIO* output, BUrlProtocolListener* listener = NULL,
+		BUrlContext* context = NULL)
 	\brief Create a BUrlRequest that can handle the given BUrl
 
 	\param url The URL to create a request for
+	\param output The BDataIO to output to
 	\param listener The BUrlProtocolListener to be registered with the created
 		BUrlRequest, can be \c NULL
 	\param context The BUrlContext to be registered with the created

headers/private/netservices/DataRequest.h

@@ -27,10 +27,16 @@ public:
 private:
 		friend class BUrlProtocolRoster;
 
+#ifdef LIBNETAPI_DEPRECATED
 							BDataRequest(const BUrl& url,
 								BUrlProtocolListener* listener = NULL,
 								BUrlContext* context = NULL);
-
+#else
+							BDataRequest(const BUrl& url,
+								BDataIO* output,
+								BUrlProtocolListener* listener = NULL,
+								BUrlContext* context = NULL);
+#endif
 		status_t			_ProtocolLoop();
 private:
 		BUrlResult			fResult;

headers/private/netservices/FileRequest.h

@@ -29,9 +29,16 @@ public:
 private:
 			friend class BUrlProtocolRoster;
 
+#ifdef LIBNETAPI_DEPRECATED
 								BFileRequest(const BUrl& url,
 									BUrlProtocolListener* listener = NULL,
 									BUrlContext* context = NULL);
+#else
+								BFileRequest(const BUrl& url,
+									BDataIO* output,
+									BUrlProtocolListener* listener = NULL,
+									BUrlContext* context = NULL);
+#endif
 
 			status_t			_ProtocolLoop();
 private:

headers/private/netservices/GopherRequest.h

@@ -29,16 +29,27 @@ public:
 private:
 			friend class BUrlProtocolRoster;
 
+#ifdef LIBNETAPI_DEPRECATED
 								BGopherRequest(const BUrl& url,
 									BUrlProtocolListener* listener = NULL,
 									BUrlContext* context = NULL);
+#else
+								BGopherRequest(const BUrl& url,
+									BDataIO* output,
+									BUrlProtocolListener* listener = NULL,
+									BUrlContext* context = NULL);
+#endif
 
 			status_t			_ProtocolLoop();
 			void				_SendRequest();
 
 			bool				_NeedsParsing();
 			bool				_NeedsLastDotStrip();
+#ifdef LIBNETAPI_DEPRECATED
 			void				_ParseInput(bool last);
+#else
+			status_t			_ParseInput(bool last);
+#endif
 
 			BString&			_HTMLEscapeString(BString &str);
 

headers/private/netservices/HttpRequest.h

@@ -41,6 +41,9 @@ public:
 			void				SetDiscardData(bool discard);
 			void				SetDisableListener(bool disable);
 			void				SetAutoReferrer(bool enable);
+#ifndef LIBNETAPI_DEPRECATED
+			void				SetStopOnError(bool stop);
+#endif
 			void				SetUserName(const BString& name);
 			void				SetPassword(const BString& password);
 			void				SetRangeStart(off_t position);
@@ -67,11 +70,20 @@ public:
 private:
 			friend class BUrlProtocolRoster;
 
+#ifdef LIBNETAPI_DEPRECATED
 								BHttpRequest(const BUrl& url,
 									bool ssl = false,
 									const char* protocolName = "HTTP",
 									BUrlProtocolListener* listener = NULL,
 									BUrlContext* context = NULL);
+#else
+								BHttpRequest(const BUrl& url,
+									BDataIO* output,
+									bool ssl = false,
+									const char* protocolName = "HTTP",
+									BUrlProtocolListener* listener = NULL,
+									BUrlContext* context = NULL);
+#endif
 								BHttpRequest(const BHttpRequest& other);
 
 			void				_ResetOptions();
@@ -101,10 +113,12 @@ private:
 	// Utility methods
 			bool				_IsDefaultPort();
 
+#ifdef LIBNETAPI_DEPRECATED
 	// Listener notification
 			void				_NotifyDataReceived(const char* data,
 									off_t pos, ssize_t length,
 									off_t bytesReceived, ssize_t bytesTotal);
+#endif
 
 private:
 			bool				fSSL;
@@ -146,6 +160,9 @@ private:
 			bool				fOptDiscardData : 1;
 			bool				fOptDisableListener : 1;
 			bool				fOptAutoReferer : 1;
+#ifndef LIBNETAPI_DEPRECATED
+			bool				fOptStopOnError : 1;
+#endif
 };
 
 // Request method

headers/private/netservices/NetworkRequest.h

@@ -23,11 +23,20 @@ namespace Network {
 class BNetworkRequest: public BUrlRequest
 {
 public:
+#ifdef LIBNETAPI_DEPRECATED
 								BNetworkRequest(const BUrl& url,
 									BUrlProtocolListener* listener,
 									BUrlContext* context,
 									const char* threadName,
 									const char* protocolName);
+#else
+								BNetworkRequest(const BUrl& url,
+									BDataIO* output,
+									BUrlProtocolListener* listener,
+									BUrlContext* context,
+									const char* threadName,
+									const char* protocolName);
+#endif
 
 	virtual	status_t			Stop();
 	virtual void				SetTimeout(bigtime_t timeout);

headers/private/netservices/UrlProtocolAsynchronousListener.h

@@ -25,9 +25,9 @@ public:
 
 	// Synchronous listener access
 			BUrlProtocolListener* SynchronousListener();
-									
+
 	// BHandler interface
-	virtual void				MessageReceived(BMessage* message);
+	virtual	void				MessageReceived(BMessage* message);
 
 private:
 			BUrlProtocolDispatchingListener*

headers/private/netservices/UrlProtocolDispatchingListener.h

@@ -28,7 +28,11 @@ enum {
 	B_URL_PROTOCOL_HOSTNAME_RESOLVED,
 	B_URL_PROTOCOL_RESPONSE_STARTED,
 	B_URL_PROTOCOL_HEADERS_RECEIVED,
+#ifdef LIBNETAPI_DEPRECATED
 	B_URL_PROTOCOL_DATA_RECEIVED,
+#else
+	B_URL_PROTOCOL_BYTES_WRITTEN,
+#endif
 	B_URL_PROTOCOL_DOWNLOAD_PROGRESS,
 	B_URL_PROTOCOL_UPLOAD_PROGRESS,
 	B_URL_PROTOCOL_REQUEST_COMPLETED,
@@ -46,44 +50,44 @@ public:
 	virtual						~BUrlProtocolDispatchingListener();
 
 	virtual	void				ConnectionOpened(BUrlRequest* caller);
-	virtual void				HostnameResolved(BUrlRequest* caller,
+	virtual	void				HostnameResolved(BUrlRequest* caller,
 									const char* ip);
-	virtual void				ResponseStarted(BUrlRequest* caller);
-#ifdef LIBNETAPI_DEPRECATED
-	virtual void				HeadersReceived(BUrlRequest* caller,
-									const BUrlResult& result);
-#else
-	virtual void				HeadersReceived(BUrlRequest* caller);
-#endif
-	virtual void				DataReceived(BUrlRequest* caller,
-									const char* data, off_t position,
-									ssize_t size);
+	virtual	void				ResponseStarted(BUrlRequest* caller);
 
 #ifdef LIBNETAPI_DEPRECATED
+	virtual	void				HeadersReceived(BUrlRequest* caller,
+									const BUrlResult& result);
+	virtual	void				DataReceived(BUrlRequest* caller,
+									const char* data, off_t position,
+									ssize_t size);
 	virtual	void				DownloadProgress(BUrlRequest* caller,
 									ssize_t bytesReceived, ssize_t bytesTotal);
-	virtual void				UploadProgress(BUrlRequest* caller,
+	virtual	void				UploadProgress(BUrlRequest* caller,
 									ssize_t bytesSent, ssize_t bytesTotal);
 #else
+	virtual	void				HeadersReceived(BUrlRequest* caller);
+	virtual	void				BytesWritten(BUrlRequest* caller,
+									size_t bytesWritten);
 	virtual	void				DownloadProgress(BUrlRequest* caller,
 									off_t bytesReceived, off_t bytesTotal);
-	virtual void				UploadProgress(BUrlRequest* caller,
+	virtual	void				UploadProgress(BUrlRequest* caller,
 									off_t bytesSent, off_t bytesTotal);
 #endif
 
-	virtual void				RequestCompleted(BUrlRequest* caller,
+	virtual	void				RequestCompleted(BUrlRequest* caller,
 									bool success);
-	virtual void				DebugMessage(BUrlRequest* caller,
+	virtual	void				DebugMessage(BUrlRequest* caller,
 									BUrlProtocolDebugMessage type,
 									const char* text);
-	virtual bool				CertificateVerificationFailed(
+	virtual	bool				CertificateVerificationFailed(
 									BUrlRequest* caller,
 									BCertificate& certificate,
 									const char* message);
 
 private:
 			void				_SendMessage(BMessage* message,
-									int8 notification, BUrlRequest* caller);
+									int8 notification,
+									BUrlRequest* caller);
 
 private:
 			BMessenger	 		fMessenger;

headers/private/netservices/UrlProtocolListener.h

@@ -36,37 +36,35 @@ enum BUrlProtocolDebugMessage {
 class BUrlProtocolListener {
 public:
 	virtual	void				ConnectionOpened(BUrlRequest* caller);
-	virtual void				HostnameResolved(BUrlRequest* caller,
+	virtual	void				HostnameResolved(BUrlRequest* caller,
 									const char* ip);
-	virtual void				ResponseStarted(BUrlRequest* caller);
+	virtual	void				ResponseStarted(BUrlRequest* caller);
 #ifdef LIBNETAPI_DEPRECATED
-	virtual void				HeadersReceived(BUrlRequest* caller,
+	virtual	void				HeadersReceived(BUrlRequest* caller,
 									const BUrlResult& result);
-#else
-	virtual void				HeadersReceived(BUrlRequest* caller);
-#endif
-	virtual void				DataReceived(BUrlRequest* caller,
+	virtual	void				DataReceived(BUrlRequest* caller,
 									const char* data, off_t position,
 									ssize_t size);
-
-#ifdef LIBNETAPI_DEPRECATED
 	virtual	void				DownloadProgress(BUrlRequest* caller,
 									ssize_t bytesReceived, ssize_t bytesTotal);
-	virtual void				UploadProgress(BUrlRequest* caller,
+	virtual	void				UploadProgress(BUrlRequest* caller,
 									ssize_t bytesSent, ssize_t bytesTotal);
 #else
+	virtual	void				HeadersReceived(BUrlRequest* caller);
+	virtual	void				BytesWritten(BUrlRequest* caller,
+									size_t bytesWritten);
 	virtual	void				DownloadProgress(BUrlRequest* caller,
 									off_t bytesReceived, off_t bytesTotal);
 	virtual void				UploadProgress(BUrlRequest* caller,
 									off_t bytesSent, off_t bytesTotal);
 #endif
 
-	virtual void				RequestCompleted(BUrlRequest* caller,
+	virtual	void				RequestCompleted(BUrlRequest* caller,
 									bool success);
-	virtual void				DebugMessage(BUrlRequest* caller,
+	virtual	void				DebugMessage(BUrlRequest* caller,
 									BUrlProtocolDebugMessage type,
 									const char* text);
-	virtual bool				CertificateVerificationFailed(
+	virtual	bool				CertificateVerificationFailed(
 									BUrlRequest* caller,
 									BCertificate& certificate,
 									const char* message);

headers/private/netservices/UrlProtocolRoster.h

@@ -10,6 +10,7 @@
 #include <stdlib.h>
 
 
+class BDataIO;
 class BUrl;
 
 #ifndef LIBNETAPI_DEPRECATED
@@ -24,9 +25,15 @@ class BUrlRequest;
 
 class BUrlProtocolRoster {
 public:
-    static  BUrlRequest*    MakeRequest(const BUrl& url,
-		                        BUrlProtocolListener* listener = NULL,
-                                BUrlContext* context = NULL);
+#ifdef LIBNETAPI_DEPRECATED
+	static	BUrlRequest*	MakeRequest(const BUrl& url,
+								BUrlProtocolListener* listener = NULL,
+								BUrlContext* context = NULL);
+#else
+	static	BUrlRequest*	MakeRequest(const BUrl& url, BDataIO* output,
+								BUrlProtocolListener* listener = NULL,
+								BUrlContext* context = NULL);
+#endif
 };
 
 #ifndef LIBNETAPI_DEPRECATED

headers/private/netservices/UrlRequest.h

@@ -22,11 +22,20 @@ namespace Network {
 
 class BUrlRequest {
 public:
+#ifdef LIBNETAPI_DEPRECATED
 									BUrlRequest(const BUrl& url,
 										BUrlProtocolListener* listener,
 										BUrlContext* context,
 										const char* threadName,
 										const char* protocolName);
+#else
+									BUrlRequest(const BUrl& url,
+										BDataIO* output,
+										BUrlProtocolListener* listener,
+										BUrlContext* context,
+										const char* threadName,
+										const char* protocolName);
+#endif
 	virtual							~BUrlRequest();
 
 	// URL protocol thread management
@@ -40,12 +49,18 @@ public:
 			status_t				SetUrl(const BUrl& url);
 			status_t				SetContext(BUrlContext* context);
 			status_t				SetListener(BUrlProtocolListener* listener);
+#ifndef LIBNETAPI_DEPRECATED
+			status_t				SetOutput(BDataIO* output);
+#endif
 
 	// URL protocol parameters access
 			const BUrl&				Url() const;
 			BUrlContext*			Context() const;
 			BUrlProtocolListener*	Listener() const;
 			const BString&			Protocol() const;
+#ifndef LIBNETAPI_DEPRECATED
+			BDataIO*				Output() const;
+#endif
 
 	// URL protocol informations
 			bool					IsRunning() const;
@@ -63,6 +78,9 @@ protected:
 			BUrl					fUrl;
 			BReference<BUrlContext>	fContext;
 			BUrlProtocolListener*	fListener;
+#ifndef LIBNETAPI_DEPRECATED
+			BDataIO*				fOutput;
+#endif
 
 			bool					fQuit;
 			bool					fRunning;

headers/private/netservices/UrlSynchronousRequest.h

@@ -20,42 +20,40 @@ class BUrlSynchronousRequest : public BUrlRequest, public BUrlProtocolListener {
 public:
 								BUrlSynchronousRequest(BUrlRequest& asynchronousRequest);
 	virtual						~BUrlSynchronousRequest() { };
-								
+
 	// Synchronous wait
 	virtual	status_t			Perform();
 	virtual	status_t			WaitUntilCompletion();
 
 	// Protocol hooks
 	virtual	void				ConnectionOpened(BUrlRequest* caller);
-	virtual void				HostnameResolved(BUrlRequest* caller,
+	virtual	void				HostnameResolved(BUrlRequest* caller,
 									const char* ip);
-	virtual void				ResponseStarted(BUrlRequest* caller);
+	virtual	void				ResponseStarted(BUrlRequest* caller);
 #ifdef LIBNETAPI_DEPRECATED
-	virtual void				HeadersReceived(BUrlRequest* caller,
+	virtual	void				HeadersReceived(BUrlRequest* caller,
 									const BUrlResult& result);
-#else
-	virtual void				HeadersReceived(BUrlRequest* caller);
-#endif
-	virtual void				DataReceived(BUrlRequest* caller,
+	virtual	void				DataReceived(BUrlRequest* caller,
 									const char* data, off_t position,
 									ssize_t size);
-
-#ifdef LIBNETAPI_DEPRECATED
 	virtual	void				DownloadProgress(BUrlRequest* caller,
 									ssize_t bytesReceived, ssize_t bytesTotal);
-	virtual void				UploadProgress(BUrlRequest* caller,
+	virtual	void				UploadProgress(BUrlRequest* caller,
 									ssize_t bytesSent, ssize_t bytesTotal);
 #else
+	virtual	void				HeadersReceived(BUrlRequest* caller);
+	virtual	void				BytesWritten(BUrlRequest* caller,
+									size_t bytesWritten);
 	virtual	void				DownloadProgress(BUrlRequest* caller,
 									off_t bytesReceived, off_t bytesTotal);
 	virtual void				UploadProgress(BUrlRequest* caller,
 									off_t bytesSent, off_t bytesTotal);
-#endif //LIBNETAPI_DEPRECATED
+#endif
 
-	virtual void				RequestCompleted(BUrlRequest* caller,
+	virtual	void				RequestCompleted(BUrlRequest* caller,
 									bool success);
-									
-									
+
+
 protected:
 			bool				fRequestComplete;
 			BUrlRequest&		fWrappedRequest;

src/add-ons/media/plugins/http_streamer/HTTPMediaIO.cpp

@@ -19,7 +19,7 @@
 using namespace BPrivate::Network;
 
 
-class FileListener : public BUrlProtocolListener {
+class FileListener : public BUrlProtocolListener, public BDataIO {
 public:
 		FileListener(HTTPMediaIO* owner)
 			:
@@ -50,33 +50,13 @@ public:
 		}
 
 		void HeadersReceived(BUrlRequest* request)
-		{
-			fAdapterIO->UpdateSize();
-		}
-
-		void DataReceived(BUrlRequest* request, const char* data,
-			off_t position, ssize_t size)
 		{
 			if (request != fRequest) {
 				delete request;
 				return;
 			}
 
-			BHttpRequest* httpReq = dynamic_cast<BHttpRequest*>(request);
-			if (httpReq != NULL) {
-				const BHttpResult& httpRes
-					= (const BHttpResult&)httpReq->Result();
-				int32 status = httpRes.StatusCode();
-				if (BHttpRequest::IsClientErrorStatusCode(status)
-						|| BHttpRequest::IsServerErrorStatusCode(status)) {
-					fRunning = false;
-				} else if (BHttpRequest::IsRedirectionStatusCode(status))
-					return;
-			}
-
-			_ReleaseInit();
-
-			fInputAdapter->Write(data, size);
+			fAdapterIO->UpdateSize();
 		}
 
 		void RequestCompleted(BUrlRequest* request, bool success)
@@ -89,6 +69,13 @@ public:
 			fRequest = NULL;
 		}
 
+		ssize_t Write(const void* data, size_t size)
+		{
+			_ReleaseInit();
+
+			return fInputAdapter->Write(data, size);
+		}
+
 		status_t LockOnInit(bigtime_t timeout)
 		{
 			return acquire_sem_etc(fInitSem, 1, B_RELATIVE_TIMEOUT, timeout);
@@ -174,11 +161,15 @@ HTTPMediaIO::Open()
 
 	fListener = new FileListener(this);
 
-	fReq = BUrlProtocolRoster::MakeRequest(fUrl, fListener);
+	fReq = BUrlProtocolRoster::MakeRequest(fUrl, fListener, fListener);
 
 	if (fReq == NULL)
 		return B_ERROR;
 
+	BHttpRequest* httpReq = dynamic_cast<BHttpRequest*>(fReq);
+	if (httpReq != NULL)
+		httpReq->SetStopOnError(true);
+
 	fReqThread = fReq->Run();
 	if (fReqThread < 0)
 		return B_ERROR;

src/apps/haikudepot/Jamfile

@@ -188,7 +188,7 @@ local applicationSources =
 	LocaleUtils.cpp
 	RepositoryUrlUtils.cpp
 	StorageUtils.cpp
-	ToFileUrlProtocolListener.cpp
+	LoggingUrlProtocolListener.cpp
 
 	# package_daemon
 	ProblemWindow.cpp

src/apps/haikudepot/server/AbstractServerProcess.cpp

@@ -11,6 +11,7 @@
 #include <string.h>
 
 #include <AutoDeleter.h>
+#include <File.h>
 #include <FileIO.h>
 #include <HttpTime.h>
 #include <UrlProtocolRoster.h>
@@ -24,7 +25,7 @@
 #include "ServerSettings.h"
 #include "StandardMetaDataJsonEventListener.h"
 #include "StorageUtils.h"
-#include "ToFileUrlProtocolListener.h"
+#include "LoggingUrlProtocolListener.h"
 
 
 using namespace BPrivate::Network;
@@ -330,8 +331,11 @@ AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath,
 	HDINFO("[%s] will stream '%s' to [%s]", Name(), url.UrlString().String(),
 		targetFilePath.Path());
 
-	ToFileUrlProtocolListener listener(targetFilePath, Name(),
-		Logger::IsTraceEnabled());
+	LoggingUrlProtocolListener listener(Name(), Logger::IsTraceEnabled());
+	BFile targetFile(targetFilePath.Path(), O_WRONLY | O_CREAT);
+	status_t err = targetFile.InitCheck();
+	if (err != B_OK)
+		return err;
 
 	BHttpHeaders headers;
 	ServerSettings::AugmentHeaders(headers);
@@ -347,7 +351,8 @@ AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath,
 
 	thread_id thread;
 
-	BUrlRequest* request = BUrlProtocolRoster::MakeRequest(url, &listener);
+	BUrlRequest* request = BUrlProtocolRoster::MakeRequest(url, &targetFile,
+		&listener);
 	if (request == NULL)
 		return B_NO_MEMORY;
 
@@ -359,6 +364,7 @@ AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath,
 	fRequest->SetHeaders(headers);
 	fRequest->SetMaxRedirections(0);
 	fRequest->SetTimeout(TIMEOUT_MICROSECONDS);
+	fRequest->SetStopOnError(true);
 	thread = fRequest->Run();
 
 	wait_for_thread(thread, NULL);

src/apps/haikudepot/server/WebAppInterface.cpp

@@ -37,8 +37,6 @@ using namespace BPrivate::Network;
 class ProtocolListener : public BUrlProtocolListener {
 public:
 	ProtocolListener()
-		:
-		fDownloadIO(NULL)
 	{
 	}
 
@@ -62,11 +60,8 @@ public:
 	{
 	}
 
-	virtual void DataReceived(BUrlRequest* caller, const char* data,
-		off_t position, ssize_t size)
+	virtual void BytesWritten(BUrlRequest* caller, size_t bytesWritten)
 	{
-		if (fDownloadIO != NULL)
-			fDownloadIO->Write(data, size);
 	}
 
 	virtual	void DownloadProgress(BUrlRequest* caller, off_t bytesReceived,
@@ -88,23 +83,16 @@ public:
 	{
 		HDTRACE("jrpc: %s", text);
 	}
-
-	void SetDownloadIO(BDataIO* downloadIO)
-	{
-		fDownloadIO = downloadIO;
-	}
-
-private:
-	BDataIO*		fDownloadIO;
 };
 
 
 static BHttpRequest*
-make_http_request(const BUrl& url, BUrlProtocolListener* listener = NULL,
+make_http_request(const BUrl& url, BDataIO* output,
+	BUrlProtocolListener* listener = NULL,
 	BUrlContext* context = NULL)
 {
-	BUrlRequest* request = BUrlProtocolRoster::MakeRequest(url, listener,
-		context);
+	BUrlRequest* request = BUrlProtocolRoster::MakeRequest(url, output,
+		listener, context);
 	BHttpRequest* httpRequest = dynamic_cast<BHttpRequest*>(request);
 	if (httpRequest == NULL) {
 		delete request;
@@ -864,7 +852,7 @@ WebAppInterface::_SendJsonRequest(const char* domain,
 	headers.AddHeader("Accept", "application/json");
 	ServerSettings::AugmentHeaders(headers);
 
-	BHttpRequest* request = make_http_request(url, &listener, &context);
+	BHttpRequest* request = make_http_request(url, NULL, &listener, &context);
 	ObjectDeleter<BHttpRequest> _(request);
 	if (request == NULL)
 		return B_ERROR;
@@ -884,7 +872,7 @@ WebAppInterface::_SendJsonRequest(const char* domain,
 	request->AdoptInputData(requestData, requestDataSize);
 
 	BMallocIO replyData;
-	listener.SetDownloadIO(&replyData);
+	request->SetOutput(&replyData);
 
 	thread_id thread = request->Run();
 	wait_for_thread(thread, NULL);
@@ -953,12 +941,11 @@ WebAppInterface::_SendRawGetRequest(const BString urlPathComponents,
 	BUrl url = ServerSettings::CreateFullUrl(urlPathComponents);
 
 	ProtocolListener listener;
-	listener.SetDownloadIO(stream);
 
 	BHttpHeaders headers;
 	ServerSettings::AugmentHeaders(headers);
 
-	BHttpRequest *request = make_http_request(url, &listener);
+	BHttpRequest *request = make_http_request(url, stream, &listener);
 	ObjectDeleter<BHttpRequest> _(request);
 	if (request == NULL)
 		return B_ERROR;

src/apps/haikudepot/util/LoggingUrlProtocolListener.cpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2017-2020, Andrew Lindesay <apl@lindesay.co.nz>.
+ * All rights reserved. Distributed under the terms of the MIT License.
+ */
+
+#include "LoggingUrlProtocolListener.h"
+
+#include <File.h>
+#include <HttpRequest.h>
+
+#include "Logger.h"
+
+using namespace BPrivate::Network;
+
+
+LoggingUrlProtocolListener::LoggingUrlProtocolListener(
+	BString traceLoggingIdentifier, bool traceLogging)
+	:
+	fTraceLogging(traceLogging),
+	fTraceLoggingIdentifier(traceLoggingIdentifier),
+	fContentLength(0)
+{
+}
+
+
+void
+LoggingUrlProtocolListener::BytesWritten(BUrlRequest* caller,
+	size_t bytesWritten)
+{
+	fContentLength += bytesWritten;
+}
+
+
+void
+LoggingUrlProtocolListener::DebugMessage(BUrlRequest* caller,
+	BUrlProtocolDebugMessage type, const char* text)
+{
+	HDTRACE("url->file <%s>; %s", fTraceLoggingIdentifier.String(), text);
+}
+
+
+size_t
+LoggingUrlProtocolListener::ContentLength()
+{
+	return fContentLength;
+}

src/apps/haikudepot/util/LoggingUrlProtocolListener.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
+ * All rights reserved. Distributed under the terms of the MIT License.
+ */
+
+#include <UrlProtocolListener.h>
+#include <UrlRequest.h>
+
+using BPrivate::Network::BUrlProtocolDebugMessage;
+using BPrivate::Network::BUrlProtocolListener;
+using BPrivate::Network::BUrlRequest;
+
+
+class LoggingUrlProtocolListener : public BUrlProtocolListener {
+public:
+								LoggingUrlProtocolListener(
+									BString traceLoggingIdentifier,
+									bool traceLogging);
+
+			size_t				ContentLength();
+
+			void				BytesWritten(BUrlRequest* caller,
+									size_t bytesWritten);
+			void				DebugMessage(BUrlRequest* caller,
+									BUrlProtocolDebugMessage type,
+									const char* text);
+
+private:
+			bool				fTraceLogging;
+			BString				fTraceLoggingIdentifier;
+			size_t				fContentLength;
+
+
+};

src/apps/haikudepot/util/ToFileUrlProtocolListener.cpp

@@ -1,127 +0,0 @@
-/*
- * Copyright 2017-2020, Andrew Lindesay <apl@lindesay.co.nz>.
- * All rights reserved. Distributed under the terms of the MIT License.
- */
-
-#include "ToFileUrlProtocolListener.h"
-
-#include <File.h>
-#include <HttpRequest.h>
-
-#include "Logger.h"
-
-
-using namespace BPrivate::Network;
-
-
-ToFileUrlProtocolListener::ToFileUrlProtocolListener(BPath path,
-	BString traceLoggingIdentifier, bool traceLogging)
-{
-	fDownloadIO = new BFile(path.Path(), O_WRONLY | O_CREAT);
-	fTraceLoggingIdentifier = traceLoggingIdentifier;
-	fTraceLogging = traceLogging;
-	fShouldDownload = true;
-	fContentLength = 0;
-}
-
-
-ToFileUrlProtocolListener::~ToFileUrlProtocolListener()
-{
-	delete fDownloadIO;
-}
-
-
-void
-ToFileUrlProtocolListener::ConnectionOpened(BUrlRequest* caller)
-{
-}
-
-
-void
-ToFileUrlProtocolListener::HostnameResolved(BUrlRequest* caller,
-	const char* ip)
-{
-}
-
-
-void
-ToFileUrlProtocolListener::ResponseStarted(BUrlRequest* caller)
-{
-}
-
-
-void
-ToFileUrlProtocolListener::HeadersReceived(BUrlRequest* caller)
-{
-
-	// check that the status code is success.  Only if it is successful
-	// should the payload be streamed to the file.
-
-	const BHttpResult& httpResult = dynamic_cast<const BHttpResult&>(
-		caller->Result());
-	int32 statusCode = httpResult.StatusCode();
-
-	if (!BHttpRequest::IsSuccessStatusCode(statusCode)) {
-		HDINFO("received http status %" B_PRId32
-			" --> will not store download to file", statusCode);
-		fShouldDownload = false;
-	}
-
-}
-
-
-void
-ToFileUrlProtocolListener::DataReceived(BUrlRequest* caller, const char* data,
-	off_t position, ssize_t size)
-{
-	fContentLength += size;
-
-	if (fShouldDownload && fDownloadIO != NULL && size > 0) {
-		size_t remaining = size;
-		size_t written = 0;
-
-		do {
-			written = fDownloadIO->WriteAt(position, &data[size - remaining],
-				remaining);
-			remaining -= written;
-		} while (remaining > 0 && written > 0);
-
-		if (remaining > 0)
-			HDERROR("unable to write all of the data to the file");
-	}
-}
-
-
-void
-ToFileUrlProtocolListener::DownloadProgress(BUrlRequest* caller,
-	off_t bytesReceived, off_t bytesTotal)
-{
-}
-
-
-void
-ToFileUrlProtocolListener::UploadProgress(BUrlRequest* caller,
-	off_t bytesSent, off_t bytesTotal)
-{
-}
-
-
-void
-ToFileUrlProtocolListener::RequestCompleted(BUrlRequest* caller, bool success)
-{
-}
-
-
-void
-ToFileUrlProtocolListener::DebugMessage(BUrlRequest* caller,
-	BUrlProtocolDebugMessage type, const char* text)
-{
-	HDTRACE("url->file <%s>; %s", fTraceLoggingIdentifier.String(), text);
-}
-
-
-ssize_t
-ToFileUrlProtocolListener::ContentLength()
-{
-	return fContentLength;
-}

src/apps/haikudepot/util/ToFileUrlProtocolListener.h

@@ -1,50 +0,0 @@
-/*
- * Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>.
- * All rights reserved. Distributed under the terms of the MIT License.
- */
-
-#include <UrlProtocolListener.h>
-#include <UrlRequest.h>
-
-
-using BPrivate::Network::BUrlProtocolDebugMessage;
-using BPrivate::Network::BUrlProtocolListener;
-using BPrivate::Network::BUrlRequest;
-using BPrivate::Network::BUrlResult;
-
-class ToFileUrlProtocolListener : public BUrlProtocolListener {
-public:
-								ToFileUrlProtocolListener(BPath path,
-									BString traceLoggingIdentifier,
-									bool traceLogging);
-			virtual				~ToFileUrlProtocolListener();
-
-			ssize_t				ContentLength();
-
-			void				ConnectionOpened(BUrlRequest* caller);
-			void				HostnameResolved(BUrlRequest* caller,
-									const char* ip);
-			void				ResponseStarted(BUrlRequest* caller);
-			void				HeadersReceived(BUrlRequest* caller);
-			void				DataReceived(BUrlRequest* caller,
-									const char* data, off_t position,
-									ssize_t size);
-			void				DownloadProgress(BUrlRequest* caller,
-									off_t bytesReceived, off_t bytesTotal);
-			void				UploadProgress(BUrlRequest* caller,
-									off_t bytesSent, off_t bytesTotal);
-			void				RequestCompleted(BUrlRequest* caller,
-									bool success);
-			void				DebugMessage(BUrlRequest* caller,
-									BUrlProtocolDebugMessage type,
-									const char* text);
-
-private:
-			bool				fShouldDownload;
-			bool				fTraceLogging;
-			BString				fTraceLoggingIdentifier;
-			BPositionIO*		fDownloadIO;
-			ssize_t				fContentLength;
-
-
-};

src/kits/network/libnetservices/DataRequest.cpp

@@ -20,6 +20,7 @@ using namespace BPrivate::Network;
 #endif
 
 
+#ifdef LIBNETAPI_DEPRECATED
 BDataRequest::BDataRequest(const BUrl& url, BUrlProtocolListener* listener,
 		BUrlContext* context)
 	: BUrlRequest(url, listener, context, "data URL parser", "data"),
@@ -28,6 +29,19 @@ BDataRequest::BDataRequest(const BUrl& url, BUrlProtocolListener* listener,
 	fResult.SetContentType("text/plain");
 }
 
+#else
+
+BDataRequest::BDataRequest(const BUrl& url, BDataIO* output,
+	BUrlProtocolListener* listener,
+	BUrlContext* context)
+	:
+	BUrlRequest(url, output, listener, context, "data URL parser", "data"),
+	fResult()
+{
+	fResult.SetContentType("text/plain");
+}
+#endif // LIBNETAPI_DEPRECATED
+
 
 const BUrlResult&
 BDataRequest::Result() const
@@ -122,17 +136,30 @@ BDataRequest::_ProtocolLoop()
 
 	fResult.SetLength(length);
 
-	if (fListener != NULL) {
 #ifdef LIBNETAPI_DEPRECATED
+	if (fListener != NULL) {
 		fListener->HeadersReceived(this, fResult);
-#else
-		fListener->HeadersReceived(this);
-#endif
 		if (length > 0) {
 			fListener->DataReceived(this, payload, 0, length);
 			fListener->DownloadProgress(this, length, length);
 		}
 	}
+#else
+	if (fListener != NULL)
+		fListener->HeadersReceived(this);
+	if (length > 0) {
+		if (fOutput != NULL) {
+			size_t written = 0;
+			status_t err = fOutput->WriteExactly(payload, length, &written);
+			if (fListener != NULL && written > 0)
+				fListener->BytesWritten(this, written);
+			if (err != B_OK)
+				return err;
+			if (fListener != NULL)
+				fListener->DownloadProgress(this, written, written);
+		}
+	}
+#endif // LIBNETAPI_DEPRECATED
 
 	return B_OK;
 }

src/kits/network/libnetservices/FileRequest.cpp

@@ -21,6 +21,8 @@
 using namespace BPrivate::Network;
 #endif
 
+
+#ifdef LIBNETAPI_DEPRECATED
 BFileRequest::BFileRequest(const BUrl& url, BUrlProtocolListener* listener,
 	BUrlContext* context)
 	:
@@ -30,6 +32,18 @@ BFileRequest::BFileRequest(const BUrl& url, BUrlProtocolListener* listener,
 	fUrl.UrlDecode(true);
 }
 
+#else
+
+BFileRequest::BFileRequest(const BUrl& url, BDataIO* output,
+	BUrlProtocolListener* listener, BUrlContext* context)
+	:
+	BUrlRequest(url, output, listener, context, "BUrlProtocol.File", "file"),
+	fResult()
+{
+	fUrl.UrlDecode(true);
+}
+#endif // LIBNETAPI_DEPRECATED
+
 
 BFileRequest::~BFileRequest()
 {
@@ -46,6 +60,7 @@ BFileRequest::Result() const
 }
 
 
+#ifdef LIBNETAPI_DEPRECATED
 status_t
 BFileRequest::_ProtocolLoop()
 {
@@ -81,11 +96,7 @@ BFileRequest::_ProtocolLoop()
 				return error;
 			fResult.SetLength(size);
 
-#ifdef LIBNETAPI_DEPRECATED
 			fListener->HeadersReceived(this, fResult);
-#else
-			fListener->HeadersReceived(this);
-#endif
 
 			ssize_t chunkSize = 0;
 			char chunk[4096];
@@ -128,11 +139,7 @@ BFileRequest::_ProtocolLoop()
 
 	if (fListener != NULL) {
 		fListener->ConnectionOpened(this);
-#ifdef LIBNETAPI_DEPRECATED
 		fListener->HeadersReceived(this, fResult);
-#else
-		fListener->HeadersReceived(this);
-#endif
 
 		// Add a parent directory entry.
 		fListener->DataReceived(this, "+/,\t..\r\n", transferredSize, 8);
@@ -184,3 +191,157 @@ BFileRequest::_ProtocolLoop()
 
 	return fQuit ? B_INTERRUPTED : B_OK;
 }
+
+#else
+
+status_t
+BFileRequest::_ProtocolLoop()
+{
+	BNode node(fUrl.Path().String());
+
+	if (node.IsSymLink()) {
+		// Traverse the symlink and start over
+		BEntry entry(fUrl.Path().String(), true);
+		node = BNode(&entry);
+	}
+
+	ssize_t transferredSize = 0;
+	if (node.IsFile()) {
+		BFile file(fUrl.Path().String(), B_READ_ONLY);
+		status_t error = file.InitCheck();
+		if (error != B_OK)
+			return error;
+
+		BNodeInfo info(&file);
+		char mimeType[B_MIME_TYPE_LENGTH + 1];
+		if (info.GetType(mimeType) != B_OK)
+			update_mime_info(fUrl.Path().String(), false, true, false);
+		if (info.GetType(mimeType) == B_OK)
+			fResult.SetContentType(mimeType);
+
+		// Send all notifications to listener, if any
+		if (fListener != NULL)
+			fListener->ConnectionOpened(this);
+
+		off_t size = 0;
+		error = file.GetSize(&size);
+		if (error != B_OK)
+			return error;
+		fResult.SetLength(size);
+
+		if (fListener != NULL)
+			fListener->HeadersReceived(this);
+
+		if (fOutput != NULL) {
+			ssize_t chunkSize = 0;
+			char chunk[4096];
+			while (!fQuit) {
+				chunkSize = file.Read(chunk, sizeof(chunk));
+				if (chunkSize > 0) {
+					size_t written = 0;
+					error = fOutput->WriteExactly(chunk, chunkSize, &written);
+					if (fListener != NULL && written > 0)
+						fListener->BytesWritten(this, written);
+					if (error != B_OK)
+						return error;
+					transferredSize += chunkSize;
+					if (fListener != NULL)
+						fListener->DownloadProgress(this, transferredSize,
+							size);
+				} else
+					break;
+			}
+			if (fQuit)
+				return B_INTERRUPTED;
+			// Return error if we didn't transfer everything
+			if (transferredSize != size) {
+				if (chunkSize < 0)
+					return (status_t)chunkSize;
+				else
+					return B_IO_ERROR;
+			}
+		}
+
+		return B_OK;
+	}
+
+	node_ref ref;
+	status_t error = node.GetNodeRef(&ref);
+
+	// Stop here, and don't hit the assert below, if the file doesn't exist.
+	if (error != B_OK)
+		return error;
+
+	assert(node.IsDirectory());
+	BDirectory directory(&ref);
+
+	fResult.SetContentType("application/x-ftp-directory; charset=utf-8");
+		// This tells WebKit to use its FTP directory rendering code.
+
+	if (fListener != NULL) {
+		fListener->ConnectionOpened(this);
+		fListener->HeadersReceived(this);
+	}
+
+	if (fOutput != NULL) {
+		// Add a parent directory entry.
+		size_t written = 0;
+		error = fOutput->WriteExactly("+/,\t..\r\n", 8, &written);
+		if (fListener != NULL && written > 0)
+			fListener->BytesWritten(this, written);
+		if (error != B_OK)
+			return error;
+		transferredSize += written;
+		if (fListener != NULL)
+			fListener->DownloadProgress(this, transferredSize, 0);
+
+		char name[B_FILE_NAME_LENGTH];
+		BEntry entry;
+		while (!fQuit && directory.GetNextEntry(&entry) != B_ENTRY_NOT_FOUND) {
+			// We read directories using the EPLF (Easily Parsed List Format)
+			// This happens to be one of the formats that WebKit can understand,
+			// and it is not too hard to parse or generate.
+			// http://tools.ietf.org/html/draft-bernstein-eplf-02
+			BString eplf("+");
+			if (entry.IsFile() || entry.IsSymLink()) {
+				eplf += "r,";
+				off_t fileSize;
+				if (entry.GetSize(&fileSize) == B_OK)
+					eplf << "s" << fileSize << ",";
+			} else if (entry.IsDirectory())
+				eplf += "/,";
+
+			time_t modification;
+			if (entry.GetModificationTime(&modification) == B_OK)
+				eplf << "m" << modification << ",";
+
+			mode_t permissions;
+			if (entry.GetPermissions(&permissions) == B_OK)
+				eplf << "up" << BString().SetToFormat("%03o", permissions) << ",";
+
+			node_ref ref;
+			if (entry.GetNodeRef(&ref) == B_OK)
+				eplf << "i" << ref.device << "." << ref.node << ",";
+
+			entry.GetName(name);
+			eplf << "\t" << name << "\r\n";
+			size_t written = 0;
+			error = fOutput->WriteExactly(eplf.String(), eplf.Length(),
+				&written);
+			if (fListener != NULL && written > 0)
+				fListener->BytesWritten(this, written);
+			if (error != B_OK)
+				return error;
+			transferredSize += written;
+			if (fListener != NULL)
+				fListener->DownloadProgress(this, transferredSize, 0);
+		}
+
+		if (!fQuit)
+			fResult.SetLength(transferredSize);
+	}
+
+	return fQuit ? B_INTERRUPTED : B_OK;
+}
+
+#endif // LIBNETAPI_DEPRECATED

src/kits/network/libnetservices/Geolocation.cpp

@@ -40,17 +40,11 @@ class GeolocationListener: public BUrlProtocolListener
 			pthread_mutex_lock(&fLock);
 		}
 
-		void DataReceived(BUrlRequest*, const char* data, off_t position,
-					ssize_t size) {
-			fResult.WriteAt(position, data, size);
-		}
-
 		void RequestCompleted(BUrlRequest* caller, bool success) {
 			pthread_cond_signal(&fCompletion);
 			pthread_mutex_unlock(&fLock);
 		}
 
-		BMallocIO fResult;
 		pthread_cond_t fCompletion;
 		pthread_mutex_t fLock;
 };
@@ -117,10 +111,11 @@ BGeolocation::LocateSelf(float& latitude, float& longitude)
 		return B_DEVICE_NOT_FOUND;
 
 	GeolocationListener listener;
+	BMallocIO resultBuffer;
 
 	// Send Request (POST JSON message)
 	BUrlRequest* request = BUrlProtocolRoster::MakeRequest(fGeolocationService,
-		&listener);
+		&resultBuffer, &listener);
 	if (request == NULL)
 		return B_BAD_DATA;
 
@@ -153,7 +148,7 @@ BGeolocation::LocateSelf(float& latitude, float& longitude)
 	}
 
 	BMessage data;
-	result = BJson::Parse((char*)listener.fResult.Buffer(), data);
+	result = BJson::Parse((char*)resultBuffer.Buffer(), data);
 	delete http;
 	if (result != B_OK) {
 		return result;
@@ -192,8 +187,9 @@ BGeolocation::Country(const float latitude, const float longitude,
 	url.SetRequest(requestString);
 
 	GeolocationListener listener;
+	BMallocIO resultBuffer;
 	BUrlRequest* request = BUrlProtocolRoster::MakeRequest(url,
-		&listener);
+		&resultBuffer, &listener);
 	if (request == NULL)
 		return B_BAD_DATA;
 
@@ -223,9 +219,9 @@ BGeolocation::Country(const float latitude, const float longitude,
 	}
 
 	off_t length = 0;
-	listener.fResult.GetSize(&length);
+	resultBuffer.GetSize(&length);
 	length -= 2; // Remove \r\n from response
-	BString countryCode((char*)listener.fResult.Buffer(), (int32)length);
+	BString countryCode((char*)resultBuffer.Buffer(), (int32)length);
 	return country.SetTo(countryCode);
 }
 

src/kits/network/libnetservices/GopherRequest.cpp

@@ -183,6 +183,7 @@ static const int32 kGopherBufferSize = 4096;
 static const bool kInlineImages = true;
 
 
+#ifdef LIBNETAPI_DEPRECATED
 BGopherRequest::BGopherRequest(const BUrl& url, BUrlProtocolListener* listener,
 	BUrlContext* context)
 	:
@@ -207,6 +208,34 @@ BGopherRequest::BGopherRequest(const BUrl& url, BUrlProtocolListener* listener,
 	}
 }
 
+#else
+
+BGopherRequest::BGopherRequest(const BUrl& url, BDataIO* output,
+	BUrlProtocolListener* listener, BUrlContext* context)
+	:
+	BNetworkRequest(url, output, listener, context, "BUrlProtocol.Gopher",
+		"gopher"),
+	fItemType(GOPHER_TYPE_NONE),
+	fPosition(0),
+	fResult()
+{
+	fSocket = new(std::nothrow) BSocket();
+
+	fUrl.UrlDecode();
+	// the first part of the path is actually the document type
+
+	fPath = Url().Path();
+	if (!Url().HasPath() || fPath.Length() == 0 || fPath == "/") {
+		// default entry
+		fItemType = GOPHER_TYPE_DIRECTORY;
+		fPath = "";
+	} else if (fPath.Length() > 1 && fPath[0] == '/') {
+		fItemType = fPath[1];
+		fPath.Remove(0, 2);
+	}
+}
+#endif // LIBNETAPI_DEPRECATED
+
 
 BGopherRequest::~BGopherRequest()
 {
@@ -234,6 +263,165 @@ BGopherRequest::Result() const
 }
 
 
+#ifdef LIBNETAPI_DEPRECATED
+status_t
+BGopherRequest::_ProtocolLoop()
+{
+	if (fSocket == NULL)
+		return B_NO_MEMORY;
+
+	if (!_ResolveHostName(fUrl.Host(), fUrl.HasPort() ? fUrl.Port() : 70)) {
+		_EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR,
+			"Unable to resolve hostname (%s), aborting.",
+				fUrl.Host().String());
+		return B_SERVER_NOT_FOUND;
+	}
+
+	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Connection to %s on port %d.",
+		fUrl.Authority().String(), fRemoteAddr.Port());
+	status_t connectError = fSocket->Connect(fRemoteAddr);
+
+	if (connectError != B_OK) {
+		_EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR, "Socket connection error %s",
+			strerror(connectError));
+		return connectError;
+	}
+
+	//! ProtocolHook:ConnectionOpened
+	if (fListener != NULL)
+		fListener->ConnectionOpened(this);
+
+	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
+		"Connection opened, sending request.");
+
+	_SendRequest();
+	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Request sent.");
+
+	// Receive loop
+	bool receiveEnd = false;
+	status_t readError = B_OK;
+	ssize_t bytesRead = 0;
+	//ssize_t bytesReceived = 0;
+	//ssize_t bytesTotal = 0;
+	bool dataValidated = false;
+	BStackOrHeapArray<char, 4096> chunk(kGopherBufferSize);
+
+	while (!fQuit && !receiveEnd) {
+		bytesRead = fSocket->Read(chunk, kGopherBufferSize);
+
+		if (bytesRead < 0) {
+			readError = bytesRead;
+			break;
+		} else if (bytesRead == 0)
+			receiveEnd = true;
+
+		fInputBuffer.AppendData(chunk, bytesRead);
+
+		if (!dataValidated) {
+			size_t i;
+			// on error (file doesn't exist, ...) the server sends
+			// a faked directory entry with an error message
+			if (fInputBuffer.Size() && fInputBuffer.Data()[0] == '3') {
+				int tabs = 0;
+				bool crlf = false;
+
+				// make sure the buffer only contains printable characters
+				// and has at least 3 tabs before a CRLF
+				for (i = 0; i < fInputBuffer.Size(); i++) {
+					char c = fInputBuffer.Data()[i];
+					if (c == '\t') {
+						if (!crlf)
+							tabs++;
+					} else if (c == '\r' || c == '\n') {
+						if (tabs < 3)
+							break;
+						crlf = true;
+					} else if (!isprint(fInputBuffer.Data()[i])) {
+						crlf = false;
+						break;
+					}
+				}
+				if (crlf && tabs > 2 && tabs < 5) {
+					// TODO:
+					//if enough data
+					// else continue
+					fItemType = GOPHER_TYPE_DIRECTORY;
+					readError = B_RESOURCE_NOT_FOUND;
+					// continue parsing the error text anyway
+				}
+			}
+			// special case for buggy(?) Gophernicus/1.5
+			static const char *buggy = "Error: File or directory not found!";
+			if (fInputBuffer.Size() > strlen(buggy)
+				&& !memcmp(fInputBuffer.Data(), buggy, strlen(buggy))) {
+				fItemType = GOPHER_TYPE_DIRECTORY;
+				readError = B_RESOURCE_NOT_FOUND;
+				// continue parsing the error text anyway
+				// but it won't look good
+			}
+
+			// now we probably have correct data
+			dataValidated = true;
+
+			//! ProtocolHook:ResponseStarted
+			if (fListener != NULL)
+				fListener->ResponseStarted(this);
+
+			// now we can assign MIME type if we know it
+			const char *mime = "application/octet-stream";
+			for (i = 0; gopher_type_map[i].type != GOPHER_TYPE_NONE; i++) {
+				if (gopher_type_map[i].type == fItemType) {
+					mime = gopher_type_map[i].mime;
+					break;
+				}
+			}
+			fResult.SetContentType(mime);
+
+			// we don't really have headers but well...
+			//! ProtocolHook:HeadersReceived
+			if (fListener != NULL)
+				fListener->HeadersReceived(this, fResult);
+		}
+
+		if (_NeedsParsing())
+			_ParseInput(receiveEnd);
+		else if (fInputBuffer.Size()) {
+			// send input directly
+			if (fListener != NULL) {
+				fListener->DataReceived(this, (const char *)fInputBuffer.Data(),
+					fPosition, fInputBuffer.Size());
+			}
+
+			fPosition += fInputBuffer.Size();
+
+			if (fListener != NULL)
+				fListener->DownloadProgress(this, fPosition, 0);
+
+			// XXX: this is plain stupid, we already copied the data
+			// and just want to drop it...
+			char *inputTempBuffer = new(std::nothrow) char[bytesRead];
+			if (inputTempBuffer == NULL) {
+				readError = B_NO_MEMORY;
+				break;
+			}
+			fInputBuffer.RemoveData(inputTempBuffer, fInputBuffer.Size());
+			delete[] inputTempBuffer;
+		}
+	}
+
+	if (fPosition > 0)
+		fResult.SetLength(fPosition);
+
+	fSocket->Disconnect();
+
+	if (readError != B_OK)
+		return readError;
+
+	return fQuit ? B_INTERRUPTED : B_OK;
+}
+
+# else
+
 status_t
 BGopherRequest::_ProtocolLoop()
 {
@@ -350,20 +538,22 @@ BGopherRequest::_ProtocolLoop()
 			// we don't really have headers but well...
 			//! ProtocolHook:HeadersReceived
 			if (fListener != NULL)
-#ifdef LIBNETAPI_DEPRECATED
-				fListener->HeadersReceived(this, fResult);
-#else
 				fListener->HeadersReceived(this);
-#endif
 		}
 
 		if (_NeedsParsing())
-			_ParseInput(receiveEnd);
+			readError = _ParseInput(receiveEnd);
 		else if (fInputBuffer.Size()) {
 			// send input directly
-			if (fListener != NULL) {
-				fListener->DataReceived(this, (const char *)fInputBuffer.Data(),
-					fPosition, fInputBuffer.Size());
+			if (fOutput != NULL) {
+				size_t written = 0;
+				readError = fOutput->WriteExactly(
+					(const char*)fInputBuffer.Data(), fInputBuffer.Size(),
+					&written);
+				if (fListener != NULL && written > 0)
+					fListener->BytesWritten(this, written);
+				if (readError != B_OK)
+					break;
 			}
 
 			fPosition += fInputBuffer.Size();
@@ -393,6 +583,7 @@ BGopherRequest::_ProtocolLoop()
 
 	return fQuit ? B_INTERRUPTED : B_OK;
 }
+#endif // LIBNETAPI_DEPRECATED
 
 
 void
@@ -432,6 +623,7 @@ BGopherRequest::_NeedsLastDotStrip()
 }
 
 
+#ifdef LIBNETAPI_DEPRECATED
 void
 BGopherRequest::_ParseInput(bool last)
 {
@@ -719,6 +911,313 @@ BGopherRequest::_ParseInput(bool last)
 	}
 }
 
+#else
+
+status_t
+BGopherRequest::_ParseInput(bool last)
+{
+	BString line;
+
+	while (_GetLine(line) == B_OK) {
+		char type = GOPHER_TYPE_NONE;
+		BStringList fields;
+
+		line.MoveInto(&type, 0, 1);
+
+		line.Split("\t", false, fields);
+
+		if (type != GOPHER_TYPE_ENDOFPAGE
+			&& fields.CountStrings() < FIELD_GPFLAG)
+			_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
+				"Unterminated gopher item (type '%c')", type);
+
+		BString pageTitle;
+		BString item;
+		BString title = fields.StringAt(FIELD_NAME);
+		BString link("gopher://");
+		BString user;
+		if (fields.CountStrings() > 3) {
+			link << fields.StringAt(FIELD_HOST);
+			if (fields.StringAt(FIELD_PORT).Length())
+				link << ":" << fields.StringAt(FIELD_PORT);
+			link << "/" << type;
+			//if (fields.StringAt(FIELD_SELECTOR).ByteAt(0) != '/')
+			//	link << "/";
+			link << fields.StringAt(FIELD_SELECTOR);
+		}
+		_HTMLEscapeString(title);
+		_HTMLEscapeString(link);
+
+		switch (type) {
+			case GOPHER_TYPE_ENDOFPAGE:
+				/* end of the page */
+				break;
+			case GOPHER_TYPE_TEXTPLAIN:
+				item << "<a href=\"" << link << "\">"
+						"<span class=\"text\">" << title << "</span></a>"
+						"<br/>\n";
+				break;
+			case GOPHER_TYPE_BINARY:
+			case GOPHER_TYPE_BINHEX:
+			case GOPHER_TYPE_BINARCHIVE:
+			case GOPHER_TYPE_UUENCODED:
+				item << "<a href=\"" << link << "\">"
+						"<span class=\"binary\">" << title << "</span></a>"
+						"<br/>\n";
+				break;
+			case GOPHER_TYPE_DIRECTORY:
+				/*
+				 * directory link
+				 */
+				item << "<a href=\"" << link << "\">"
+						"<span class=\"dir\">" << title << "</span></a>"
+						"<br/>\n";
+				break;
+			case GOPHER_TYPE_ERROR:
+				item << "<span class=\"error\">" << title << "</span>"
+						"<br/>\n";
+				if (fPosition == 0 && pageTitle.Length() == 0)
+					pageTitle << "Error: " << title;
+				break;
+			case GOPHER_TYPE_QUERY:
+				/* TODO: handle search better.
+				 * For now we use an unnamed input field and accept sending ?=foo
+				 * as it seems at least Veronica-2 ignores the = but it's unclean.
+				 */
+				item << "<form method=\"get\" action=\"" << link << "\" "
+							"onsubmit=\"window.location = this.action + '?' + "
+								"this.elements['q'].value; return false;\">"
+						"<span class=\"query\">"
+						"<label>" << title << " "
+						"<input id=\"q\" name=\"\" type=\"text\" align=\"right\" />"
+						"</label>"
+						"</span></form>"
+						"<br/>\n";
+				break;
+			case GOPHER_TYPE_TELNET:
+				/* telnet: links
+				 * cf. gopher://78.80.30.202/1/ps3
+				 * -> gopher://78.80.30.202:23/8/ps3/new -> [email protected]
+				 */
+				link = "telnet://";
+				user = fields.StringAt(FIELD_SELECTOR);
+				if (user.FindLast('/') > -1) {
+					user.Remove(0, user.FindLast('/'));
+					link << user << "@";
+				}
+				link << fields.StringAt(FIELD_HOST);
+				if (fields.StringAt(FIELD_PORT) != "23")
+					link << ":" << fields.StringAt(FIELD_PORT);
+
+				item << "<a href=\"" << link << "\">"
+						"<span class=\"telnet\">" << title << "</span></a>"
+						"<br/>\n";
+				break;
+			case GOPHER_TYPE_TN3270:
+				/* tn3270: URI scheme, cf. http://tools.ietf.org/html/rfc6270 */
+				link = "tn3270://";
+				user = fields.StringAt(FIELD_SELECTOR);
+				if (user.FindLast('/') > -1) {
+					user.Remove(0, user.FindLast('/'));
+					link << user << "@";
+				}
+				link << fields.StringAt(FIELD_HOST);
+				if (fields.StringAt(FIELD_PORT) != "23")
+					link << ":" << fields.StringAt(FIELD_PORT);
+
+				item << "<a href=\"" << link << "\">"
+						"<span class=\"telnet\">" << title << "</span></a>"
+						"<br/>\n";
+				break;
+			case GOPHER_TYPE_CSO_SEARCH:
+				/* CSO search.
+				 * At least Lynx supports a cso:// URI scheme:
+				 * http://lynx.isc.org/lynx2.8.5/lynx2-8-5/lynx_help/lynx_url_support.html
+				 */
+				link = "cso://";
+				user = fields.StringAt(FIELD_SELECTOR);
+				if (user.FindLast('/') > -1) {
+					user.Remove(0, user.FindLast('/'));
+					link << user << "@";
+				}
+				link << fields.StringAt(FIELD_HOST);
+				if (fields.StringAt(FIELD_PORT) != "105")
+					link << ":" << fields.StringAt(FIELD_PORT);
+
+				item << "<a href=\"" << link << "\">"
+						"<span class=\"cso\">" << title << "</span></a>"
+						"<br/>\n";
+				break;
+			case GOPHER_TYPE_GIF:
+			case GOPHER_TYPE_IMAGE:
+			case GOPHER_TYPE_PNG:
+			case GOPHER_TYPE_BITMAP:
+				/* quite dangerous, cf. gopher://namcub.accela-labs.com/1/pics */
+				if (kInlineImages) {
+					item << "<a href=\"" << link << "\">"
+							"<span class=\"img\">" << title << " "
+							"<img src=\"" << link << "\" "
+								"alt=\"" << title << "\"/>"
+							"</span></a>"
+							"<br/>\n";
+					break;
+				}
+				/* fallback to default, link them */
+				item << "<a href=\"" << link << "\">"
+						"<span class=\"img\">" << title << "</span></a>"
+						"<br/>\n";
+				break;
+			case GOPHER_TYPE_HTML:
+				/* cf. gopher://pineapple.vg/1 */
+				if (fields.StringAt(FIELD_SELECTOR).StartsWith("URL:")) {
+					link = fields.StringAt(FIELD_SELECTOR);
+					link.Remove(0, 4);
+				}
+				/* cf. gopher://sdf.org/1/sdf/classes/ */
+
+				item << "<a href=\"" << link << "\">"
+						"<span class=\"html\">" << title << "</span></a>"
+						"<br/>\n";
+				break;
+			case GOPHER_TYPE_INFO:
+				// TITLE resource, cf.
+				// gopher://gophernicus.org/0/doc/gopher/gopher-title-resource.txt
+				if (fPosition == 0 && pageTitle.Length() == 0
+					&& fields.StringAt(FIELD_SELECTOR) == "TITLE") {
+						pageTitle = title;
+						break;
+				}
+				item << "<span class=\"info\">" << title << "</span>"
+						"<br/>\n";
+				break;
+			case GOPHER_TYPE_AUDIO:
+			case GOPHER_TYPE_SOUND:
+				item << "<a href=\"" << link << "\">"
+						"<span class=\"audio\">" << title << "</span></a>"
+						"<audio src=\"" << link << "\" "
+							//TODO:Fix crash in WebPositive with these
+							//"controls=\"controls\" "
+							//"width=\"300\" height=\"50\" "
+							"alt=\"" << title << "\"/>"
+						"<span>[player]</span></audio>"
+						"<br/>\n";
+				break;
+			case GOPHER_TYPE_PDF:
+			case GOPHER_TYPE_DOC:
+				/* generic case for known-to-work items */
+				item << "<a href=\"" << link << "\">"
+						"<span class=\"document\">" << title << "</span></a>"
+						"<br/>\n";
+				break;
+			case GOPHER_TYPE_MOVIE:
+				item << "<a href=\"" << link << "\">"
+						"<span class=\"video\">" << title << "</span></a>"
+						"<video src=\"" << link << "\" "
+							//TODO:Fix crash in WebPositive with these
+							//"controls=\"controls\" "
+							//"width=\"300\" height=\"300\" "
+							"alt=\"" << title << "\"/>"
+						"<span>[player]</span></audio>"
+						"<br/>\n";
+				break;
+			default:
+				_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
+					"Unknown gopher item (type 0x%02x '%c')", type, type);
+				item << "<a href=\"" << link << "\">"
+						"<span class=\"unknown\">" << title << "</span></a>"
+						"<br/>\n";
+				break;
+		}
+
+		if (fPosition == 0) {
+			if (pageTitle.Length() == 0)
+				pageTitle << "Index of " << Url();
+
+			const char *uplink = ".";
+			if (fPath.EndsWith("/"))
+				uplink = "..";
+
+			// emit header
+			BString header;
+			header <<
+				"<html>\n"
+				"<head>\n"
+				"<meta http-equiv=\"Content-Type\""
+					" content=\"text/html; charset=UTF-8\" />\n"
+				//FIXME: fix links
+				//"<link rel=\"icon\" type=\"image/png\""
+				//	" href=\"resource:icons/directory.png\">\n"
+				"<style type=\"text/css\">\n" << kStyleSheet << "</style>\n"
+				"<title>" << pageTitle << "</title>\n"
+				"</head>\n"
+				"<body id=\"gopher\">\n"
+				"<div class=\"uplink dontprint\">\n"
+				"<a href=" << uplink << ">[up]</a>\n"
+				"<a href=\"/\">[top]</a>\n"
+				"</div>\n"
+				"<h1>" << pageTitle << "</h1>\n";
+
+			if (fOutput != NULL) {
+				size_t written = 0;
+				status_t error = fOutput->WriteExactly(header.String(),
+					header.Length(), &written);
+				if (fListener != NULL && written > 0)
+					fListener->BytesWritten(this, written);
+				if (error != B_OK)
+					return error;
+			}
+
+			fPosition += header.Length();
+
+			if (fListener != NULL)
+				fListener->DownloadProgress(this, fPosition, 0);
+		}
+
+		if (item.Length()) {
+			if (fOutput != NULL) {
+				size_t written = 0;
+				status_t error = fOutput->WriteExactly(item.String(),
+					item.Length(), &written);
+				if (fListener != NULL && written > 0)
+					fListener->BytesWritten(this, written);
+				if (error != B_OK)
+					return error;
+			}
+
+			fPosition += item.Length();
+
+			if (fListener != NULL)
+				fListener->DownloadProgress(this, fPosition, 0);
+		}
+	}
+
+	if (last) {
+		// emit footer
+		BString footer =
+			"</div>\n"
+			"</body>\n"
+			"</html>\n";
+
+		if (fListener != NULL) {
+			size_t written = 0;
+			status_t error = fOutput->WriteExactly(footer.String(),
+				footer.Length(), &written);
+			if (fListener != NULL && written > 0)
+				fListener->BytesWritten(this, written);
+			if (error != B_OK)
+				return error;
+		}
+
+		fPosition += footer.Length();
+
+		if (fListener != NULL)
+			fListener->DownloadProgress(this, fPosition, 0);
+	}
+
+	return B_OK;
+}
+#endif
+
 
 BString&
 BGopherRequest::_HTMLEscapeString(BString &str)

src/kits/network/libnetservices/HttpRequest.cpp

@@ -101,6 +101,7 @@ namespace BPrivate {
 };
 
 
+#ifdef LIBNETAPI_DEPRECATED
 BHttpRequest::BHttpRequest(const BUrl& url, bool ssl, const char* protocolName,
 	BUrlProtocolListener* listener, BUrlContext* context)
 	:
@@ -145,6 +146,55 @@ BHttpRequest::BHttpRequest(const BHttpRequest& other)
 	fSocket = NULL;
 }
 
+#else
+
+BHttpRequest::BHttpRequest(const BUrl& url, BDataIO* output, bool ssl,
+	const char* protocolName, BUrlProtocolListener* listener,
+	BUrlContext* context)
+	:
+	BNetworkRequest(url, output, listener, context, "BUrlProtocol.HTTP",
+		protocolName),
+	fSSL(ssl),
+	fRequestMethod(B_HTTP_GET),
+	fHttpVersion(B_HTTP_11),
+	fResult(url),
+	fRequestStatus(kRequestInitialState),
+	fOptHeaders(NULL),
+	fOptPostFields(NULL),
+	fOptInputData(NULL),
+	fOptInputDataSize(-1),
+	fOptRangeStart(-1),
+	fOptRangeEnd(-1),
+	fOptFollowLocation(true)
+{
+	_ResetOptions();
+	fSocket = NULL;
+}
+
+
+BHttpRequest::BHttpRequest(const BHttpRequest& other)
+	:
+	BNetworkRequest(other.Url(), other.Output(), other.fListener,
+		other.fContext, "BUrlProtocol.HTTP", other.fSSL ? "HTTPS" : "HTTP"),
+	fSSL(other.fSSL),
+	fRequestMethod(other.fRequestMethod),
+	fHttpVersion(other.fHttpVersion),
+	fResult(other.fUrl),
+	fRequestStatus(kRequestInitialState),
+	fOptHeaders(NULL),
+	fOptPostFields(NULL),
+	fOptInputData(NULL),
+	fOptInputDataSize(-1),
+	fOptRangeStart(other.fOptRangeStart),
+	fOptRangeEnd(other.fOptRangeEnd),
+	fOptFollowLocation(other.fOptFollowLocation)
+{
+	_ResetOptions();
+		// FIXME some options may be copied from other instead.
+	fSocket = NULL;
+}
+#endif // LIBNETAPI_DEPRECATED
+
 
 BHttpRequest::~BHttpRequest()
 {
@@ -214,6 +264,15 @@ BHttpRequest::SetAutoReferrer(bool enable)
 }
 
 
+#ifndef LIBNETAPI_DEPRECATED
+void
+BHttpRequest::SetStopOnError(bool stop)
+{
+	fOptStopOnError = stop;
+}
+#endif
+
+
 void
 BHttpRequest::SetUserName(const BString& name)
 {
@@ -543,6 +602,7 @@ BHttpRequest::_ProtocolLoop()
 }
 
 
+#ifdef LIBNETAPI_DEPRECATED
 status_t
 BHttpRequest::_MakeRequest()
 {
@@ -641,13 +701,6 @@ BHttpRequest::_MakeRequest()
 		if (fRequestStatus < kRequestStatusReceived) {
 			_ParseStatus();
 
-#ifndef LIBNETAPI_DEPRECATED
-			// Deprecated behavior is to not disable the listener on redirect
-			if (fOptFollowLocation
-					&& IsRedirectionStatusCode(fResult.StatusCode()))
-				disableListener = true;
-#endif
-
 			//! ProtocolHook:ResponseStarted
 			if (fRequestStatus >= kRequestStatusReceived && fListener != NULL
 					&& !disableListener)
@@ -672,11 +725,7 @@ BHttpRequest::_MakeRequest()
 
 				//! ProtocolHook:HeadersReceived
 				if (fListener != NULL && !disableListener)
-#ifdef LIBNETAPI_DEPRECATED
 					fListener->HeadersReceived(this, fResult);
-#else
-					fListener->HeadersReceived(this);
-#endif
 
 
 				if (BString(fHeaders["Transfer-Encoding"]) == "chunked")
@@ -836,6 +885,322 @@ BHttpRequest::_MakeRequest()
 	return fQuit ? B_INTERRUPTED : B_OK;
 }
 
+#else
+
+status_t
+BHttpRequest::_MakeRequest()
+{
+	delete fSocket;
+
+	if (fSSL) {
+		if (fContext->UseProxy()) {
+			BNetworkAddress proxy(fContext->GetProxyHost(), fContext->GetProxyPort());
+			fSocket = new(std::nothrow) BPrivate::CheckedProxySecureSocket(proxy, this);
+		} else
+			fSocket = new(std::nothrow) BPrivate::CheckedSecureSocket(this);
+	} else
+		fSocket = new(std::nothrow) BSocket();
+
+	if (fSocket == NULL)
+		return B_NO_MEMORY;
+
+	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Connection to %s on port %d.",
+		fUrl.Authority().String(), fRemoteAddr.Port());
+	status_t connectError = fSocket->Connect(fRemoteAddr);
+
+	if (connectError != B_OK) {
+		_EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR, "Socket connection error %s",
+			strerror(connectError));
+		return connectError;
+	}
+
+	//! ProtocolHook:ConnectionOpened
+	if (fListener != NULL)
+		fListener->ConnectionOpened(this);
+
+	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
+		"Connection opened, sending request.");
+
+	BString requestHeaders;
+	requestHeaders.Append(_SerializeRequest());
+	requestHeaders.Append(_SerializeHeaders());
+	requestHeaders.Append("\r\n");
+	fSocket->Write(requestHeaders.String(), requestHeaders.Length());
+	_EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Request sent.");
+
+	_SendPostData();
+	fRequestStatus = kRequestInitialState;
+
+
+
+	// Receive loop
+	bool disableListener = false;
+	bool receiveEnd = false;
+	bool parseEnd = false;
+	bool readByChunks = false;
+	bool decompress = false;
+	status_t readError = B_OK;
+	ssize_t bytesRead = 0;
+	off_t bytesReceived = 0;
+	off_t bytesTotal = 0;
+	size_t previousBufferSize = 0;
+	off_t bytesUnpacked = 0;
+	char* inputTempBuffer = new(std::nothrow) char[kHttpBufferSize];
+	ArrayDeleter<char> inputTempBufferDeleter(inputTempBuffer);
+	ssize_t inputTempSize = kHttpBufferSize;
+	ssize_t chunkSize = -1;
+	DynamicBuffer decompressorStorage;
+	BDataIO* decompressingStream;
+	ObjectDeleter<BDataIO> decompressingStreamDeleter;
+
+	while (!fQuit && !(receiveEnd && parseEnd)) {
+		if ((!receiveEnd) && (fInputBuffer.Size() == previousBufferSize)) {
+			BStackOrHeapArray<char, 4096> chunk(kHttpBufferSize);
+			bytesRead = fSocket->Read(chunk, kHttpBufferSize);
+
+			if (bytesRead < 0) {
+				readError = bytesRead;
+				break;
+			} else if (bytesRead == 0) {
+				// Check if we got the expected number of bytes.
+				// Exceptions:
+				// - If the content-length is not known (bytesTotal is 0), for
+				//   example in the case of a chunked transfer, we can't know
+				// - If the request method is "HEAD" which explicitly asks the
+				//   server to not send any data (only the headers)
+				if (bytesTotal > 0 && bytesReceived != bytesTotal
+					&& fRequestMethod != B_HTTP_HEAD) {
+					readError = B_IO_ERROR;
+					break;
+				}
+				receiveEnd = true;
+			}
+
+			fInputBuffer.AppendData(chunk, bytesRead);
+		} else
+			bytesRead = 0;
+
+		previousBufferSize = fInputBuffer.Size();
+
+		if (fRequestStatus < kRequestStatusReceived) {
+			_ParseStatus();
+
+			if (fOptFollowLocation
+					&& IsRedirectionStatusCode(fResult.StatusCode()))
+				disableListener = true;
+
+			if (fOptStopOnError
+					&& fResult.StatusCode() >= B_HTTP_STATUS_CLASS_CLIENT_ERROR)
+			{
+				fQuit = true;
+				break;
+			}
+
+			//! ProtocolHook:ResponseStarted
+			if (fRequestStatus >= kRequestStatusReceived && fListener != NULL
+					&& !disableListener)
+				fListener->ResponseStarted(this);
+		}
+
+		if (fRequestStatus < kRequestHeadersReceived) {
+			_ParseHeaders();
+
+			if (fRequestStatus >= kRequestHeadersReceived) {
+				_ResultHeaders() = fHeaders;
+
+				// Parse received cookies
+				if (fContext != NULL) {
+					for (int32 i = 0;  i < fHeaders.CountHeaders(); i++) {
+						if (fHeaders.HeaderAt(i).NameIs("Set-Cookie")) {
+							fContext->GetCookieJar().AddCookie(
+								fHeaders.HeaderAt(i).Value(), fUrl);
+						}
+					}
+				}
+
+				//! ProtocolHook:HeadersReceived
+				if (fListener != NULL && !disableListener)
+					fListener->HeadersReceived(this);
+
+
+				if (BString(fHeaders["Transfer-Encoding"]) == "chunked")
+					readByChunks = true;
+
+				BString contentEncoding(fHeaders["Content-Encoding"]);
+				// We don't advertise "deflate" support (see above), but we
+				// still try to decompress it, if a server ever sends a deflate
+				// stream despite it not being in our Accept-Encoding list.
+				if (contentEncoding == "gzip"
+						|| contentEncoding == "deflate") {
+					decompress = true;
+					readError = BZlibCompressionAlgorithm()
+						.CreateDecompressingOutputStream(&decompressorStorage,
+							NULL, decompressingStream);
+					if (readError != B_OK)
+						break;
+
+					decompressingStreamDeleter.SetTo(decompressingStream);
+				}
+
+				int32 index = fHeaders.HasHeader("Content-Length");
+				if (index != B_ERROR)
+					bytesTotal = atoll(fHeaders.HeaderAt(index).Value());
+				else
+					bytesTotal = -1;
+			}
+		}
+
+		if (fRequestStatus >= kRequestHeadersReceived) {
+			// If Transfer-Encoding is chunked, we should read a complete
+			// chunk in buffer before handling it
+			if (readByChunks) {
+				if (chunkSize >= 0) {
+					if ((ssize_t)fInputBuffer.Size() >= chunkSize + 2) {
+							// 2 more bytes to handle the closing CR+LF
+						bytesRead = chunkSize;
+						if (inputTempSize < chunkSize + 2) {
+							inputTempSize = chunkSize + 2;
+							inputTempBuffer
+								= new(std::nothrow) char[inputTempSize];
+							inputTempBufferDeleter.SetTo(inputTempBuffer);
+						}
+
+						if (inputTempBuffer == NULL) {
+							readError = B_NO_MEMORY;
+							break;
+						}
+
+						fInputBuffer.RemoveData(inputTempBuffer,
+							chunkSize + 2);
+						chunkSize = -1;
+					} else {
+						// Not enough data, try again later
+						bytesRead = -1;
+					}
+				} else {
+					BString chunkHeader;
+					if (_GetLine(chunkHeader) == B_ERROR) {
+						chunkSize = -1;
+						bytesRead = -1;
+					} else {
+						// Format of a chunk header:
+						// <chunk size in hex>[; optional data]
+						int32 semiColonIndex = chunkHeader.FindFirst(';', 0);
+
+						// Cut-off optional data if present
+						if (semiColonIndex != -1) {
+							chunkHeader.Remove(semiColonIndex,
+								chunkHeader.Length() - semiColonIndex);
+						}
+
+						chunkSize = strtol(chunkHeader.String(), NULL, 16);
+						if (chunkSize == 0)
+							fRequestStatus = kRequestContentReceived;
+
+						bytesRead = -1;
+					}
+				}
+
+				// A chunk of 0 bytes indicates the end of the chunked transfer
+				if (bytesRead == 0)
+					receiveEnd = true;
+			} else {
+				bytesRead = fInputBuffer.Size();
+
+				if (bytesRead > 0) {
+					if (inputTempSize < bytesRead) {
+						inputTempSize = bytesRead;
+						inputTempBuffer = new(std::nothrow) char[bytesRead];
+						inputTempBufferDeleter.SetTo(inputTempBuffer);
+					}
+
+					if (inputTempBuffer == NULL) {
+						readError = B_NO_MEMORY;
+						break;
+					}
+					fInputBuffer.RemoveData(inputTempBuffer, bytesRead);
+				}
+			}
+
+			if (bytesRead >= 0) {
+				bytesReceived += bytesRead;
+
+				if (fOutput != NULL && !disableListener) {
+					if (decompress) {
+						readError = decompressingStream->WriteExactly(
+							inputTempBuffer, bytesRead);
+						if (readError != B_OK)
+							break;
+
+						ssize_t size = decompressorStorage.Size();
+						BStackOrHeapArray<char, 4096> buffer(size);
+						size = decompressorStorage.Read(buffer, size);
+						if (size > 0) {
+							size_t written = 0;
+							readError = fOutput->WriteExactly(buffer,
+								size, &written);
+							if (fListener != NULL && written > 0)
+								fListener->BytesWritten(this, written);
+							if (readError != B_OK)
+								break;
+							bytesUnpacked += size;
+						}
+					} else if (bytesRead > 0) {
+						size_t written = 0;
+						readError = fOutput->WriteExactly(inputTempBuffer,
+							bytesRead, &written);
+						if (fListener != NULL && written > 0)
+							fListener->BytesWritten(this, written);
+						if (readError != B_OK)
+							break;
+					}
+				}
+
+				if (fListener != NULL && !disableListener)
+					fListener->DownloadProgress(this, bytesReceived,
+						std::max((off_t)0, bytesTotal));
+
+				if (bytesTotal >= 0 && bytesReceived >= bytesTotal)
+					receiveEnd = true;
+
+				if (decompress && receiveEnd && !disableListener) {
+					readError = decompressingStream->Flush();
+
+					if (readError == B_BUFFER_OVERFLOW)
+						readError = B_OK;
+
+					if (readError != B_OK)
+						break;
+
+					ssize_t size = decompressorStorage.Size();
+					BStackOrHeapArray<char, 4096> buffer(size);
+					size = decompressorStorage.Read(buffer, size);
+					if (fOutput != NULL && size > 0 && !disableListener) {
+						size_t written = 0;
+						readError = fOutput->WriteExactly(buffer, size,
+							&written);
+						if (fListener != NULL && written > 0)
+							fListener->BytesWritten(this, written);
+						if (readError != B_OK)
+							break;
+						bytesUnpacked += size;
+					}
+				}
+			}
+		}
+
+		parseEnd = (fInputBuffer.Size() == 0);
+	}
+
+	fSocket->Disconnect();
+
+	if (readError != B_OK)
+		return readError;
+
+	return fQuit ? B_INTERRUPTED : B_OK;
+}
+#endif // LIBNETAPI_DEPRECATED
+
 
 void
 BHttpRequest::_ParseStatus()
@@ -1220,6 +1585,7 @@ BHttpRequest::_IsDefaultPort()
 }
 
 
+#ifdef LIBNETAPI_DEPRECATED
 void
 BHttpRequest::_NotifyDataReceived(const char* data, off_t pos, ssize_t size,
 	off_t bytesReceived, ssize_t bytesTotal)
@@ -1246,3 +1612,4 @@ BHttpRequest::_NotifyDataReceived(const char* data, off_t pos, ssize_t size,
 	fListener->DownloadProgress(this, bytesReceived,
 		std::max((ssize_t)0, bytesTotal));
 }
+#endif

src/kits/network/libnetservices/NetworkRequest.cpp

@@ -18,6 +18,8 @@
 using namespace BPrivate::Network;
 #endif
 
+
+#ifdef LIBNETAPI_DEPRECATED
 BNetworkRequest::BNetworkRequest(const BUrl& url, BUrlProtocolListener* listener,
 		BUrlContext* context, const char* threadName, const char* protocolName)
 	:
@@ -26,6 +28,18 @@ BNetworkRequest::BNetworkRequest(const BUrl& url, BUrlProtocolListener* listener
 {
 }
 
+#else
+
+BNetworkRequest::BNetworkRequest(const BUrl& url, BDataIO* output,
+	BUrlProtocolListener* listener, BUrlContext* context,
+	const char* threadName, const char* protocolName)
+	:
+	BUrlRequest(url, output, listener, context, threadName, protocolName),
+	fSocket(NULL)
+{
+}
+#endif // LIBNETAPI_DEPRECATED
+
 
 status_t
 BNetworkRequest::Stop()

src/kits/network/libnetservices/UrlProtocolAsynchronousListener.cpp

@@ -116,11 +116,6 @@ BUrlProtocolAsynchronousListener::MessageReceived(BMessage* message)
 				HeadersReceived(caller, *result);
 				delete result;
 			}
-#else
-		case B_URL_PROTOCOL_HEADERS_RECEIVED:
-			HeadersReceived(caller);
-			break;
-#endif
 
 		case B_URL_PROTOCOL_DATA_RECEIVED:
 			{
@@ -139,7 +134,6 @@ BUrlProtocolAsynchronousListener::MessageReceived(BMessage* message)
 			}
 			break;
 
-#ifdef LIBNETAPI_DEPRECATED
 		case B_URL_PROTOCOL_DOWNLOAD_PROGRESS:
 			{
 				int32 bytesReceived;
@@ -162,6 +156,18 @@ BUrlProtocolAsynchronousListener::MessageReceived(BMessage* message)
 			}
 			break;
 #else
+		case B_URL_PROTOCOL_HEADERS_RECEIVED:
+			HeadersReceived(caller);
+			break;
+
+		case B_URL_PROTOCOL_BYTES_WRITTEN:
+			{
+				size_t bytesWritten = message->FindInt32("url:bytesWritten");
+
+				BytesWritten(caller, bytesWritten);
+			}
+			break;
+
 		case B_URL_PROTOCOL_DOWNLOAD_PROGRESS:
 			{
 				off_t bytesReceived;

src/kits/network/libnetservices/UrlProtocolDispatchingListener.cpp

@@ -26,16 +26,16 @@ const char* kUrlProtocolCaller = "be:urlProtocolCaller";
 
 BUrlProtocolDispatchingListener::BUrlProtocolDispatchingListener
 	(BHandler* handler)
-		:
-		fMessenger(handler)
+	:
+	fMessenger(handler)
 {
 }
 
 
 BUrlProtocolDispatchingListener::BUrlProtocolDispatchingListener
 	(const BMessenger& messenger)
-		:
-		fMessenger(messenger)
+	:
+	fMessenger(messenger)
 {
 }
 
@@ -59,7 +59,7 @@ BUrlProtocolDispatchingListener::HostnameResolved(BUrlRequest* caller,
 {
 	BMessage message(B_URL_PROTOCOL_NOTIFICATION);
 	message.AddString("url:hostIp", ip);
-	
+
 	_SendMessage(&message, B_URL_PROTOCOL_HOSTNAME_RESOLVED, caller);
 }
 
@@ -88,16 +88,6 @@ BUrlProtocolDispatchingListener::HeadersReceived(BUrlRequest* caller,
 	_SendMessage(&message, B_URL_PROTOCOL_HEADERS_RECEIVED, caller);
 }
 
-#else
-
-void
-BUrlProtocolDispatchingListener::HeadersReceived(BUrlRequest* caller)
-{
-	BMessage message(B_URL_PROTOCOL_NOTIFICATION);
-	_SendMessage(&message, B_URL_PROTOCOL_HEADERS_RECEIVED, caller);
-}
-#endif // LIBNETAPI_DEPRECATED
-
 
 void
 BUrlProtocolDispatchingListener::DataReceived(BUrlRequest* caller,
@@ -110,12 +100,11 @@ BUrlProtocolDispatchingListener::DataReceived(BUrlRequest* caller,
 
 	result = message.AddInt32("url:position", position);
 	assert(result == B_OK);
-	
+
 	_SendMessage(&message, B_URL_PROTOCOL_DATA_RECEIVED, caller);
 }
 
 
-#ifdef LIBNETAPI_DEPRECATED
 void
 BUrlProtocolDispatchingListener::DownloadProgress(BUrlRequest* caller,
 	ssize_t bytesReceived, ssize_t bytesTotal)
@@ -123,7 +112,7 @@ BUrlProtocolDispatchingListener::DownloadProgress(BUrlRequest* caller,
 	BMessage message(B_URL_PROTOCOL_NOTIFICATION);
 	message.AddInt32("url:bytesReceived", bytesReceived);
 	message.AddInt32("url:bytesTotal", bytesTotal);
-	
+
 	_SendMessage(&message, B_URL_PROTOCOL_DOWNLOAD_PROGRESS, caller);
 }
 
@@ -135,7 +124,7 @@ BUrlProtocolDispatchingListener::UploadProgress(BUrlRequest* caller,
 	BMessage message(B_URL_PROTOCOL_NOTIFICATION);
 	message.AddInt32("url:bytesSent", bytesSent);
 	message.AddInt32("url:bytesTotal", bytesTotal);
-	
+
 	_SendMessage(&message, B_URL_PROTOCOL_UPLOAD_PROGRESS, caller);
 }
 
@@ -143,6 +132,25 @@ BUrlProtocolDispatchingListener::UploadProgress(BUrlRequest* caller,
 #else
 
 
+void
+BUrlProtocolDispatchingListener::HeadersReceived(BUrlRequest* caller)
+{
+	BMessage message(B_URL_PROTOCOL_NOTIFICATION);
+	_SendMessage(&message, B_URL_PROTOCOL_HEADERS_RECEIVED, caller);
+}
+
+
+void
+BUrlProtocolDispatchingListener::BytesWritten(BUrlRequest* caller,
+	size_t bytesWritten)
+{
+	BMessage message(B_URL_PROTOCOL_NOTIFICATION);
+	message.AddInt32("url:bytesWritten", bytesWritten);
+
+	_SendMessage(&message, B_URL_PROTOCOL_BYTES_WRITTEN, caller);
+}
+
+
 void
 BUrlProtocolDispatchingListener::DownloadProgress(BUrlRequest* caller,
 	off_t bytesReceived, off_t bytesTotal)
@@ -150,7 +158,7 @@ BUrlProtocolDispatchingListener::DownloadProgress(BUrlRequest* caller,
 	BMessage message(B_URL_PROTOCOL_NOTIFICATION);
 	message.AddInt64("url:bytesReceived", bytesReceived);
 	message.AddInt64("url:bytesTotal", bytesTotal);
-	
+
 	_SendMessage(&message, B_URL_PROTOCOL_DOWNLOAD_PROGRESS, caller);
 }
 
@@ -162,20 +170,19 @@ BUrlProtocolDispatchingListener::UploadProgress(BUrlRequest* caller,
 	BMessage message(B_URL_PROTOCOL_NOTIFICATION);
 	message.AddInt64("url:bytesSent", bytesSent);
 	message.AddInt64("url:bytesTotal", bytesTotal);
-	
+
 	_SendMessage(&message, B_URL_PROTOCOL_UPLOAD_PROGRESS, caller);
 }
-
-
 #endif // LIBNETAPI_DEPRECATED
 
+
 void
 BUrlProtocolDispatchingListener::RequestCompleted(BUrlRequest* caller,
 	bool success)
 {
 	BMessage message(B_URL_PROTOCOL_NOTIFICATION);
 	message.AddBool("url:success", success);
-	
+
 	_SendMessage(&message, B_URL_PROTOCOL_REQUEST_COMPLETED, caller);
 }
 

src/kits/network/libnetservices/UrlProtocolListener.cpp

@@ -51,14 +51,6 @@ BUrlProtocolListener::HeadersReceived(BUrlRequest*, const BUrlResult& result)
 {
 }
 
-#else
-
-void
-BUrlProtocolListener::HeadersReceived(BUrlRequest*)
-{
-}
-#endif // LIBNETAPI_DEPRECATED
-
 
 void
 BUrlProtocolListener::DataReceived(BUrlRequest*, const char*, off_t, ssize_t)
@@ -66,7 +58,6 @@ BUrlProtocolListener::DataReceived(BUrlRequest*, const char*, off_t, ssize_t)
 }
 
 
-#ifdef LIBNETAPI_DEPRECATED
 void
 BUrlProtocolListener::DownloadProgress(BUrlRequest*, ssize_t, ssize_t)
 {
@@ -80,6 +71,18 @@ BUrlProtocolListener::UploadProgress(BUrlRequest*, ssize_t, ssize_t)
 
 #else
 
+void
+BUrlProtocolListener::HeadersReceived(BUrlRequest*)
+{
+}
+
+
+void
+BUrlProtocolListener::BytesWritten(BUrlRequest*, size_t)
+{
+}
+
+
 void
 BUrlProtocolListener::DownloadProgress(BUrlRequest*, off_t, off_t)
 {
@@ -108,22 +111,22 @@ BUrlProtocolListener::DebugMessage(BUrlRequest* caller,
 		case B_URL_PROTOCOL_DEBUG_TEXT:
 			cout << "   ";
 			break;
-			
+
 		case B_URL_PROTOCOL_DEBUG_ERROR:
 			cout << "!!!";
 			break;
-			
+
 		case B_URL_PROTOCOL_DEBUG_TRANSFER_IN:
 		case B_URL_PROTOCOL_DEBUG_HEADER_IN:
 			cout << "<--";
 			break;
-			
+
 		case B_URL_PROTOCOL_DEBUG_TRANSFER_OUT:
 		case B_URL_PROTOCOL_DEBUG_HEADER_OUT:
 			cout << "-->";
 			break;
 	}
-	
+
 	cout << " " << caller->Protocol() << ": " << text << endl;
 #endif
 }

src/kits/network/libnetservices/UrlProtocolRoster.cpp

@@ -22,6 +22,8 @@
 using namespace BPrivate::Network;
 #endif
 
+
+#ifdef LIBNETAPI_DEPRECATED
 /* static */ BUrlRequest*
 BUrlProtocolRoster::MakeRequest(const BUrl& url,
 	BUrlProtocolListener* listener, BUrlContext* context)
@@ -43,3 +45,28 @@ BUrlProtocolRoster::MakeRequest(const BUrl& url,
 
 	return NULL;
 }
+
+#else
+
+/* static */ BUrlRequest*
+BUrlProtocolRoster::MakeRequest(const BUrl& url, BDataIO* output,
+	BUrlProtocolListener* listener, BUrlContext* context)
+{
+	// TODO: instanciate the correct BUrlProtocol using add-on interface
+	if (url.Protocol() == "http") {
+		return new(std::nothrow) BHttpRequest(url, output, false, "HTTP",
+			listener, context);
+	} else if (url.Protocol() == "https") {
+		return new(std::nothrow) BHttpRequest(url, output, true, "HTTPS",
+			listener, context);
+	} else if (url.Protocol() == "file") {
+		return new(std::nothrow) BFileRequest(url, output, listener, context);
+	} else if (url.Protocol() == "data") {
+		return new(std::nothrow) BDataRequest(url, output, listener, context);
+	} else if (url.Protocol() == "gopher") {
+		return new(std::nothrow) BGopherRequest(url, output, listener, context);
+	}
+
+	return NULL;
+}
+#endif //LIBNETAPI_DEPRECATED

src/kits/network/libnetservices/UrlRequest.cpp

@@ -19,6 +19,7 @@ using namespace BPrivate::Network;
 static BReference<BUrlContext> gDefaultContext = new(std::nothrow) BUrlContext();
 
 
+#ifdef LIBNETAPI_DEPRECATED
 BUrlRequest::BUrlRequest(const BUrl& url, BUrlProtocolListener* listener,
 	BUrlContext* context, const char* threadName, const char* protocolName)
 	:
@@ -36,6 +37,28 @@ BUrlRequest::BUrlRequest(const BUrl& url, BUrlProtocolListener* listener,
 		fContext = gDefaultContext;
 }
 
+#else
+
+BUrlRequest::BUrlRequest(const BUrl& url, BDataIO* output,
+	BUrlProtocolListener* listener, BUrlContext* context,
+	const char* threadName, const char* protocolName)
+	:
+	fUrl(url),
+	fContext(context),
+	fListener(listener),
+	fOutput(output),
+	fQuit(false),
+	fRunning(false),
+	fThreadStatus(B_NO_INIT),
+	fThreadId(0),
+	fThreadName(threadName),
+	fProtocol(protocolName)
+{
+	if (fContext == NULL)
+		fContext = gDefaultContext;
+}
+#endif // LIBNETAPI_DEPRECATED
+
 
 BUrlRequest::~BUrlRequest()
 {
@@ -139,6 +162,19 @@ BUrlRequest::SetListener(BUrlProtocolListener* listener)
 }
 
 
+#ifndef LIBNETAPI_DEPRECATED
+status_t
+BUrlRequest::SetOutput(BDataIO* output)
+{
+	if (IsRunning())
+		return B_ERROR;
+
+	fOutput = output;
+	return B_OK;
+}
+#endif
+
+
 // #pragma mark URL protocol parameters access
 
 
@@ -170,6 +206,14 @@ BUrlRequest::Protocol() const
 }
 
 
+#ifndef LIBNETAPI_DEPRECATED
+BDataIO*
+BUrlRequest::Output() const
+{
+	return fOutput;
+}
+#endif
+
 // #pragma mark URL protocol informations
 
 

src/kits/network/libnetservices/UrlSynchronousRequest.cpp

@@ -16,6 +16,7 @@
 using namespace BPrivate::Network;
 #endif
 
+#ifdef LIBNETAPI_DEPRECATED
 BUrlSynchronousRequest::BUrlSynchronousRequest(BUrlRequest& request)
 	:
 	BUrlRequest(request.Url(), NULL, request.Context(),
@@ -25,6 +26,18 @@ BUrlSynchronousRequest::BUrlSynchronousRequest(BUrlRequest& request)
 {
 }
 
+#else
+
+BUrlSynchronousRequest::BUrlSynchronousRequest(BUrlRequest& request)
+	:
+	BUrlRequest(request.Url(), request.Output(), NULL, request.Context(),
+		"BUrlSynchronousRequest", request.Protocol()),
+	fRequestComplete(false),
+	fWrappedRequest(request)
+{
+}
+#endif // LIBNETAPI_DEPRECATED
+
 
 status_t
 BUrlSynchronousRequest::Perform()
@@ -80,15 +93,6 @@ BUrlSynchronousRequest::HeadersReceived(BUrlRequest*, const BUrlResult& result)
 	PRINT(("SynchronousRequest::HeadersReceived()\n"));
 }
 
-#else
-
-void
-BUrlSynchronousRequest::HeadersReceived(BUrlRequest*)
-{
-	PRINT(("SynchronousRequest::HeadersReceived()\n"));
-}
-#endif // LIBNETAPI_DEPRECATED
-
 
 void
 BUrlSynchronousRequest::DataReceived(BUrlRequest*, const char*,
@@ -98,7 +102,6 @@ BUrlSynchronousRequest::DataReceived(BUrlRequest*, const char*,
 }
 
 
-#ifdef LIBNETAPI_DEPRECATED
 void
 BUrlSynchronousRequest::DownloadProgress(BUrlRequest*,
 	ssize_t bytesReceived, ssize_t bytesTotal)
@@ -116,9 +119,22 @@ BUrlSynchronousRequest::UploadProgress(BUrlRequest*, ssize_t bytesSent,
 		bytesTotal));
 }
 
-
 #else
 
+void
+BUrlSynchronousRequest::HeadersReceived(BUrlRequest*)
+{
+	PRINT(("SynchronousRequest::HeadersReceived()\n"));
+}
+
+
+void
+BUrlSynchronousRequest::BytesWritten(BUrlRequest* caller, size_t bytesWritten)
+{
+	PRINT(("SynchronousRequest::BytesWritten(%" B_PRIdSSIZE ")\n",
+		bytesWritten));
+}
+
 
 void
 BUrlSynchronousRequest::DownloadProgress(BUrlRequest*,

src/kits/package/FetchFileJob.cpp

@@ -94,30 +94,10 @@ FetchFileJob::Execute()
 	if (result != B_OK)
 		return result;
 
-	result = FetchUtils::SetFileType(fTargetFile,
-		"application/x-vnd.haiku-package");
-	if (result != B_OK) {
-		fprintf(stderr, "failed to set file type for '%s': %s\n",
-			DownloadFileName(), strerror(result));
-	}
-
-	do {
-		BUrlRequest* request = BUrlProtocolRoster::MakeRequest(DownloadURL(),
-			this);
-		if (request == NULL)
-			return B_BAD_VALUE;
-
-		// Try to resume the download where we left off
-		off_t currentPosition;
-		BHttpRequest* http= dynamic_cast<BHttpRequest*>(request);
-		if (http != NULL && fTargetFile.GetSize(&currentPosition) == B_OK
-			&& currentPosition > 0) {
-			http->SetRangeStart(currentPosition);
-		}
-
-		thread_id thread = request->Run();
-		wait_for_thread(thread, NULL);
-	} while (fError == B_IO_ERROR || fError == B_DEV_TIMEOUT);
+	BUrlRequest* request = BUrlProtocolRoster::MakeRequest(fFileURL.String(),
+		&fTargetFile, this);
+	if (request == NULL)
+		return B_BAD_VALUE;
 
 	if (fError == B_OK) {
 		result = FetchUtils::MarkDownloadComplete(fTargetFile);
@@ -131,14 +111,6 @@ FetchFileJob::Execute()
 }
 
 
-void
-FetchFileJob::DataReceived(BUrlRequest*, const char* data, off_t position,
-	ssize_t size)
-{
-	fTargetFile.WriteAt(position, data, size);
-}
-
-
 void
 FetchFileJob::DownloadProgress(BUrlRequest*, off_t bytesReceived,
 	off_t bytesTotal)

src/kits/package/FetchFileJob.h

@@ -51,8 +51,6 @@ public:
 			off_t				DownloadTotalBytes() const;
 
 #ifdef HAIKU_TARGET_PLATFORM_HAIKU
-	virtual void	DataReceived(BUrlRequest*, const char* data,
-						off_t position, ssize_t size);
 	virtual void	DownloadProgress(BUrlRequest*,
 						off_t bytesReceived, off_t bytesTotal);
 	virtual void 	RequestCompleted(BUrlRequest* request,

src/tests/kits/net/service/DataTest.cpp

@@ -7,6 +7,7 @@
 #include "DataTest.h"
 
 #include <AutoDeleter.h>
+#include <DataIO.h>
 #include <DataRequest.h>
 #include <UrlProtocolRoster.h>
 
@@ -125,13 +126,6 @@ DataTest::UrlDecodeTest()
 }
 
 
-void
-DataTest::DataReceived(BUrlRequest*, const char* data, off_t, ssize_t size)
-{
-	fReceivedData.insert(fReceivedData.end(), data, data + size);
-}
-
-
 /* static */ void
 DataTest::AddTests(BTestSuite& parent)
 {
@@ -160,17 +154,17 @@ DataTest::_RunTest(BString url, const char* expected, size_t expectedLength)
 	NextSubTest();
 
 	BUrl testUrl(url);
+	BMallocIO buffer;
 	ObjectDeleter<BUrlRequest> requestDeleter(
-		BUrlProtocolRoster::MakeRequest(testUrl, this));
+		BUrlProtocolRoster::MakeRequest(testUrl, &buffer));
 	BDataRequest* request = dynamic_cast<BDataRequest*>(requestDeleter.Get());
 	CPPUNIT_ASSERT(request != NULL);
-	fReceivedData.clear();
 	request->Run();
 
 	while(request->IsRunning())
 		snooze(1000);
 
-	CPPUNIT_ASSERT_EQUAL(expectedLength, fReceivedData.size());
-	CPPUNIT_ASSERT(memcmp(&fReceivedData.front(), expected, expectedLength) == 0);
+	CPPUNIT_ASSERT_EQUAL(expectedLength, buffer.BufferLength());
+	CPPUNIT_ASSERT(memcmp(buffer.Buffer(), expected, expectedLength) == 0);
 }
 

src/tests/kits/net/service/DataTest.h

@@ -9,7 +9,6 @@
 
 
 #include <Url.h>
-#include <UrlProtocolListener.h>
 
 #include <TestCase.h>
 #include <TestSuite.h>
@@ -21,7 +20,7 @@ using BPrivate::Network::BUrlProtocolListener;
 using BPrivate::Network::BUrlRequest;
 
 
-class DataTest: public BTestCase, BUrlProtocolListener {
+class DataTest: public BTestCase {
 public:
 								DataTest();
 	virtual						~DataTest();
@@ -33,16 +32,11 @@ public:
 			void				Base64Test();
 			void				UrlDecodeTest();
 
-			void				DataReceived(BUrlRequest*, const char* data,
-									off_t, ssize_t size);
-
 	static	void				AddTests(BTestSuite& suite);
 
 private:
 			void				_RunTest(BString url, const char* expected,
 									size_t expectedLength);
-private:
-			std::vector<char>	fReceivedData;
 };
 
 

src/tests/kits/net/service/FileTest.cpp

@@ -22,14 +22,26 @@
 using namespace BPrivate::Network;
 
 
-class StopTestListener : public BUrlProtocolListener {
+class StopTestListener : public BDataIO {
 public:
-	StopTestListener() {}
-
-	void DataReceived(BUrlRequest *caller, const char*, off_t, ssize_t)
+	StopTestListener(BUrlRequest* request = NULL)
 	{
-		caller->Stop();
+		SetRequest(request);
 	}
+
+	ssize_t Write(const void*, size_t size)
+	{
+		fRequest->Stop();
+		return size;
+	}
+
+	void SetRequest(BUrlRequest* request)
+	{
+		fRequest = request;
+	}
+
+private:
+	BUrlRequest* fRequest;
 };
 
 
@@ -55,6 +67,7 @@ FileTest::StopTest()
 	BUrlRequest *request = BUrlProtocolRoster::MakeRequest(url, &listener);
 	CHK(request != NULL);
 
+	listener.SetRequest(request);
 	thread_id thr = request->Run();
 	status_t dummy;
 	wait_for_thread(thr, &dummy);
@@ -66,6 +79,7 @@ FileTest::StopTest()
 	request = BUrlProtocolRoster::MakeRequest("file:///", &listener);
 	CHK(request != NULL);
 
+	listener.SetRequest(request);
 	thr = request->Run();
 	wait_for_thread(thr, &dummy);
 

src/tests/kits/net/service/HttpTest.cpp

@@ -35,7 +35,7 @@ namespace {
 typedef std::map<std::string, std::string> HttpHeaderMap;
 
 
-class TestListener : public BUrlProtocolListener {
+class TestListener : public BUrlProtocolListener, public BDataIO {
 public:
 	TestListener(const std::string& expectedResponseBody,
 				 const HttpHeaderMap& expectedResponseHeaders)
@@ -45,16 +45,15 @@ public:
 	{
 	}
 
-	virtual void DataReceived(
-		BUrlRequest *caller,
-		const char *data,
-		off_t position,
-		ssize_t size)
+	virtual ssize_t Write(
+		const void *data,
+		size_t size)
 	{
 		std::copy_n(
-			data + position,
+			(const char*)data,
 			size,
 			std::back_inserter(fActualResponseBody));
+		return size;
 	}
 
 	virtual void HeadersReceived(
@@ -129,7 +128,8 @@ void SendAuthenticatedRequest(
 	TestListener listener(expectedResponseBody, expectedResponseHeaders);
 
 	ObjectDeleter<BUrlRequest> requestDeleter(
-		BUrlProtocolRoster::MakeRequest(testUrl, &listener, &context));
+		BUrlProtocolRoster::MakeRequest(testUrl, &listener, &listener,
+			&context));
 	BHttpRequest* request = dynamic_cast<BHttpRequest*>(requestDeleter.Get());
 	CPPUNIT_ASSERT(request != NULL);
 
@@ -243,7 +243,8 @@ HttpTest::ProxyTest()
 	TestListener listener(expectedResponseBody, expectedResponseHeaders);
 
 	ObjectDeleter<BUrlRequest> requestDeleter(
-		BUrlProtocolRoster::MakeRequest(testUrl, &listener, context));
+		BUrlProtocolRoster::MakeRequest(testUrl, &listener, &listener,
+			context));
 	BHttpRequest* request = dynamic_cast<BHttpRequest*>(requestDeleter.Get());
 	CPPUNIT_ASSERT(request != NULL);
 
@@ -335,7 +336,8 @@ HttpTest::UploadTest()
 	BUrlContext context;
 
 	ObjectDeleter<BUrlRequest> requestDeleter(
-		BUrlProtocolRoster::MakeRequest(testUrl, &listener, &context));
+		BUrlProtocolRoster::MakeRequest(testUrl, &listener, &listener,
+			&context));
 	BHttpRequest* request = dynamic_cast<BHttpRequest*>(requestDeleter.Get());
 	CPPUNIT_ASSERT(request != NULL);
 
@@ -530,7 +532,8 @@ HttpTest::_GetTest(const BString& path)
 	TestListener listener(expectedResponseBody, expectedResponseHeaders);
 
 	ObjectDeleter<BUrlRequest> requestDeleter(
-		BUrlProtocolRoster::MakeRequest(testUrl, &listener, context));
+		BUrlProtocolRoster::MakeRequest(testUrl, &listener, &listener,
+			context));
 	BHttpRequest* request = dynamic_cast<BHttpRequest*>(requestDeleter.Get());
 	CPPUNIT_ASSERT(request != NULL);