googlefs: use libnetservices for HTTP requests

This allows us to do HTTPS more easily and removes a lot of the code.

A file is converted to C++ to allow use of these APIs.

Now Google replies to the query with… a page asking to accept cookies,
and no results at all. They are definitely not cool anymore.

Change-Id: I4bed3b1f8630b324b979d26fbb71c4f05165fbb5
Reviewed-on: https://review.haiku-os.org/c/haiku/+/5494
Tested-by: Commit checker robot <[email protected]>
Reviewed-by: Adrien Destugues <[email protected]>

src/add-ons/kernel/file_systems/googlefs/Jamfile

@@ -4,20 +4,22 @@ local userlandFSTop = [ FDirName $(HAIKU_TOP) src add-ons kernel file_systems
 	userlandfs ] ;
 local userlandFSIncludes = [ PrivateHeaders userlandfs ] ;
 
+UsePrivateHeaders shared ;
+UsePrivateHeaders netservices ;
+
 #SubDirCcFlags -DTRACK_FILENAME ;
 #SubDirCcFlags -DDEBUG_GOOGLEFS=1 ;
 SubDirCcFlags -D_IMPEXP_KERNEL= ;
 
 Addon googlefs :
 	attrs.c
-	google_request.c
+	google_request.cpp
 	googlefs.c
-	http_cnx.c
 	lists2.c
 	parse_google_html.c
 	query.c
 	settings.c
 	string_utils.c
 	vnidpool.c
-	: libuserlandfs_haiku_kernel.so network
+	: libuserlandfs_haiku_kernel.so libnetservices.a bnetapi be network shared [ TargetLibstdc++ ]
 ;

src/add-ons/kernel/file_systems/googlefs/google_request.cpp

@@ -12,13 +12,18 @@
 #include <arpa/inet.h>
 #include <malloc.h>
 #include <sys/socket.h>
-#include "http_cnx.h"
-#include "googlefs.h"
 #include "google_request.h"
+
+#include "googlefs.h"
 #include "lists2.h"
 #include "settings.h"
 #include "string_utils.h"
 
+#include <UrlProtocolRoster.h>
+#include <UrlRequest.h>
+
+using namespace BPrivate::Network;
+
 #define DO_PUBLISH
 //#define FAKE_INPUT "/boot/home/devel/drivers/googlefs/log2.html"
 
@@ -29,83 +34,34 @@
 //#define EXTRAURL "&num=50&hl=en&ie=ISO-8859-1&btnG=Google+Search&as_epq=frequently+asked&as_oq=help&as_eq=plop&lr=lang_en&as_ft=i&as_filetype=&as_qdr=m3&as_nlo=&as_nhi=&as_occt=any&as_dt=i&as_sitesearch="
 
 #define TESTURL "http://www.google.com/search?hl=en&ie=UTF-8&num=50&q=beos"
-//#define BASEURL "http://www.google.com/search?hl=en&ie=UTF-8&num=50&q="
+#define BASEURL "https://www.google.com/search?hl=en&ie=UTF-8&oe=UTF-8&gws_rd=cr"
-#define BASEURL "/search?hl=en&ie=UTF-8&oe=UTF-8&gws_rd=cr"
 #define FMT_NUM "&num=%u"
 #define FMT_Q "&q=%s"
 
 /* parse_google_html.c */
 extern int google_parse_results(const char *html, size_t htmlsize, long *nextid, struct google_result **results);
 
-// move that to ksocket inlined
-static int kinet_aton(const char *in, struct in_addr *addr)
-{
-	int i;
-	unsigned long a;
-	uint32 inaddr = 0L;
-	const char *p = in;
-	for (i = 0; i < 4; i++) {
-		a = strtoul(p, (char **)&p, 10);
-		if (!p)
-			return -1;
-		inaddr = (inaddr >> 8) | ((a & 0x0ff) << 24);
-		*(uint32 *)addr = inaddr;
-		if (!*p)
-			return 0;
-		p++;
-	}
-	return 0;
-}
-
-
-status_t google_request_init(void)
-{
-	status_t err;
-	err = http_init();
-	return err;
-}
-
-status_t google_request_uninit(void)
-{
-	status_t err;
-	err = http_uninit();
-	return err;
-}
 
 status_t google_request_process(struct google_request *req)
 {
-	struct sockaddr_in sin;
+	struct BUrlRequest *cnx = NULL;
-	struct http_cnx *cnx = NULL;
 	struct google_result *res;
 	status_t err;
 	int count;
 	char *p = NULL;
 	char *url = NULL;
+	BMallocIO output;
+	thread_id t;
 	
-	err = http_create(&cnx);
-	if (err)
-		return err;
 	err = ENOMEM;
 	req->cnx = cnx;
 #ifndef FAKE_INPUT
-	sin.sin_len = sizeof(struct sockaddr_in);
-	sin.sin_family = AF_INET;
-	sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
-	if (kinet_aton(google_server, &sin.sin_addr) < 0)
-		goto err_cnx;
-	sin.sin_port = htons(google_server_port);
-	err = http_connect(cnx, &sin);
-	fprintf(stderr, "google_request: http_connect: error 0x%08" B_PRIx32 "\n", err);
-	if (err)
-		goto err_cnx;
-	
-	err = ENOMEM;
 	p = urlify_string(req->query_string);
 	if (!p)
 		goto err_con;
 	
 	err = ENOMEM;
-	url = malloc(strlen(BASEURL)+strlen(FMT_NUM)+10+strlen(FMT_Q)+strlen(p)+2);
+	url = (char*)malloc(strlen(BASEURL)+strlen(FMT_NUM)+10+strlen(FMT_Q)+strlen(p)+2);
 	if (!url)
 		goto err_url;
 	strcpy(url, BASEURL);
@@ -114,19 +70,19 @@ status_t google_request_process(struct google_request *req)
 	
 	fprintf(stderr, "google_request: final URL: %s\n", url);
 	
-	err = http_get(cnx, url);
+	cnx = BUrlProtocolRoster::MakeRequest(url, &output, NULL);
-	fprintf(stderr, "google_request: http_get: error 0x%08" B_PRIx32 "\n", err);
+	if (cnx == NULL)
-	if (err < 0)
+		return ENOMEM;
-		goto err_url2;
+
-	fprintf(stderr, "google_request: http_get: HEADERS %ld:%s\n", cnx->headerslen, cnx->headers);
+	t = cnx->Run();
-	//fprintf(stderr, "DATA: %d:%s\n", cnx->datalen, cnx->data);
+	wait_for_thread(t, &err);
 	
-	fprintf(stderr, "google_request: buffer @ %p, len %ld\n", cnx->data, cnx->datalen);
+	fprintf(stderr, "google_request: buffer @ %p, len %ld\n", output.Buffer(), output.BufferLength());
 	{
 		int fd;
 		// debug output
 		fd = open("/tmp/google.html", O_CREAT|O_TRUNC|O_RDWR, 0644);
-		write(fd, cnx->data, cnx->datalen);
+		write(fd, output.Buffer(), output.BufferLength());
 		close(fd);
 	}
 #else
@@ -150,7 +106,8 @@ status_t google_request_process(struct google_request *req)
 		close(fd);
 	}
 #endif /* FAKE_INPUT */	
-	err = count = google_parse_results(req->cnx->data, req->cnx->datalen, &req->nextid, &req->results);
+	err = count = google_parse_results((const char*)output.Buffer(), output.BufferLength(),
+		&req->nextid, &req->results);
 	if (err < 0)
 		goto err_get;
 #ifdef DO_PUBLISH
@@ -161,21 +118,16 @@ status_t google_request_process(struct google_request *req)
 #endif
 	free(url);
 	free(p);
-	/* close now */
+	// request is kept and deleted in google_request_close
-	http_close(cnx);
-	
 	return B_OK;
 
 
 err_get:
-err_url2:
 	free(url);
 err_url:
 	free(p);
 err_con:
-	http_close(cnx);
+	delete cnx;
-err_cnx:
-	http_delete(cnx);
 	req->cnx = NULL;
 	return err;
 }
@@ -191,8 +143,7 @@ status_t google_request_close(struct google_request *req)
 		return EINVAL;
 	if (!req->cnx)
 		return B_OK;
-	http_close(req->cnx);
+	delete(req->cnx);
-	http_delete(req->cnx);
 	req->cnx = NULL;
 	return B_OK;
 }
@@ -202,7 +153,7 @@ status_t google_request_open(const char *query_string, struct fs_volume *volume,
 	struct google_request *r;
 	if (!req)
 		return EINVAL;
-	r = malloc(sizeof(struct google_request));
+	r = (google_request*)malloc(sizeof(struct google_request));
 	if (!r)
 		return ENOMEM;
 	memset(r, 0, sizeof(struct google_request));

src/add-ons/kernel/file_systems/googlefs/google_request.h

@@ -5,12 +5,21 @@
 #ifndef _GOOGLE_REQUEST_H
 #define _GOOGLE_REQUEST_H
 
+#ifdef __cplusplus
+#include <UrlRequest.h>
+using namespace BPrivate::Network;
+extern "C" {
+#else
+struct BUrlRequest;
+typedef struct BUrlRequest BUrlRequest;
+#endif
+
 struct google_request {
 	struct google_request *next;
 	struct fs_volume *volume;
 	struct fs_node *query_node; /* root folder for that query */
 	char *query_string;
-	struct http_cnx *cnx;
+	BUrlRequest *cnx;
 	struct google_result *results;
 	long nextid;
 };
@@ -31,8 +40,6 @@ struct google_result {
 	char similar_url[GR_MAX_URL];
 };
 
-extern status_t google_request_init(void);
-extern status_t google_request_uninit(void);
 extern status_t google_request_process(struct google_request *req);
 extern status_t google_request_process_async(struct google_request *req);
 extern status_t google_request_close(struct google_request *req);
@@ -41,4 +48,8 @@ extern status_t google_request_free(struct google_request *req);
 
 extern int google_parse_results(const char *html, size_t htmlsize, long *nextid, struct google_result **results);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* _GOOGLE_REQUEST_H */

src/add-ons/kernel/file_systems/googlefs/googlefs.c

@@ -133,10 +133,6 @@ static status_t googlefs_mount(fs_volume *_vol, const char *devname, uint32 flag
 
 	err = load_settings();
 
-	err = google_request_init();
-	if (err)
-		goto err_http;
-
 	ns = malloc(sizeof(fs_nspace));
 	if (!ns)
 		return B_NO_MEMORY;
@@ -184,7 +180,6 @@ static status_t googlefs_mount(fs_volume *_vol, const char *devname, uint32 flag
 	}
 	free_lock(&ns->l);
 	free(ns);
-err_http:
 	atomic_add(&refcount, -1);
 	return err;
 }
@@ -211,8 +206,6 @@ static status_t googlefs_unmount(fs_volume *_volume)
 	vnidpool_free(ns->vnids);
 	free(ns);
 
-	google_request_uninit();
-
 	atomic_add(&refcount, -1);
 
 	return B_OK;

src/add-ons/kernel/file_systems/googlefs/googlefs.h

@@ -44,6 +44,9 @@ typedef dev_t nspace_id;
 
 #define MAX_VNIDS 5000
 
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 struct attr_entry {
 	const char *name;
@@ -132,4 +135,8 @@ int googlefs_event(fs_nspace *ns, fs_node *node, int flags);
  */
 extern int googlefs_push_result_to_query(struct google_request *request, struct google_result *result);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif

src/add-ons/kernel/file_systems/googlefs/http_cnx.c

@@ -1,262 +0,0 @@
-/*
- * Copyright 2004-2008, François Revol, <[email protected]>.
- * Distributed under the terms of the MIT License.
- */
-
-#include <errno.h>
-#include <sys/param.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include "ksocket.h"
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include "http_cnx.h"
-
-//#define TESTME
-
-#ifdef _KERNEL_MODE
-#include <KernelExport.h>
-#define printf dprintf
-#undef TESTME
-#endif
-
-#define DBG "googlefs: http_cnx: "
-
-#define HTTPVER "1.0"
-
-//#define GOOGLEFS_UA "GoogleFS"
-//Mozilla/3.0 (compatible; NetPositive/2.2.2; BeOS)
-//Mozilla/5.0 (BeOS; U; BeOS BePC; en-US; rv:1.8.1.18) Gecko/20081114 BonEcho/2.0.0.18
-#ifdef __HAIKU__
-#define GOOGLEFS_UA "Mozilla/5.0 (compatible; GoogleFS/0.1; Haiku)"
-#else
-#define GOOGLEFS_UA "Mozilla/5.0 (compatible; GoogleFS/0.1; BeOS)"
-#endif
-
-#ifdef TESTME
-#define BUFSZ (128*1024)
-#endif
-
-KSOCKET_MODULE_DECL;
-
-status_t http_init(void)
-{
-	return ksocket_init();
-}
-
-status_t http_uninit(void)
-{
-	return ksocket_cleanup();
-}
-
-status_t http_create(struct http_cnx **cnx)
-{
-	struct http_cnx *c;
-	int err;
-	c = malloc(sizeof(struct http_cnx));
-	if (!c)
-		return ENOMEM;
-	memset(c, 0, sizeof(struct http_cnx));
-	err = c->sock = ksocket(AF_INET, SOCK_STREAM, 0);
-	if (err < 0) {
-		free(c);
-		return errno;
-	}
-	*cnx = c;
-	return B_OK;
-}
-
-status_t http_delete(struct http_cnx *cnx)
-{
-	if (!cnx)
-		return EINVAL;
-	if ((unsigned long)cnx < 0x40) {
-		fprintf(stderr, "http: WARNING: cnx ptr = %p\n", cnx);
-		return B_OK;
-	}	
-	if (cnx->sock >= 0) {
-		kclosesocket(cnx->sock);
-		cnx->sock = -1;
-	}
-	if (cnx->buffer)
-		free(cnx->buffer);
-	cnx->buffer = (char *)0xaa55aa55;//XXX
-	free(cnx);
-	return B_OK;
-}
-
-status_t http_connect(struct http_cnx *cnx, struct sockaddr_in *sin)
-{
-	int err;
-	uint32 ip;
-	uint16 port;
-	if (!sin) {
-		fprintf(stderr, "http_connect(, NULL)!!\n");
-		return EINVAL;
-	}
-	ip = sin->sin_addr.s_addr;
-	port = sin->sin_port;
-	fprintf(stderr, "http_connect(, %"B_PRId32".%"B_PRId32".%" B_PRId32 ".%" B_PRId32 ":%d), sock = %d\n", ip & 0xff, (ip >> 8) & 0xff, (ip >> 16) & 0xff, (ip >> 24) & 0xff, ntohs(port), cnx->sock);
-	err = kconnect(cnx->sock, (struct sockaddr *)sin, sin->sin_len);
-	cnx->err = 0;
-	if (err == -1)
-		return errno;
-	return err;
-}
-
-status_t http_close(struct http_cnx *cnx)
-{
-	int err;
-	if (!cnx)
-		return EINVAL;
-	if (cnx->sock < 0)
-		return 0;
-	err = kclosesocket(cnx->sock);
-	cnx->sock = -1;
-	return err;
-}
-
-status_t http_get(struct http_cnx *cnx, const char *url)
-{
-	char *req, *p;
-	int reqlen;
-	int len;
-	int err;
-	long headerslen = 0;
-	long contentlen = 0;
-	if (strlen(url) > 4096 - 128)
-		return EINVAL;
-	req = malloc(4096+1);
-	if (!req)
-		return B_NO_MEMORY;
-	req[4096] = '\0';
-	/* no snprintf in kernel :( */
-	sprintf(req, "GET %s HTTP/"HTTPVER"\r\nUser-Agent: " GOOGLEFS_UA "\r\nAccept: */*\r\n\r\n", url);
-	reqlen = strlen(req);
-	err = len = write(cnx->sock, req, reqlen);
-	if (len < 1)
-		goto err0;
-	reqlen = 4096;
-	err = len = read(cnx->sock, req, reqlen);
-	printf("read(sock) = %d\n", len);
-	if (err < 0)
-		goto err0;
-	//write(1, req, len);
-	{
-		int fd;
-		// debug output
-		fd = open("/tmp/google.html_", O_CREAT|O_TRUNC|O_RDWR, 0644);
-		write(fd, req, len);
-		close(fd);
-	}
-
-	err = EINVAL;
-	if (len < 10)
-		goto err0;
-	if (!strstr(req, "HTTP/1.0 200"))
-		goto err0;
-
-	err = B_NO_MEMORY;
-	if (!strstr(req, "\r\n\r\n")) {
-		if (!strstr(req, "\n\n")) /* shouldn't happen */
-			goto err0;
-		headerslen = strstr(req, "\n\n") - req + 2;
-		req[headerslen-1] = '\0';
-		req[headerslen-2] = '\0';
-	} else {
-		headerslen = strstr(req, "\r\n\r\n") - req + 4;
-		req[headerslen-3] = '\0';
-		req[headerslen-4] = '\0';
-	}
-	if (headerslen < 2 || headerslen > 4095)
-		goto err0;
-	p = strstr(req, "HTTP/1.");
-	if (!p)
-		goto err0;
-	p += strlen("HTTP/1.");
-	if (strlen(p) < 5)
-		goto err0;
-	p += strlen("1 ");
-	cnx->err = strtol(p, &p, 10);
-	if (cnx->err < 200 || cnx->err > 299)
-		goto err0;
-	printf("REQ: %" B_PRId32 "\n", cnx->err);
-	contentlen = len - headerslen;
-//	if (!strstr(req, "\n\n") && !strstr(req, "\r\n\r\n"))
-//		goto err0;
-	while (len > 0) {
-		long left = reqlen - headerslen - contentlen;
-		if (left < 128) {
-			reqlen += 4096;
-			p = realloc(req, reqlen);
-			left += 4096;
-			if (!p)
-				goto err0;
-			req = p;
-		}
-		err = len = read(cnx->sock, req + headerslen + contentlen, left);
-		if (len < 0)
-			goto err0;
-		contentlen += len;
-	}
-	cnx->buffer = req;
-	cnx->headers = req;
-	cnx->headerslen = headerslen;
-	cnx->data = req + headerslen;
-	cnx->datalen = contentlen;
-	cnx->data[contentlen] = '\0'; /* we have at least 128 bytes allocated ahead */
-	return B_OK;
-err0:
-	if (err > -1)
-		err = -1;
-	free(req);
-	return err;
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-#ifdef TESTME
-//#define TESTURL "/"
-//#define TESTURL "http://www.google.com/search?as_q=google+api+&num=50&hl=en&ie=ISO-8859-1&btnG=Google+Search&as_epq=frequently+asked&as_oq=help&as_eq=plop&lr=lang_en&as_ft=i&as_filetype=&as_qdr=m3&as_nlo=&as_nhi=&as_occt=any&as_dt=i&as_sitesearch="
-#define TESTURL "http://www.google.com/search?hl=en&ie=UTF-8&num=50&q=beos"
-int main(int argc, char **argv)
-{
-	struct sockaddr_in sin;
-	struct http_cnx *cnx;
-	size_t len;
-	char *p;
-	int err;
-	
-	http_init();
-	err = http_create(&cnx);
-	printf("error 0x%08lx\n", err);
-	sin.sin_len = sizeof(struct sockaddr_in);
-	sin.sin_family = AF_INET;
-	sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
-	inet_aton("66.102.11.99", &sin.sin_addr);
-	sin.sin_port = htons(80);
-	err = http_connect(cnx, &sin);
-	printf("error 0x%08lx\n", err);
-	err = http_get(cnx, TESTURL);
-	printf("error 0x%08lx\n", err);
-	if (err < 0)
-		return 1;
-	printf("HEADERS %d:%s\n", cnx->headerslen, cnx->headers);
-	printf("DATA: %d:%s\n", cnx->datalen, cnx->data);
-	http_close(cnx);
-	http_delete(cnx);
-	return 0;
-}
-#endif

src/add-ons/kernel/file_systems/googlefs/http_cnx.h

@@ -1,37 +0,0 @@
-/*
- * Copyright 2004-2008, François Revol, <[email protected]>.
- * Distributed under the terms of the MIT License.
- */
-#ifndef _HTTP_CNX_H
-#define _HTTP_CNX_H
-
-#include <OS.h>
-
-struct http_cnx {
-	int sock;
-	status_t err; /* 404, ... */
-	char *headers;
-	size_t headerslen;
-	char *data;
-	size_t datalen;
-	/* for bookkeeping */
-	char *buffer;
-};
-
-struct sockaddr_in;
-
-extern status_t http_init(void);
-extern status_t http_uninit(void);
-
-extern status_t http_create(struct http_cnx **);
-extern status_t http_delete(struct http_cnx *cnx);
-
-extern status_t http_connect(struct http_cnx *cnx, struct sockaddr_in *sin);
-extern status_t http_close(struct http_cnx *cnx);
-
-extern status_t http_get(struct http_cnx *cnx, const char *url);
-//extern status_t http_post(struct http_cnx *cnx, const char *url);
-/* read the actual data in the buffer */
-//extern status_t http_fetch_data(struct http_cnx *cnx);
-
-#endif /* _HTTP_CNX_H */

src/add-ons/kernel/file_systems/googlefs/lists2.h

@@ -6,6 +6,10 @@
 #ifndef _LISTS2_H
 #define _LISTS2_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef int (*sll_compare_func)(const void *item, void *id);
 
 extern void *sll_find(long nextoff, void *head, sll_compare_func func, void *id);
@@ -27,4 +31,8 @@ extern void *sll_next(long nextoff, void *item);
 #define SLL_DEQUEUE(_head,_nextp) (typeof(_head))sll_dequeue_tail(offsetof(typeof(*_head),_nextp), (void **)&(_head))
 #define SLL_REMOVE(_head,_nextp,_item) sll_remove(offsetof(typeof(*_head),_nextp), (void **)&(_head), _item)
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* _LISTS2_H */

src/add-ons/kernel/file_systems/googlefs/settings.h

@@ -5,6 +5,10 @@
 #ifndef _SETTINGS_H
 #define _SETTINGS_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /* settings */
 extern char google_server[20];
 extern int google_server_port;
@@ -14,4 +18,8 @@ extern bool sync_unlink_queries;
 
 extern status_t load_settings(void);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* _SETTINGS_H */

src/add-ons/kernel/file_systems/googlefs/string_utils.h

@@ -5,6 +5,10 @@
 #ifndef _STRING_UTILS_H
 #define _STRING_UTILS_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /* converts a string to a format suitable for use in urls
  * ex: "foo bar+" -> "foo+bar%2D"
  * caller must free() result
@@ -17,4 +21,8 @@ extern char *urlify_string(const char *str);
  */
 extern char *unentitify_string(const char *str);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* _STRING_UTILS_H */