HaikuDepot: Change communication mechanism with server for repos

(last commit with same title only included new files - added those now)

Previously the desktop application would make a number of JSON-RPC calls
over HTTP to get the repositories.  Now it will make a single call to get
the repositories and cache the result.  This uses standard HTTP cache
signalling techniques and allows the server-side the ability to cache
the generated data as well.  Note that the model classes and parse-
related classes are generated and may not be code-style compliant.  They
are generated from JSON schema files in the server-side project.
Information about this as well as the python files used to generate the
C++ classes and headers are included in the server-side project.

src/apps/haikudepot/Jamfile

@@ -4,7 +4,7 @@ UsePrivateHeaders interface shared storage package support net ;
 
 # source directories
 local sourceDirs =
-	edits_generic model textview ui ui_generic server tar util
+	edits_generic model textview ui ui_generic server server/dumpexportrepository tar util
 ;
 
 local sourceDir ;
@@ -74,10 +74,16 @@ Application HaikuDepot :
 	SharedBitmap.cpp
 	UserLoginWindow.cpp
 	
+	# network + server - model
+	DumpExportRepository.cpp
+	DumpExportRepositorySource.cpp
+	DumpExportRepositoryJsonListener.cpp
+
 	# network + server
 	AbstractServerProcess.cpp
 	ServerSettings.cpp
 	WebAppInterface.cpp
+	RepositoryDataUpdateProcess.cpp
 	ServerIconExportUpdateProcess.cpp
 	StandardMetaDataJsonEventListener.cpp
 	StandardMetaData.cpp

src/apps/haikudepot/model/Model.cpp

@@ -6,6 +6,7 @@
  */
 
 #include "Model.h"
+#include "RepositoryDataUpdateProcess.h"
 #include "StorageUtils.h"
 
 #include <ctime>
@@ -834,59 +835,47 @@ Model::SetAuthorization(const BString& username, const BString& password,
 
 // #pragma mark - private
 
+/*! When bulk repository data comes down from the server, it will
+    arrive as a json.gz payload.  This is stored locally as a cache
+    and this method will provide the on-disk storage location for
+    this file.
+*/
 
-void
+status_t
-Model::PopulateWebAppRepositoryCode(DepotInfo& depotInfo)
+Model::_DumpExportRepositoryDataPath(BPath& path) const
 {
-	if (depotInfo.BaseURL().Length() > 0) {
+	BPath repoDataPath;
 
-		BMessage repositoriesEnvelope;
+	if (find_directory(B_USER_CACHE_DIRECTORY, &repoDataPath) == B_OK
-		BMessage result;
+		&& repoDataPath.Append("HaikuDepot") == B_OK
-		double total;
+		&& create_directory(repoDataPath.Path(), 0777) == B_OK
-		StringList repositorySourceBaseURLs;
+		&& repoDataPath.Append("repository-all_en.json.gz") == B_OK) {
-
+		path.SetTo(repoDataPath.Path());
-		repositorySourceBaseURLs.Add(depotInfo.BaseURL());
+		return B_OK;
-
-		// TODO; better API call handling around errors.
-
-		if (fWebAppInterface.RetrieveRepositoriesForSourceBaseURLs(
-			repositorySourceBaseURLs, repositoriesEnvelope) == B_OK
-			&& repositoriesEnvelope.FindMessage("result", &result) == B_OK
-			&& result.FindDouble("total", &total) == B_OK) {
-
-			if ((int64)total > 0) {
-				BMessage repositories;
-				BMessage repository;
-				BString repositoryCode;
-
-				if (result.FindMessage("items", &repositories) == B_OK
-					&& repositories.FindMessage("0", &repository) == B_OK
-					&& repository.FindString("code", &repositoryCode) == B_OK) {
-
-					depotInfo.SetWebAppRepositoryCode(repositoryCode);
-
-					printf("did assign web app repository code '%s' to local "
-						"depot '%s'\n",
-						depotInfo.WebAppRepositoryCode().String(),
-						depotInfo.Name().String());
-				} else {
-					printf("unable to find the 'code' in the api response for "
-						"local depot '%s'\n",
-						depotInfo.Name().String());
-				}
-			} else {
-				printf("unable to find a repository code for '%s'\n",
-					depotInfo.BaseURL().String());
-			}
-		} else {
-			printf("unexpected result obtaining repository code for '%s'\n",
-				depotInfo.BaseURL().String());
-		}
-	} else {
-		printf("missing base url for depot info %s --> will not obtain web app "
-			"repository code\n",
-			depotInfo.Name().String());
 	}
+
+	path.Unset();
+	fprintf(stdout, "unable to find the user cache file for repositories'"
+		" data");
+	return B_ERROR;
+}
+
+
+status_t
+Model::PopulateWebAppRepositoryCodes()
+{
+	status_t result = B_OK;
+	BPath dataPath;
+
+	result = _DumpExportRepositoryDataPath(dataPath);
+
+	if (result != B_OK)
+		return result;
+
+	RepositoryDataUpdateProcess process(dataPath, &fDepots);
+	result = process.Run();
+
+	return result;
 }
 
 

src/apps/haikudepot/model/Model.h

@@ -118,8 +118,7 @@ public:
 			bool				ShowDevelopPackages() const
 									{ return fShowDevelopPackages; }
 
-			void				PopulateWebAppRepositoryCode(
+			status_t			PopulateWebAppRepositoryCodes();
-									DepotInfo& depotInfo);
 
 			// Retrieve package information
 	static	const uint32		POPULATE_CACHED_RATING	= 1 << 0;
@@ -152,6 +151,8 @@ public:
 
 
 private:
+			status_t			_DumpExportRepositoryDataPath(
+									BPath& path) const;
 			void				_UpdateIsFeaturedFilter();
 
 	static	int32				_PopulateAllPackagesEntry(void* cookie);

src/apps/haikudepot/server/AbstractServerProcess.cpp

@@ -4,9 +4,240 @@
  */
 #include "AbstractServerProcess.h"
 
+#include <errno.h>
+#include <string.h>
+
+#include <AutoDeleter.h>
+#include <FileIO.h>
+#include <HttpRequest.h>
+#include <HttpTime.h>
+#include <UrlProtocolRoster.h>
+
+#include <support/ZlibCompressionAlgorithm.h>
+
+#include "ServerSettings.h"
+#include "StandardMetaDataJsonEventListener.h"
+#include "ToFileUrlProtocolListener.h"
+
+
+#define MAX_REDIRECTS 3
+#define MAX_FAILURES 2
+
+#define HTTP_STATUS_FOUND 302
+#define HTTP_STATUS_NOT_MODIFIED 304
+
+// 30 seconds
+#define TIMEOUT_MICROSECONDS 3e+7
+
 
 status_t
-AbstractServerProcess::Run()
+AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue) const
 {
+	BPath metaDataPath;
+	BString jsonPath;
+
+	GetStandardMetaDataPath(metaDataPath);
+	GetStandardMetaDataJsonPath(jsonPath);
+
+	return IfModifiedSinceHeaderValue(headerValue, metaDataPath, jsonPath);
+}
+
+
+status_t
+AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue,
+	const BPath& metaDataPath, const BString& jsonPath) const
+{
+	headerValue.SetTo("");
+	struct stat s;
+
+	if (-1 == stat(metaDataPath.Path(), &s)) {
+		if (ENOENT != errno)
+			 return B_ERROR;
+
+		return B_FILE_NOT_FOUND;
+	}
+
+	if (s.st_size == 0)
+		return B_BAD_VALUE;
+
+	StandardMetaData metaData;
+	status_t result = PopulateMetaData(metaData, metaDataPath, jsonPath);
+
+	if (result == B_OK) {
+
+		// An example of this output would be; 'Fri, 24 Oct 2014 19:32:27 +0000'
+
+		BDateTime modifiedDateTime = metaData
+			.GetDataModifiedTimestampAsDateTime();
+		BPrivate::BHttpTime modifiedHttpTime(modifiedDateTime);
+		headerValue.SetTo(modifiedHttpTime
+			.ToString(BPrivate::B_HTTP_TIME_FORMAT_COOKIE));
+	} else {
+		fprintf(stderr, "unable to parse the meta-data date and time -"
+			" cannot set the 'If-Modified-Since' header\n");
+	}
+
+	return result;
+}
+
+
+status_t
+AbstractServerProcess::PopulateMetaData(
+	StandardMetaData& metaData, const BPath& path,
+	const BString& jsonPath) const
+{
+	StandardMetaDataJsonEventListener listener(jsonPath, metaData);
+	status_t result = ParseJsonFromFileWithListener(&listener, path);
+
+	if (result != B_OK)
+		return result;
+
+	result = listener.ErrorStatus();
+
+	if (result != B_OK)
+		return result;
+
+	if (!metaData.IsPopulated()) {
+		fprintf(stderr, "the meta data was read from [%s], but no values "
+			"were extracted\n", path.Path());
+		return B_BAD_DATA;
+	}
+
 	return B_OK;
 }
+
+
+bool
+AbstractServerProcess::LooksLikeGzip(const char *pathStr) const
+{
+	int l = strlen(pathStr);
+	return l > 4 && 0 == strncmp(&pathStr[l - 3], ".gz", 3);
+}
+
+
+/*!	Note that a B_OK return code from this method may not indicate that the
+	listening process went well.  One has to see if there was an error in
+	the listener.
+*/
+
+status_t
+AbstractServerProcess::ParseJsonFromFileWithListener(
+	BJsonEventListener *listener,
+	const BPath& path) const
+{
+	const char* pathStr = path.Path();
+	FILE* file = fopen(pathStr, "rb");
+
+	if (file == NULL) {
+		fprintf(stderr, "unable to find the meta data file at [%s]\n",
+			path.Path());
+		return B_FILE_NOT_FOUND;
+	}
+
+	BFileIO rawInput(file, true); // takes ownership
+
+		// if the file extension ends with '.gz' then the data will be
+		// compressed and the algorithm needs to decompress the data as
+		// it is parsed.
+
+	if (LooksLikeGzip(pathStr)) {
+		BDataIO* gzDecompressedInput = NULL;
+		BZlibDecompressionParameters* zlibDecompressionParameters
+			= new BZlibDecompressionParameters();
+
+		status_t result = BZlibCompressionAlgorithm()
+			.CreateDecompressingInputStream(&rawInput,
+				zlibDecompressionParameters, gzDecompressedInput);
+
+		if (B_OK != result)
+			return result;
+
+		ObjectDeleter<BDataIO> gzDecompressedInputDeleter(gzDecompressedInput);
+		BPrivate::BJson::Parse(gzDecompressedInput, listener);
+	} else {
+		BPrivate::BJson::Parse(&rawInput, listener);
+	}
+
+	return B_OK;
+}
+
+
+status_t
+AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath,
+	const BUrl& url, uint32 redirects, uint32 failures)
+{
+	if (redirects > MAX_REDIRECTS) {
+		fprintf(stdout, "exceeded %d redirects --> failure\n", MAX_REDIRECTS);
+		return B_IO_ERROR;
+	}
+
+	if (failures > MAX_FAILURES) {
+		fprintf(stdout, "exceeded %d failures\n", MAX_FAILURES);
+		return B_IO_ERROR;
+	}
+
+	fprintf(stdout, "will stream '%s' to [%s]\n", url.UrlString().String(),
+		targetFilePath.Path());
+
+	ToFileUrlProtocolListener listener(targetFilePath, LoggingName(),
+		ServerSettings::UrlConnectionTraceLoggingEnabled());
+
+	BHttpHeaders headers;
+	ServerSettings::AugmentHeaders(headers);
+
+	BString ifModifiedSinceHeader;
+	status_t ifModifiedSinceHeaderStatus = IfModifiedSinceHeaderValue(
+		ifModifiedSinceHeader);
+
+	if (ifModifiedSinceHeaderStatus == B_OK &&
+		ifModifiedSinceHeader.Length() > 0) {
+		headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader);
+	}
+
+	BHttpRequest *request = dynamic_cast<BHttpRequest *>(
+		BUrlProtocolRoster::MakeRequest(url, &listener));
+	ObjectDeleter<BHttpRequest> requestDeleter(request);
+	request->SetHeaders(headers);
+	request->SetMaxRedirections(0);
+	request->SetTimeout(TIMEOUT_MICROSECONDS);
+
+	thread_id thread = request->Run();
+	wait_for_thread(thread, NULL);
+
+	const BHttpResult& result = dynamic_cast<const BHttpResult&>(
+		request->Result());
+
+	int32 statusCode = result.StatusCode();
+
+	if (BHttpRequest::IsSuccessStatusCode(statusCode)) {
+		fprintf(stdout, "did complete streaming data\n");
+		return B_OK;
+	} else if (statusCode == HTTP_STATUS_NOT_MODIFIED) {
+		fprintf(stdout, "remote data has not changed since [%s]\n",
+			ifModifiedSinceHeader.String());
+		return APP_ERR_NOT_MODIFIED;
+	} else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) {
+		const BHttpHeaders responseHeaders = result.Headers();
+		const char *locationValue = responseHeaders["Location"];
+
+		if (locationValue != NULL && strlen(locationValue) != 0) {
+			BUrl location(result.Url(), locationValue);
+			fprintf(stdout, "will redirect to; %s\n",
+				location.UrlString().String());
+				return DownloadToLocalFile(targetFilePath, location, redirects + 1, 0);
+		}
+
+		fprintf(stdout, "unable to find 'Location' header for redirect\n");
+		return B_IO_ERROR;
+	} else {
+		if (statusCode == 0 || (statusCode / 100) == 5) {
+			fprintf(stdout, "error response from server; %" B_PRId32 " --> "
+				"retry...\n", statusCode);
+			return DownloadToLocalFile(targetFilePath, url, redirects, failures + 1);
+		}
+
+		fprintf(stdout, "unexpected response from server; %" B_PRId32 "\n",
+			statusCode);
+		return B_IO_ERROR;
+	}
+}

src/apps/haikudepot/server/AbstractServerProcess.h

@@ -6,11 +6,51 @@
 #ifndef ABSTRACT_SERVER_PROCESS_H
 #define ABSTRACT_SERVER_PROCESS_H
 
+#include <Json.h>
 #include <String.h>
+#include <Url.h>
+
+#include "StandardMetaData.h"
+
+
+#define APP_ERR_NOT_MODIFIED (B_APP_ERROR_BASE + 452)
+
 
 class AbstractServerProcess {
 public:
-			status_t			Run();
+	virtual	status_t			Run() = 0;
+
+protected:
+	virtual	void				GetStandardMetaDataPath(
+									BPath& path) const = 0;
+	virtual	void				GetStandardMetaDataJsonPath(
+									BString& jsonPath) const = 0;
+	virtual const char*			LoggingName() const = 0;
+
+			status_t			IfModifiedSinceHeaderValue(
+									BString& headerValue) const;
+			status_t			IfModifiedSinceHeaderValue(
+									BString& headerValue,
+									const BPath& metaDataPath,
+									const BString& jsonPath) const;
+
+			status_t			PopulateMetaData(
+									StandardMetaData& metaData,
+									const BPath& path,
+									const BString& jsonPath) const;
+
+			status_t			ParseJsonFromFileWithListener(
+									BJsonEventListener *listener,
+									const BPath& path) const;
+
+			status_t			DownloadToLocalFile(
+									const BPath& targetFilePath,
+									const BUrl& url,
+									uint32 redirects, uint32 failures);
+
+private:
+			bool				LooksLikeGzip(const char *pathStr) const;
+
 };
 
 #endif // ABSTRACT_SERVER_PROCESS_H

src/apps/haikudepot/server/RepositoryDataUpdateProcess.cpp

@@ -177,7 +177,7 @@ RepositoryDataUpdateProcess::Run()
 
 		// TODO: add language ISO code to the path; just 'en' for now.
 	status_t result = DownloadToLocalFile(fLocalFilePath,
-		ServerSettings::CreateFullUrl("/__repository/all_en.json.gz"),
+		ServerSettings::CreateFullUrl("/__repository/all-en.json.gz"),
 		0, 0);
 
 	if (result == B_OK) {

src/apps/haikudepot/server/ServerIconExportUpdateProcess.cpp

@@ -5,38 +5,16 @@
 
 #include "ServerIconExportUpdateProcess.h"
 
-#include <errno.h>
 #include <stdio.h>
 #include <sys/stat.h>
 #include <time.h>
 
-#include <AutoDeleter.h>
 #include <FileIO.h>
-#include <HttpRequest.h>
-#include <HttpTime.h>
-#include <Json.h>
-#include <Url.h>
-#include <UrlProtocolRoster.h>
 #include <support/ZlibCompressionAlgorithm.h>
 
 #include "ServerSettings.h"
-#include "StandardMetaDataJsonEventListener.h"
 #include "StorageUtils.h"
 #include "TarArchiveService.h"
-#include "ToFileUrlProtocolListener.h"
-
-
-#define MAX_REDIRECTS 3
-#define MAX_FAILURES 2
-
-#define HTTP_STATUS_OK 200
-#define HTTP_STATUS_FOUND 302
-#define HTTP_STATUS_NOT_MODIFIED 304
-
-#define APP_ERR_NOT_MODIFIED (B_APP_ERROR_BASE + 452)
-
-// 30 seconds
-#define TIMEOUT_MICROSECONDS 3e+7
 
 
 /*! This constructor will locate the cached data in a standardized location */
@@ -48,6 +26,11 @@ ServerIconExportUpdateProcess::ServerIconExportUpdateProcess(
 }
 
 
+ServerIconExportUpdateProcess::~ServerIconExportUpdateProcess()
+{
+}
+
+
 status_t
 ServerIconExportUpdateProcess::Run()
 {
@@ -56,7 +39,7 @@ ServerIconExportUpdateProcess::Run()
 
 	fprintf(stdout, "will start fetching icons\n");
 
-	result = _Download(tarGzFilePath);
+	result = Download(tarGzFilePath);
 
 	if (result != APP_ERR_NOT_MODIFIED) {
 		if (result != B_OK)
@@ -96,168 +79,33 @@ ServerIconExportUpdateProcess::Run()
 }
 
 
-status_t
+void
-ServerIconExportUpdateProcess::_IfModifiedSinceHeaderValue(BString& headerValue,
+ServerIconExportUpdateProcess::GetStandardMetaDataPath(BPath& path) const
-	BPath& iconMetaDataPath) const
 {
-	headerValue.SetTo("");
+	path.SetTo(fLocalStorageDirectoryPath.Path());
-	struct stat s;
+	path.Append("hicn/info.json");
-
-	if (-1 == stat(iconMetaDataPath.Path(), &s)) {
-		if (ENOENT != errno)
-			 return B_ERROR;
-
-		return B_FILE_NOT_FOUND;
-	}
-
-	StandardMetaData iconMetaData;
-	status_t result = _PopulateIconMetaData(iconMetaData, iconMetaDataPath);
-
-	if (result == B_OK) {
-
-		// An example of this output would be; 'Fri, 24 Oct 2014 19:32:27 +0000'
-
-		BDateTime modifiedDateTime = iconMetaData
-			.GetDataModifiedTimestampAsDateTime();
-		BPrivate::BHttpTime modifiedHttpTime(modifiedDateTime);
-		headerValue.SetTo(modifiedHttpTime
-			.ToString(BPrivate::B_HTTP_TIME_FORMAT_COOKIE));
-	} else {
-		fprintf(stderr, "unable to parse the icon meta-data date and time -"
-			" cannot set the 'If-Modified-Since' header\n");
-	}
-
-	return result;
 }
 
 
-status_t
+void
-ServerIconExportUpdateProcess::_IfModifiedSinceHeaderValue(BString& headerValue)
+ServerIconExportUpdateProcess::GetStandardMetaDataJsonPath(
-    const
+	BString& jsonPath) const
 {
-	BPath iconMetaDataPath(fLocalStorageDirectoryPath);
-	iconMetaDataPath.Append("hicn/info.json");
-	return _IfModifiedSinceHeaderValue(headerValue, iconMetaDataPath);
-}
-
-
-status_t
-ServerIconExportUpdateProcess::_Download(BPath& tarGzFilePath)
-{
-	return _Download(tarGzFilePath,
-		ServerSettings::CreateFullUrl("/__pkgicon/all.tar.gz"), 0, 0);
-}
-
-
-status_t
-ServerIconExportUpdateProcess::_Download(BPath& tarGzFilePath, const BUrl& url,
-	uint32 redirects, uint32 failures)
-{
-	if (redirects > MAX_REDIRECTS) {
-		fprintf(stdout, "exceeded %d redirects --> failure\n", MAX_REDIRECTS);
-		return B_IO_ERROR;
-	}
-
-	if (failures > MAX_FAILURES) {
-		fprintf(stdout, "exceeded %d failures\n", MAX_FAILURES);
-		return B_IO_ERROR;
-	}
-
-	fprintf(stdout, "will stream '%s' to [%s]\n", url.UrlString().String(),
-		tarGzFilePath.Path());
-
-	ToFileUrlProtocolListener listener(tarGzFilePath, "icon-export",
-		ServerSettings::UrlConnectionTraceLoggingEnabled());
-
-	BHttpHeaders headers;
-	ServerSettings::AugmentHeaders(headers);
-
-	BString ifModifiedSinceHeader;
-	status_t ifModifiedSinceHeaderStatus = _IfModifiedSinceHeaderValue(
-		ifModifiedSinceHeader);
-
-	if (ifModifiedSinceHeaderStatus == B_OK &&
-		ifModifiedSinceHeader.Length() > 0) {
-		headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader);
-	}
-
-	BHttpRequest *request = dynamic_cast<BHttpRequest *>(
-		BUrlProtocolRoster::MakeRequest(url, &listener));
-	ObjectDeleter<BHttpRequest> requestDeleter(request);
-	request->SetHeaders(headers);
-	request->SetMaxRedirections(0);
-	request->SetTimeout(TIMEOUT_MICROSECONDS);
-
-	thread_id thread = request->Run();
-	wait_for_thread(thread, NULL);
-
-	const BHttpResult& result = dynamic_cast<const BHttpResult&>(
-		request->Result());
-
-	int32 statusCode = result.StatusCode();
-
-	if (BHttpRequest::IsSuccessStatusCode(statusCode)) {
-		fprintf(stdout, "did complete streaming data\n");
-		return B_OK;
-	} else if (statusCode == HTTP_STATUS_NOT_MODIFIED) {
-		fprintf(stdout, "remote data has not changed since [%s]\n",
-			ifModifiedSinceHeader.String());
-		return APP_ERR_NOT_MODIFIED;
-	} else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) {
-		const BHttpHeaders responseHeaders = result.Headers();
-		const char *locationValue = responseHeaders["Location"];
-
-		if (locationValue != NULL && strlen(locationValue) != 0) {
-			BUrl location(result.Url(), locationValue);
-			fprintf(stdout, "will redirect to; %s\n",
-				location.UrlString().String());
-				return _Download(tarGzFilePath, location, redirects + 1, 0);
-		}
-
-		fprintf(stdout, "unable to find 'Location' header for redirect\n");
-		return B_IO_ERROR;
-	} else {
-		if (statusCode == 0 || (statusCode / 100) == 5) {
-			fprintf(stdout, "error response from server; %" B_PRId32 " --> "
-				"retry...\n", statusCode);
-			return _Download(tarGzFilePath, url, redirects, failures + 1);
-		}
-
-		fprintf(stdout, "unexpected response from server; %" B_PRId32 "\n",
-			statusCode);
-		return B_IO_ERROR;
-	}
-}
-
-
-status_t
-ServerIconExportUpdateProcess::_PopulateIconMetaData(
-	StandardMetaData& iconMetaData, BPath& path) const
-{
-	FILE *file = fopen(path.Path(), "rb");
-
-	if (file == NULL) {
-		fprintf(stderr, "unable to find the icon meta data file at [%s]\n",
-			path.Path());
-		return B_FILE_NOT_FOUND;
-	}
-
-	BFileIO iconMetaDataFile(file, true); // takes ownership
 		// the "$" here indicates that the data is at the top level.
-	StandardMetaDataJsonEventListener listener("$", iconMetaData);
+	jsonPath.SetTo("$");
-	BPrivate::BJson::Parse(&iconMetaDataFile, &listener);
-
-	status_t result = listener.ErrorStatus();
-
-	if (result != B_OK)
-		return result;
-
-	if (!iconMetaData.IsPopulated()) {
-		fprintf(stderr, "the icon meta data was read from [%s], but no values "
-			"were extracted\n", path.Path());
-		return B_BAD_DATA;
-	}
-
-	return B_OK;
 }
 
+
+const char*
+ServerIconExportUpdateProcess::LoggingName() const
+{
+	return "icon-export-update";
+}
+
+
+status_t
+ServerIconExportUpdateProcess::Download(BPath& tarGzFilePath)
+{
+	return DownloadToLocalFile(tarGzFilePath,
+		ServerSettings::CreateFullUrl("/__pkgicon/all.tar.gz"), 0, 0);
+}

src/apps/haikudepot/server/ServerIconExportUpdateProcess.h

@@ -3,8 +3,8 @@
  * All rights reserved. Distributed under the terms of the MIT License.
  */
 
-#ifndef SERVER_ICON_PROCESS_H
+#ifndef SERVER_ICON_EXPORT_UPDATE_PROCESS_H
-#define SERVER_ICON_PROCESS_H
+#define SERVER_ICON_EXPORT_UPDATE_PROCESS_H
 
 
 #include "AbstractServerProcess.h"
@@ -14,35 +14,28 @@
 #include <String.h>
 #include <Url.h>
 
-#include "StandardMetaData.h"
-
 
 class ServerIconExportUpdateProcess : public AbstractServerProcess {
 public:
 
 								ServerIconExportUpdateProcess(
 									const BPath& localStorageDirectoryPath);
+	virtual						~ServerIconExportUpdateProcess();
 
 			status_t			Run();
 
+protected:
+			void				GetStandardMetaDataPath(BPath& path) const;
+			void				GetStandardMetaDataJsonPath(
+									BString& jsonPath) const;
+			const char*			LoggingName() const;
+
+
 private:
-			status_t			_Download(BPath& tarGzFilePath);
+			status_t			Download(BPath& tarGzFilePath);
-			status_t			_Download(BPath& tarGzFilePath, const BUrl& url,
-									uint32 redirects, uint32 failures);
-			BString				_FormFullUrl(const BString& suffix) const;
-			status_t			_IfModifiedSinceHeaderValue(
-									BString& headerValue) const;
-			status_t			_IfModifiedSinceHeaderValue(
-									BString& headerValue,
-									BPath& iconMetaDataPath) const;
 
-			status_t			_PopulateIconMetaData(
-									StandardMetaData& iconMetaData, BPath& path)
-									const;
-
-			BString				fBaseUrl;
 			BPath				fLocalStorageDirectoryPath;
 
 };
 
-#endif // SERVER_ICON_PROCESS_H
+#endif // SERVER_ICON_EXPORT_UPDATE_PROCESS_H

src/apps/haikudepot/ui/MainWindow.cpp

@@ -851,12 +851,6 @@ MainWindow::_RefreshPackageList(bool force)
 
 		if (getRepositoryConfigStatus == B_OK) {
 			depotInfo.SetBaseURL(repoConfig.BaseURL());
-
-			// it would be nice if this could be more logically located such as
-			// when the repository is added to the model, but that is probably
-			// a bigger change.
-
-			fModel.PopulateWebAppRepositoryCode(depotInfo);
 		} else {
 			printf("unable to obtain the repository config for local "
 				"repository '%s'; %s\n",
@@ -1000,6 +994,8 @@ MainWindow::_RefreshPackageList(bool force)
 			fModel.AddDepot(it->second);
 	}
 
+	fModel.PopulateWebAppRepositoryCodes();
+
 	// start retrieving package icons and average ratings
 	if (force || wasEmpty)
 		fModel.PopulateAllPackages();

src/tests/apps/haikudepot/HaikuDepotTestAddon.cpp

@@ -7,7 +7,7 @@
 #include <TestSuiteAddon.h>
 
 #include "StandardMetaDataJsonEventListenerTest.h"
-
+#include "DumpExportRepositoryJsonListenerTest.h"
 
 BTestSuite*
 getTestSuite()
@@ -15,6 +15,7 @@ getTestSuite()
 	BTestSuite* suite = new BTestSuite("HaikuDepot");
 
 	StandardMetaDataJsonEventListenerTest::AddTests(*suite);
+	DumpExportRepositoryJsonListenerTest::AddTests(*suite);
 
 	return suite;
 }

src/tests/apps/haikudepot/Jamfile

@@ -3,13 +3,20 @@ SubDir HAIKU_TOP src tests apps haikudepot ;
 SetSubDirSupportedPlatformsBeOSCompatible ;
 AddSubDirSupportedPlatforms libbe_test ;
 
+SubDirHdrs [ FDirName $(HAIKU_TOP) src apps haikudepot ] ;
 SubDirHdrs [ FDirName $(HAIKU_TOP) src apps haikudepot server ] ;
+SubDirHdrs [ FDirName $(HAIKU_TOP) src apps haikudepot server dumpexportrepository ] ;
 
 UsePrivateHeaders shared ;
 
 UnitTestLib haikudepottest.so :
 	HaikuDepotTestAddon.cpp
 
+	DumpExportRepositorySource.cpp
+	DumpExportRepository.cpp
+	DumpExportRepositoryJsonListener.cpp
+	DumpExportRepositoryJsonListenerTest.cpp
+
 	StandardMetaData.cpp
 	StandardMetaDataJsonEventListener.cpp
 	StandardMetaDataJsonEventListenerTest.cpp
@@ -18,4 +25,7 @@ UnitTestLib haikudepottest.so :
 ;
 
 SEARCH on [ FGristFiles StandardMetaData.cpp StandardMetaDataJsonEventListener.cpp ]
-	= [ FDirName $(HAIKU_TOP) src apps haikudepot server ] ;
+	= [ FDirName $(HAIKU_TOP) src apps haikudepot server ] ;
+
+SEARCH on [ FGristFiles DumpExportRepositorySource.cpp DumpExportRepository.cpp DumpExportRepositoryJsonListener.cpp ]
+	= [ FDirName $(HAIKU_TOP) src apps haikudepot server dumpexportrepository ] ;