HaikuDepot: Change communication mechanism with server for repos
(last commit with same title only included new files - added those now) Previously the desktop application would make a number of JSON-RPC calls over HTTP to get the repositories. Now it will make a single call to get the repositories and cache the result. This uses standard HTTP cache signalling techniques and allows the server-side the ability to cache the generated data as well. Note that the model classes and parse- related classes are generated and may not be code-style compliant. They are generated from JSON schema files in the server-side project. Information about this as well as the python files used to generate the C++ classes and headers are included in the server-side project.
This commit is contained in:
Andrew Lindesay
@@ -4,7 +4,7 @@ UsePrivateHeaders interface shared storage package support net ;
|
||||
|
||||
# source directories
|
||||
local sourceDirs =
|
||||
edits_generic model textview ui ui_generic server tar util
|
||||
edits_generic model textview ui ui_generic server server/dumpexportrepository tar util
|
||||
;
|
||||
|
||||
local sourceDir ;
|
||||
@@ -74,10 +74,16 @@ Application HaikuDepot :
|
||||
SharedBitmap.cpp
|
||||
UserLoginWindow.cpp
|
||||
|
||||
# network + server - model
|
||||
DumpExportRepository.cpp
|
||||
DumpExportRepositorySource.cpp
|
||||
DumpExportRepositoryJsonListener.cpp
|
||||
|
||||
# network + server
|
||||
AbstractServerProcess.cpp
|
||||
ServerSettings.cpp
|
||||
WebAppInterface.cpp
|
||||
RepositoryDataUpdateProcess.cpp
|
||||
ServerIconExportUpdateProcess.cpp
|
||||
StandardMetaDataJsonEventListener.cpp
|
||||
StandardMetaData.cpp
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
#include "Model.h"
|
||||
#include "RepositoryDataUpdateProcess.h"
|
||||
#include "StorageUtils.h"
|
||||
|
||||
#include <ctime>
|
||||
@@ -834,59 +835,47 @@ Model::SetAuthorization(const BString& username, const BString& password,
|
||||
|
||||
// #pragma mark - private
|
||||
|
||||
/*! When bulk repository data comes down from the server, it will
|
||||
arrive as a json.gz payload. This is stored locally as a cache
|
||||
and this method will provide the on-disk storage location for
|
||||
this file.
|
||||
*/
|
||||
|
||||
void
|
||||
Model::PopulateWebAppRepositoryCode(DepotInfo& depotInfo)
|
||||
status_t
|
||||
Model::_DumpExportRepositoryDataPath(BPath& path) const
|
||||
{
|
||||
if (depotInfo.BaseURL().Length() > 0) {
|
||||
BPath repoDataPath;
|
||||
|
||||
BMessage repositoriesEnvelope;
|
||||
BMessage result;
|
||||
double total;
|
||||
StringList repositorySourceBaseURLs;
|
||||
|
||||
repositorySourceBaseURLs.Add(depotInfo.BaseURL());
|
||||
|
||||
// TODO; better API call handling around errors.
|
||||
|
||||
if (fWebAppInterface.RetrieveRepositoriesForSourceBaseURLs(
|
||||
repositorySourceBaseURLs, repositoriesEnvelope) == B_OK
|
||||
&& repositoriesEnvelope.FindMessage("result", &result) == B_OK
|
||||
&& result.FindDouble("total", &total) == B_OK) {
|
||||
|
||||
if ((int64)total > 0) {
|
||||
BMessage repositories;
|
||||
BMessage repository;
|
||||
BString repositoryCode;
|
||||
|
||||
if (result.FindMessage("items", &repositories) == B_OK
|
||||
&& repositories.FindMessage("0", &repository) == B_OK
|
||||
&& repository.FindString("code", &repositoryCode) == B_OK) {
|
||||
|
||||
depotInfo.SetWebAppRepositoryCode(repositoryCode);
|
||||
|
||||
printf("did assign web app repository code '%s' to local "
|
||||
"depot '%s'\n",
|
||||
depotInfo.WebAppRepositoryCode().String(),
|
||||
depotInfo.Name().String());
|
||||
} else {
|
||||
printf("unable to find the 'code' in the api response for "
|
||||
"local depot '%s'\n",
|
||||
depotInfo.Name().String());
|
||||
}
|
||||
} else {
|
||||
printf("unable to find a repository code for '%s'\n",
|
||||
depotInfo.BaseURL().String());
|
||||
}
|
||||
} else {
|
||||
printf("unexpected result obtaining repository code for '%s'\n",
|
||||
depotInfo.BaseURL().String());
|
||||
}
|
||||
} else {
|
||||
printf("missing base url for depot info %s --> will not obtain web app "
|
||||
"repository code\n",
|
||||
depotInfo.Name().String());
|
||||
if (find_directory(B_USER_CACHE_DIRECTORY, &repoDataPath) == B_OK
|
||||
&& repoDataPath.Append("HaikuDepot") == B_OK
|
||||
&& create_directory(repoDataPath.Path(), 0777) == B_OK
|
||||
&& repoDataPath.Append("repository-all_en.json.gz") == B_OK) {
|
||||
path.SetTo(repoDataPath.Path());
|
||||
return B_OK;
|
||||
}
|
||||
|
||||
path.Unset();
|
||||
fprintf(stdout, "unable to find the user cache file for repositories'"
|
||||
" data");
|
||||
return B_ERROR;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
Model::PopulateWebAppRepositoryCodes()
|
||||
{
|
||||
status_t result = B_OK;
|
||||
BPath dataPath;
|
||||
|
||||
result = _DumpExportRepositoryDataPath(dataPath);
|
||||
|
||||
if (result != B_OK)
|
||||
return result;
|
||||
|
||||
RepositoryDataUpdateProcess process(dataPath, &fDepots);
|
||||
result = process.Run();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -118,8 +118,7 @@ public:
|
||||
bool ShowDevelopPackages() const
|
||||
{ return fShowDevelopPackages; }
|
||||
|
||||
void PopulateWebAppRepositoryCode(
|
||||
DepotInfo& depotInfo);
|
||||
status_t PopulateWebAppRepositoryCodes();
|
||||
|
||||
// Retrieve package information
|
||||
static const uint32 POPULATE_CACHED_RATING = 1 << 0;
|
||||
@@ -152,6 +151,8 @@ public:
|
||||
|
||||
|
||||
private:
|
||||
status_t _DumpExportRepositoryDataPath(
|
||||
BPath& path) const;
|
||||
void _UpdateIsFeaturedFilter();
|
||||
|
||||
static int32 _PopulateAllPackagesEntry(void* cookie);
|
||||
|
||||
@@ -4,9 +4,240 @@
|
||||
*/
|
||||
#include "AbstractServerProcess.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <AutoDeleter.h>
|
||||
#include <FileIO.h>
|
||||
#include <HttpRequest.h>
|
||||
#include <HttpTime.h>
|
||||
#include <UrlProtocolRoster.h>
|
||||
|
||||
#include <support/ZlibCompressionAlgorithm.h>
|
||||
|
||||
#include "ServerSettings.h"
|
||||
#include "StandardMetaDataJsonEventListener.h"
|
||||
#include "ToFileUrlProtocolListener.h"
|
||||
|
||||
|
||||
#define MAX_REDIRECTS 3
|
||||
#define MAX_FAILURES 2
|
||||
|
||||
#define HTTP_STATUS_FOUND 302
|
||||
#define HTTP_STATUS_NOT_MODIFIED 304
|
||||
|
||||
// 30 seconds
|
||||
#define TIMEOUT_MICROSECONDS 3e+7
|
||||
|
||||
|
||||
status_t
|
||||
AbstractServerProcess::Run()
|
||||
AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue) const
|
||||
{
|
||||
BPath metaDataPath;
|
||||
BString jsonPath;
|
||||
|
||||
GetStandardMetaDataPath(metaDataPath);
|
||||
GetStandardMetaDataJsonPath(jsonPath);
|
||||
|
||||
return IfModifiedSinceHeaderValue(headerValue, metaDataPath, jsonPath);
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
AbstractServerProcess::IfModifiedSinceHeaderValue(BString& headerValue,
|
||||
const BPath& metaDataPath, const BString& jsonPath) const
|
||||
{
|
||||
headerValue.SetTo("");
|
||||
struct stat s;
|
||||
|
||||
if (-1 == stat(metaDataPath.Path(), &s)) {
|
||||
if (ENOENT != errno)
|
||||
return B_ERROR;
|
||||
|
||||
return B_FILE_NOT_FOUND;
|
||||
}
|
||||
|
||||
if (s.st_size == 0)
|
||||
return B_BAD_VALUE;
|
||||
|
||||
StandardMetaData metaData;
|
||||
status_t result = PopulateMetaData(metaData, metaDataPath, jsonPath);
|
||||
|
||||
if (result == B_OK) {
|
||||
|
||||
// An example of this output would be; 'Fri, 24 Oct 2014 19:32:27 +0000'
|
||||
|
||||
BDateTime modifiedDateTime = metaData
|
||||
.GetDataModifiedTimestampAsDateTime();
|
||||
BPrivate::BHttpTime modifiedHttpTime(modifiedDateTime);
|
||||
headerValue.SetTo(modifiedHttpTime
|
||||
.ToString(BPrivate::B_HTTP_TIME_FORMAT_COOKIE));
|
||||
} else {
|
||||
fprintf(stderr, "unable to parse the meta-data date and time -"
|
||||
" cannot set the 'If-Modified-Since' header\n");
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
AbstractServerProcess::PopulateMetaData(
|
||||
StandardMetaData& metaData, const BPath& path,
|
||||
const BString& jsonPath) const
|
||||
{
|
||||
StandardMetaDataJsonEventListener listener(jsonPath, metaData);
|
||||
status_t result = ParseJsonFromFileWithListener(&listener, path);
|
||||
|
||||
if (result != B_OK)
|
||||
return result;
|
||||
|
||||
result = listener.ErrorStatus();
|
||||
|
||||
if (result != B_OK)
|
||||
return result;
|
||||
|
||||
if (!metaData.IsPopulated()) {
|
||||
fprintf(stderr, "the meta data was read from [%s], but no values "
|
||||
"were extracted\n", path.Path());
|
||||
return B_BAD_DATA;
|
||||
}
|
||||
|
||||
return B_OK;
|
||||
}
|
||||
|
||||
|
||||
bool
|
||||
AbstractServerProcess::LooksLikeGzip(const char *pathStr) const
|
||||
{
|
||||
int l = strlen(pathStr);
|
||||
return l > 4 && 0 == strncmp(&pathStr[l - 3], ".gz", 3);
|
||||
}
|
||||
|
||||
|
||||
/*! Note that a B_OK return code from this method may not indicate that the
|
||||
listening process went well. One has to see if there was an error in
|
||||
the listener.
|
||||
*/
|
||||
|
||||
status_t
|
||||
AbstractServerProcess::ParseJsonFromFileWithListener(
|
||||
BJsonEventListener *listener,
|
||||
const BPath& path) const
|
||||
{
|
||||
const char* pathStr = path.Path();
|
||||
FILE* file = fopen(pathStr, "rb");
|
||||
|
||||
if (file == NULL) {
|
||||
fprintf(stderr, "unable to find the meta data file at [%s]\n",
|
||||
path.Path());
|
||||
return B_FILE_NOT_FOUND;
|
||||
}
|
||||
|
||||
BFileIO rawInput(file, true); // takes ownership
|
||||
|
||||
// if the file extension ends with '.gz' then the data will be
|
||||
// compressed and the algorithm needs to decompress the data as
|
||||
// it is parsed.
|
||||
|
||||
if (LooksLikeGzip(pathStr)) {
|
||||
BDataIO* gzDecompressedInput = NULL;
|
||||
BZlibDecompressionParameters* zlibDecompressionParameters
|
||||
= new BZlibDecompressionParameters();
|
||||
|
||||
status_t result = BZlibCompressionAlgorithm()
|
||||
.CreateDecompressingInputStream(&rawInput,
|
||||
zlibDecompressionParameters, gzDecompressedInput);
|
||||
|
||||
if (B_OK != result)
|
||||
return result;
|
||||
|
||||
ObjectDeleter<BDataIO> gzDecompressedInputDeleter(gzDecompressedInput);
|
||||
BPrivate::BJson::Parse(gzDecompressedInput, listener);
|
||||
} else {
|
||||
BPrivate::BJson::Parse(&rawInput, listener);
|
||||
}
|
||||
|
||||
return B_OK;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
AbstractServerProcess::DownloadToLocalFile(const BPath& targetFilePath,
|
||||
const BUrl& url, uint32 redirects, uint32 failures)
|
||||
{
|
||||
if (redirects > MAX_REDIRECTS) {
|
||||
fprintf(stdout, "exceeded %d redirects --> failure\n", MAX_REDIRECTS);
|
||||
return B_IO_ERROR;
|
||||
}
|
||||
|
||||
if (failures > MAX_FAILURES) {
|
||||
fprintf(stdout, "exceeded %d failures\n", MAX_FAILURES);
|
||||
return B_IO_ERROR;
|
||||
}
|
||||
|
||||
fprintf(stdout, "will stream '%s' to [%s]\n", url.UrlString().String(),
|
||||
targetFilePath.Path());
|
||||
|
||||
ToFileUrlProtocolListener listener(targetFilePath, LoggingName(),
|
||||
ServerSettings::UrlConnectionTraceLoggingEnabled());
|
||||
|
||||
BHttpHeaders headers;
|
||||
ServerSettings::AugmentHeaders(headers);
|
||||
|
||||
BString ifModifiedSinceHeader;
|
||||
status_t ifModifiedSinceHeaderStatus = IfModifiedSinceHeaderValue(
|
||||
ifModifiedSinceHeader);
|
||||
|
||||
if (ifModifiedSinceHeaderStatus == B_OK &&
|
||||
ifModifiedSinceHeader.Length() > 0) {
|
||||
headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader);
|
||||
}
|
||||
|
||||
BHttpRequest *request = dynamic_cast<BHttpRequest *>(
|
||||
BUrlProtocolRoster::MakeRequest(url, &listener));
|
||||
ObjectDeleter<BHttpRequest> requestDeleter(request);
|
||||
request->SetHeaders(headers);
|
||||
request->SetMaxRedirections(0);
|
||||
request->SetTimeout(TIMEOUT_MICROSECONDS);
|
||||
|
||||
thread_id thread = request->Run();
|
||||
wait_for_thread(thread, NULL);
|
||||
|
||||
const BHttpResult& result = dynamic_cast<const BHttpResult&>(
|
||||
request->Result());
|
||||
|
||||
int32 statusCode = result.StatusCode();
|
||||
|
||||
if (BHttpRequest::IsSuccessStatusCode(statusCode)) {
|
||||
fprintf(stdout, "did complete streaming data\n");
|
||||
return B_OK;
|
||||
} else if (statusCode == HTTP_STATUS_NOT_MODIFIED) {
|
||||
fprintf(stdout, "remote data has not changed since [%s]\n",
|
||||
ifModifiedSinceHeader.String());
|
||||
return APP_ERR_NOT_MODIFIED;
|
||||
} else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) {
|
||||
const BHttpHeaders responseHeaders = result.Headers();
|
||||
const char *locationValue = responseHeaders["Location"];
|
||||
|
||||
if (locationValue != NULL && strlen(locationValue) != 0) {
|
||||
BUrl location(result.Url(), locationValue);
|
||||
fprintf(stdout, "will redirect to; %s\n",
|
||||
location.UrlString().String());
|
||||
return DownloadToLocalFile(targetFilePath, location, redirects + 1, 0);
|
||||
}
|
||||
|
||||
fprintf(stdout, "unable to find 'Location' header for redirect\n");
|
||||
return B_IO_ERROR;
|
||||
} else {
|
||||
if (statusCode == 0 || (statusCode / 100) == 5) {
|
||||
fprintf(stdout, "error response from server; %" B_PRId32 " --> "
|
||||
"retry...\n", statusCode);
|
||||
return DownloadToLocalFile(targetFilePath, url, redirects, failures + 1);
|
||||
}
|
||||
|
||||
fprintf(stdout, "unexpected response from server; %" B_PRId32 "\n",
|
||||
statusCode);
|
||||
return B_IO_ERROR;
|
||||
}
|
||||
}
|
||||
@@ -6,11 +6,51 @@
|
||||
#ifndef ABSTRACT_SERVER_PROCESS_H
|
||||
#define ABSTRACT_SERVER_PROCESS_H
|
||||
|
||||
#include <Json.h>
|
||||
#include <String.h>
|
||||
#include <Url.h>
|
||||
|
||||
#include "StandardMetaData.h"
|
||||
|
||||
|
||||
#define APP_ERR_NOT_MODIFIED (B_APP_ERROR_BASE + 452)
|
||||
|
||||
|
||||
class AbstractServerProcess {
|
||||
public:
|
||||
status_t Run();
|
||||
virtual status_t Run() = 0;
|
||||
|
||||
protected:
|
||||
virtual void GetStandardMetaDataPath(
|
||||
BPath& path) const = 0;
|
||||
virtual void GetStandardMetaDataJsonPath(
|
||||
BString& jsonPath) const = 0;
|
||||
virtual const char* LoggingName() const = 0;
|
||||
|
||||
status_t IfModifiedSinceHeaderValue(
|
||||
BString& headerValue) const;
|
||||
status_t IfModifiedSinceHeaderValue(
|
||||
BString& headerValue,
|
||||
const BPath& metaDataPath,
|
||||
const BString& jsonPath) const;
|
||||
|
||||
status_t PopulateMetaData(
|
||||
StandardMetaData& metaData,
|
||||
const BPath& path,
|
||||
const BString& jsonPath) const;
|
||||
|
||||
status_t ParseJsonFromFileWithListener(
|
||||
BJsonEventListener *listener,
|
||||
const BPath& path) const;
|
||||
|
||||
status_t DownloadToLocalFile(
|
||||
const BPath& targetFilePath,
|
||||
const BUrl& url,
|
||||
uint32 redirects, uint32 failures);
|
||||
|
||||
private:
|
||||
bool LooksLikeGzip(const char *pathStr) const;
|
||||
|
||||
};
|
||||
|
||||
#endif // ABSTRACT_SERVER_PROCESS_H
|
||||
|
||||
@@ -177,7 +177,7 @@ RepositoryDataUpdateProcess::Run()
|
||||
|
||||
// TODO: add language ISO code to the path; just 'en' for now.
|
||||
status_t result = DownloadToLocalFile(fLocalFilePath,
|
||||
ServerSettings::CreateFullUrl("/__repository/all_en.json.gz"),
|
||||
ServerSettings::CreateFullUrl("/__repository/all-en.json.gz"),
|
||||
0, 0);
|
||||
|
||||
if (result == B_OK) {
|
||||
|
||||
@@ -5,38 +5,16 @@
|
||||
|
||||
#include "ServerIconExportUpdateProcess.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
|
||||
#include <AutoDeleter.h>
|
||||
#include <FileIO.h>
|
||||
#include <HttpRequest.h>
|
||||
#include <HttpTime.h>
|
||||
#include <Json.h>
|
||||
#include <Url.h>
|
||||
#include <UrlProtocolRoster.h>
|
||||
#include <support/ZlibCompressionAlgorithm.h>
|
||||
|
||||
#include "ServerSettings.h"
|
||||
#include "StandardMetaDataJsonEventListener.h"
|
||||
#include "StorageUtils.h"
|
||||
#include "TarArchiveService.h"
|
||||
#include "ToFileUrlProtocolListener.h"
|
||||
|
||||
|
||||
#define MAX_REDIRECTS 3
|
||||
#define MAX_FAILURES 2
|
||||
|
||||
#define HTTP_STATUS_OK 200
|
||||
#define HTTP_STATUS_FOUND 302
|
||||
#define HTTP_STATUS_NOT_MODIFIED 304
|
||||
|
||||
#define APP_ERR_NOT_MODIFIED (B_APP_ERROR_BASE + 452)
|
||||
|
||||
// 30 seconds
|
||||
#define TIMEOUT_MICROSECONDS 3e+7
|
||||
|
||||
|
||||
/*! This constructor will locate the cached data in a standardized location */
|
||||
@@ -48,6 +26,11 @@ ServerIconExportUpdateProcess::ServerIconExportUpdateProcess(
|
||||
}
|
||||
|
||||
|
||||
ServerIconExportUpdateProcess::~ServerIconExportUpdateProcess()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::Run()
|
||||
{
|
||||
@@ -56,7 +39,7 @@ ServerIconExportUpdateProcess::Run()
|
||||
|
||||
fprintf(stdout, "will start fetching icons\n");
|
||||
|
||||
result = _Download(tarGzFilePath);
|
||||
result = Download(tarGzFilePath);
|
||||
|
||||
if (result != APP_ERR_NOT_MODIFIED) {
|
||||
if (result != B_OK)
|
||||
@@ -96,168 +79,33 @@ ServerIconExportUpdateProcess::Run()
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::_IfModifiedSinceHeaderValue(BString& headerValue,
|
||||
BPath& iconMetaDataPath) const
|
||||
void
|
||||
ServerIconExportUpdateProcess::GetStandardMetaDataPath(BPath& path) const
|
||||
{
|
||||
headerValue.SetTo("");
|
||||
struct stat s;
|
||||
|
||||
if (-1 == stat(iconMetaDataPath.Path(), &s)) {
|
||||
if (ENOENT != errno)
|
||||
return B_ERROR;
|
||||
|
||||
return B_FILE_NOT_FOUND;
|
||||
}
|
||||
|
||||
StandardMetaData iconMetaData;
|
||||
status_t result = _PopulateIconMetaData(iconMetaData, iconMetaDataPath);
|
||||
|
||||
if (result == B_OK) {
|
||||
|
||||
// An example of this output would be; 'Fri, 24 Oct 2014 19:32:27 +0000'
|
||||
|
||||
BDateTime modifiedDateTime = iconMetaData
|
||||
.GetDataModifiedTimestampAsDateTime();
|
||||
BPrivate::BHttpTime modifiedHttpTime(modifiedDateTime);
|
||||
headerValue.SetTo(modifiedHttpTime
|
||||
.ToString(BPrivate::B_HTTP_TIME_FORMAT_COOKIE));
|
||||
} else {
|
||||
fprintf(stderr, "unable to parse the icon meta-data date and time -"
|
||||
" cannot set the 'If-Modified-Since' header\n");
|
||||
}
|
||||
|
||||
return result;
|
||||
path.SetTo(fLocalStorageDirectoryPath.Path());
|
||||
path.Append("hicn/info.json");
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::_IfModifiedSinceHeaderValue(BString& headerValue)
|
||||
const
|
||||
void
|
||||
ServerIconExportUpdateProcess::GetStandardMetaDataJsonPath(
|
||||
BString& jsonPath) const
|
||||
{
|
||||
BPath iconMetaDataPath(fLocalStorageDirectoryPath);
|
||||
iconMetaDataPath.Append("hicn/info.json");
|
||||
return _IfModifiedSinceHeaderValue(headerValue, iconMetaDataPath);
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::_Download(BPath& tarGzFilePath)
|
||||
{
|
||||
return _Download(tarGzFilePath,
|
||||
ServerSettings::CreateFullUrl("/__pkgicon/all.tar.gz"), 0, 0);
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::_Download(BPath& tarGzFilePath, const BUrl& url,
|
||||
uint32 redirects, uint32 failures)
|
||||
{
|
||||
if (redirects > MAX_REDIRECTS) {
|
||||
fprintf(stdout, "exceeded %d redirects --> failure\n", MAX_REDIRECTS);
|
||||
return B_IO_ERROR;
|
||||
}
|
||||
|
||||
if (failures > MAX_FAILURES) {
|
||||
fprintf(stdout, "exceeded %d failures\n", MAX_FAILURES);
|
||||
return B_IO_ERROR;
|
||||
}
|
||||
|
||||
fprintf(stdout, "will stream '%s' to [%s]\n", url.UrlString().String(),
|
||||
tarGzFilePath.Path());
|
||||
|
||||
ToFileUrlProtocolListener listener(tarGzFilePath, "icon-export",
|
||||
ServerSettings::UrlConnectionTraceLoggingEnabled());
|
||||
|
||||
BHttpHeaders headers;
|
||||
ServerSettings::AugmentHeaders(headers);
|
||||
|
||||
BString ifModifiedSinceHeader;
|
||||
status_t ifModifiedSinceHeaderStatus = _IfModifiedSinceHeaderValue(
|
||||
ifModifiedSinceHeader);
|
||||
|
||||
if (ifModifiedSinceHeaderStatus == B_OK &&
|
||||
ifModifiedSinceHeader.Length() > 0) {
|
||||
headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader);
|
||||
}
|
||||
|
||||
BHttpRequest *request = dynamic_cast<BHttpRequest *>(
|
||||
BUrlProtocolRoster::MakeRequest(url, &listener));
|
||||
ObjectDeleter<BHttpRequest> requestDeleter(request);
|
||||
request->SetHeaders(headers);
|
||||
request->SetMaxRedirections(0);
|
||||
request->SetTimeout(TIMEOUT_MICROSECONDS);
|
||||
|
||||
thread_id thread = request->Run();
|
||||
wait_for_thread(thread, NULL);
|
||||
|
||||
const BHttpResult& result = dynamic_cast<const BHttpResult&>(
|
||||
request->Result());
|
||||
|
||||
int32 statusCode = result.StatusCode();
|
||||
|
||||
if (BHttpRequest::IsSuccessStatusCode(statusCode)) {
|
||||
fprintf(stdout, "did complete streaming data\n");
|
||||
return B_OK;
|
||||
} else if (statusCode == HTTP_STATUS_NOT_MODIFIED) {
|
||||
fprintf(stdout, "remote data has not changed since [%s]\n",
|
||||
ifModifiedSinceHeader.String());
|
||||
return APP_ERR_NOT_MODIFIED;
|
||||
} else if (BHttpRequest::IsRedirectionStatusCode(statusCode)) {
|
||||
const BHttpHeaders responseHeaders = result.Headers();
|
||||
const char *locationValue = responseHeaders["Location"];
|
||||
|
||||
if (locationValue != NULL && strlen(locationValue) != 0) {
|
||||
BUrl location(result.Url(), locationValue);
|
||||
fprintf(stdout, "will redirect to; %s\n",
|
||||
location.UrlString().String());
|
||||
return _Download(tarGzFilePath, location, redirects + 1, 0);
|
||||
}
|
||||
|
||||
fprintf(stdout, "unable to find 'Location' header for redirect\n");
|
||||
return B_IO_ERROR;
|
||||
} else {
|
||||
if (statusCode == 0 || (statusCode / 100) == 5) {
|
||||
fprintf(stdout, "error response from server; %" B_PRId32 " --> "
|
||||
"retry...\n", statusCode);
|
||||
return _Download(tarGzFilePath, url, redirects, failures + 1);
|
||||
}
|
||||
|
||||
fprintf(stdout, "unexpected response from server; %" B_PRId32 "\n",
|
||||
statusCode);
|
||||
return B_IO_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::_PopulateIconMetaData(
|
||||
StandardMetaData& iconMetaData, BPath& path) const
|
||||
{
|
||||
FILE *file = fopen(path.Path(), "rb");
|
||||
|
||||
if (file == NULL) {
|
||||
fprintf(stderr, "unable to find the icon meta data file at [%s]\n",
|
||||
path.Path());
|
||||
return B_FILE_NOT_FOUND;
|
||||
}
|
||||
|
||||
BFileIO iconMetaDataFile(file, true); // takes ownership
|
||||
// the "$" here indicates that the data is at the top level.
|
||||
StandardMetaDataJsonEventListener listener("$", iconMetaData);
|
||||
BPrivate::BJson::Parse(&iconMetaDataFile, &listener);
|
||||
|
||||
status_t result = listener.ErrorStatus();
|
||||
|
||||
if (result != B_OK)
|
||||
return result;
|
||||
|
||||
if (!iconMetaData.IsPopulated()) {
|
||||
fprintf(stderr, "the icon meta data was read from [%s], but no values "
|
||||
"were extracted\n", path.Path());
|
||||
return B_BAD_DATA;
|
||||
}
|
||||
|
||||
return B_OK;
|
||||
jsonPath.SetTo("$");
|
||||
}
|
||||
|
||||
|
||||
const char*
|
||||
ServerIconExportUpdateProcess::LoggingName() const
|
||||
{
|
||||
return "icon-export-update";
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ServerIconExportUpdateProcess::Download(BPath& tarGzFilePath)
|
||||
{
|
||||
return DownloadToLocalFile(tarGzFilePath,
|
||||
ServerSettings::CreateFullUrl("/__pkgicon/all.tar.gz"), 0, 0);
|
||||
}
|
||||
@@ -3,8 +3,8 @@
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
#ifndef SERVER_ICON_PROCESS_H
|
||||
#define SERVER_ICON_PROCESS_H
|
||||
#ifndef SERVER_ICON_EXPORT_UPDATE_PROCESS_H
|
||||
#define SERVER_ICON_EXPORT_UPDATE_PROCESS_H
|
||||
|
||||
|
||||
#include "AbstractServerProcess.h"
|
||||
@@ -14,35 +14,28 @@
|
||||
#include <String.h>
|
||||
#include <Url.h>
|
||||
|
||||
#include "StandardMetaData.h"
|
||||
|
||||
|
||||
class ServerIconExportUpdateProcess : public AbstractServerProcess {
|
||||
public:
|
||||
|
||||
ServerIconExportUpdateProcess(
|
||||
const BPath& localStorageDirectoryPath);
|
||||
virtual ~ServerIconExportUpdateProcess();
|
||||
|
||||
status_t Run();
|
||||
|
||||
protected:
|
||||
void GetStandardMetaDataPath(BPath& path) const;
|
||||
void GetStandardMetaDataJsonPath(
|
||||
BString& jsonPath) const;
|
||||
const char* LoggingName() const;
|
||||
|
||||
|
||||
private:
|
||||
status_t _Download(BPath& tarGzFilePath);
|
||||
status_t _Download(BPath& tarGzFilePath, const BUrl& url,
|
||||
uint32 redirects, uint32 failures);
|
||||
BString _FormFullUrl(const BString& suffix) const;
|
||||
status_t _IfModifiedSinceHeaderValue(
|
||||
BString& headerValue) const;
|
||||
status_t _IfModifiedSinceHeaderValue(
|
||||
BString& headerValue,
|
||||
BPath& iconMetaDataPath) const;
|
||||
status_t Download(BPath& tarGzFilePath);
|
||||
|
||||
status_t _PopulateIconMetaData(
|
||||
StandardMetaData& iconMetaData, BPath& path)
|
||||
const;
|
||||
|
||||
BString fBaseUrl;
|
||||
BPath fLocalStorageDirectoryPath;
|
||||
|
||||
};
|
||||
|
||||
#endif // SERVER_ICON_PROCESS_H
|
||||
#endif // SERVER_ICON_EXPORT_UPDATE_PROCESS_H
|
||||
|
||||
@@ -851,12 +851,6 @@ MainWindow::_RefreshPackageList(bool force)
|
||||
|
||||
if (getRepositoryConfigStatus == B_OK) {
|
||||
depotInfo.SetBaseURL(repoConfig.BaseURL());
|
||||
|
||||
// it would be nice if this could be more logically located such as
|
||||
// when the repository is added to the model, but that is probably
|
||||
// a bigger change.
|
||||
|
||||
fModel.PopulateWebAppRepositoryCode(depotInfo);
|
||||
} else {
|
||||
printf("unable to obtain the repository config for local "
|
||||
"repository '%s'; %s\n",
|
||||
@@ -1000,6 +994,8 @@ MainWindow::_RefreshPackageList(bool force)
|
||||
fModel.AddDepot(it->second);
|
||||
}
|
||||
|
||||
fModel.PopulateWebAppRepositoryCodes();
|
||||
|
||||
// start retrieving package icons and average ratings
|
||||
if (force || wasEmpty)
|
||||
fModel.PopulateAllPackages();
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
#include <TestSuiteAddon.h>
|
||||
|
||||
#include "StandardMetaDataJsonEventListenerTest.h"
|
||||
|
||||
#include "DumpExportRepositoryJsonListenerTest.h"
|
||||
|
||||
BTestSuite*
|
||||
getTestSuite()
|
||||
@@ -15,6 +15,7 @@ getTestSuite()
|
||||
BTestSuite* suite = new BTestSuite("HaikuDepot");
|
||||
|
||||
StandardMetaDataJsonEventListenerTest::AddTests(*suite);
|
||||
DumpExportRepositoryJsonListenerTest::AddTests(*suite);
|
||||
|
||||
return suite;
|
||||
}
|
||||
|
||||
@@ -3,13 +3,20 @@ SubDir HAIKU_TOP src tests apps haikudepot ;
|
||||
SetSubDirSupportedPlatformsBeOSCompatible ;
|
||||
AddSubDirSupportedPlatforms libbe_test ;
|
||||
|
||||
SubDirHdrs [ FDirName $(HAIKU_TOP) src apps haikudepot ] ;
|
||||
SubDirHdrs [ FDirName $(HAIKU_TOP) src apps haikudepot server ] ;
|
||||
SubDirHdrs [ FDirName $(HAIKU_TOP) src apps haikudepot server dumpexportrepository ] ;
|
||||
|
||||
UsePrivateHeaders shared ;
|
||||
|
||||
UnitTestLib haikudepottest.so :
|
||||
HaikuDepotTestAddon.cpp
|
||||
|
||||
DumpExportRepositorySource.cpp
|
||||
DumpExportRepository.cpp
|
||||
DumpExportRepositoryJsonListener.cpp
|
||||
DumpExportRepositoryJsonListenerTest.cpp
|
||||
|
||||
StandardMetaData.cpp
|
||||
StandardMetaDataJsonEventListener.cpp
|
||||
StandardMetaDataJsonEventListenerTest.cpp
|
||||
@@ -18,4 +25,7 @@ UnitTestLib haikudepottest.so :
|
||||
;
|
||||
|
||||
SEARCH on [ FGristFiles StandardMetaData.cpp StandardMetaDataJsonEventListener.cpp ]
|
||||
= [ FDirName $(HAIKU_TOP) src apps haikudepot server ] ;
|
||||
= [ FDirName $(HAIKU_TOP) src apps haikudepot server ] ;
|
||||
|
||||
SEARCH on [ FGristFiles DumpExportRepositorySource.cpp DumpExportRepository.cpp DumpExportRepositoryJsonListener.cpp ]
|
||||
= [ FDirName $(HAIKU_TOP) src apps haikudepot server dumpexportrepository ] ;
|
||||
Reference in New Issue
Block a user