diff --git a/src/apps/haikudepot/Jamfile b/src/apps/haikudepot/Jamfile index 30cc734b84..ce30a8516a 100644 --- a/src/apps/haikudepot/Jamfile +++ b/src/apps/haikudepot/Jamfile @@ -1,10 +1,10 @@ SubDir HAIKU_TOP src apps haikudepot ; -UsePrivateHeaders interface shared package ; +UsePrivateHeaders interface shared package support ; # source directories local sourceDirs = - edits_generic model textview ui ui_generic + edits_generic model textview ui ui_generic server tar util ; local sourceDir ; @@ -49,6 +49,7 @@ Application HaikuDepot : DecisionProvider.cpp FeaturedPackagesView.cpp FilterView.cpp + LocalIconStore.cpp JobStateListener.cpp LinkView.cpp LinkedBitmapView.cpp @@ -72,8 +73,22 @@ Application HaikuDepot : ScrollableGroupView.cpp SharedBitmap.cpp UserLoginWindow.cpp + + # network + server + AbstractServerProcess.cpp + ServerSettings.cpp WebAppInterface.cpp - + ServerIconExportUpdateProcess.cpp + IconMetaData.cpp + + # tar + TarArchiveHeader.cpp + TarArchiveService.cpp + + #util + ToFileUrlProtocolListener.cpp + StorageUtils.cpp + # package_daemon ProblemWindow.cpp ResultWindow.cpp diff --git a/src/apps/haikudepot/model/LocalIconStore.cpp b/src/apps/haikudepot/model/LocalIconStore.cpp new file mode 100644 index 0000000000..bf604d98f4 --- /dev/null +++ b/src/apps/haikudepot/model/LocalIconStore.cpp @@ -0,0 +1,127 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#include "LocalIconStore.h" + +#include +#include + +#include "ServerIconExportUpdateProcess.h" +#include "StorageUtils.h" + + +LocalIconStore::LocalIconStore() +{ + if (_EnsureIconStoragePath(fIconStoragePath) != B_OK) + fprintf(stdout, "unable to setup icon storage\n"); +} + + +LocalIconStore::~LocalIconStore() +{ +} + + +bool +LocalIconStore::_HasIconStoragePath() const +{ + return fIconStoragePath.InitCheck() == B_OK; +} + + +/* This method will try to find an icon for the package name supplied. If an + * icon was able to be found then the method will return B_OK and will update + * the supplied path object with the path to the icon file. + */ + +status_t +LocalIconStore::TryFindIconPath(const BString& pkgName, BPath& path) const +{ + if (_HasIconStoragePath()) { + BPath bestIconPath; + BPath iconPkgPath(fIconStoragePath); + bool exists; + bool isDir; + + if ( (iconPkgPath.Append("hicn") == B_OK) + && (iconPkgPath.Append(pkgName) == B_OK) + && (StorageUtils::ExistsDirectory(iconPkgPath, &exists, &isDir) + == B_OK) + && exists + && isDir + && (_IdentifyBestIconFileAtDirectory(iconPkgPath, bestIconPath) + == B_OK) + ) { + path = bestIconPath; + return B_OK; + } + } + + path.Unset(); + return B_FILE_NOT_FOUND; +} + + +void +LocalIconStore::UpdateFromServerIfNecessary() const +{ + if (_HasIconStoragePath()) { + BPath iconStoragePath(fIconStoragePath); + ServerIconExportUpdateProcess service(iconStoragePath); + service.Run(); + } +} + + +status_t +LocalIconStore::_EnsureIconStoragePath(BPath& path) const +{ + BPath iconStoragePath; + + if (find_directory(B_USER_CACHE_DIRECTORY, &iconStoragePath) == B_OK + && iconStoragePath.Append("HaikuDepot") == B_OK + && iconStoragePath.Append("__allicons") == B_OK + && create_directory(iconStoragePath.Path(), 0777) == B_OK) { + path.SetTo(iconStoragePath.Path()); + return B_OK; + } + + path.Unset(); + fprintf(stdout, "unable to find the user cache directory for icons"); + return B_ERROR; +} + + +status_t +LocalIconStore::_IdentifyBestIconFileAtDirectory(const BPath& directory, + BPath& bestIconPath) const +{ + StringList iconLeafnames; + + iconLeafnames.Add("icon.hvif"); + iconLeafnames.Add("64.png"); + iconLeafnames.Add("32.png"); + iconLeafnames.Add("16.png"); + + bestIconPath.Unset(); + + for (int32 i = 0; i < iconLeafnames.CountItems(); i++) { + BString iconLeafname = iconLeafnames.ItemAt(i); + BPath workingPath(directory); + bool exists; + bool isDir; + + if ( (workingPath.Append(iconLeafname) == B_OK + && StorageUtils::ExistsDirectory( + workingPath, &exists, &isDir) == B_OK) + && exists + && !isDir) { + bestIconPath.SetTo(workingPath.Path()); + return B_OK; + } + } + + return B_FILE_NOT_FOUND; +} diff --git a/src/apps/haikudepot/model/LocalIconStore.h b/src/apps/haikudepot/model/LocalIconStore.h new file mode 100644 index 0000000000..350455a99c --- /dev/null +++ b/src/apps/haikudepot/model/LocalIconStore.h @@ -0,0 +1,37 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + + +#ifndef LOCAL_ICON_STORE_H +#define LOCAL_ICON_STORE_H + +#include +#include +#include + +#include "PackageInfo.h" + + +class LocalIconStore { +public: + LocalIconStore(); + virtual ~LocalIconStore(); + status_t TryFindIconPath(const BString& pkgName, + BPath& path) const; + void UpdateFromServerIfNecessary() const; + +private: + bool _HasIconStoragePath() const; + status_t _EnsureIconStoragePath(BPath& path) const; + status_t _IdentifyBestIconFileAtDirectory( + const BPath& directory, + BPath& bestIconPath) const; + + BPath fIconStoragePath; + +}; + + +#endif // LOCAL_ICON_STORE_H diff --git a/src/apps/haikudepot/model/Model.cpp b/src/apps/haikudepot/model/Model.cpp index 94757a2ef5..05f2c04e11 100644 --- a/src/apps/haikudepot/model/Model.cpp +++ b/src/apps/haikudepot/model/Model.cpp @@ -1,11 +1,12 @@ /* * Copyright 2013-2014, Stephan Aßmus . * Copyright 2014, Axel Dörfler . - * Copyright 2016, Andrew Lindesay . + * Copyright 2016-2017, Andrew Lindesay . * All rights reserved. Distributed under the terms of the MIT License. */ #include "Model.h" +#include "StorageUtils.h" #include #include @@ -642,7 +643,7 @@ Model::SetShowDevelopPackages(bool show) } -// #pragma mark - information retrival +// #pragma mark - information retrieval void @@ -905,17 +906,60 @@ Model::_PopulateAllPackagesEntry(void* cookie) Model* model = static_cast(cookie); model->_PopulateAllPackagesThread(true); model->_PopulateAllPackagesThread(false); + model->_PopulateAllPackagesIcons(); return 0; } +void +Model::_PopulateAllPackagesIcons() +{ + fLocalIconStore.UpdateFromServerIfNecessary(); + + int32 depotIndex = 0; + int32 packageIndex = 0; + int32 countIconsSet = 0; + + fprintf(stdout, "will populate all packages' icons\n"); + + while (true) { + PackageInfoRef package; + BAutolock locker(&fLock); + + if (depotIndex > fDepots.CountItems()) { + fprintf(stdout, "did populate %ld packages' icons\n", + countIconsSet); + return; + } + + const DepotInfo& depot = fDepots.ItemAt(depotIndex); + const PackageList& packages = depot.Packages(); + + if (packageIndex >= packages.CountItems()) { + // Need the next depot + packageIndex = 0; + depotIndex++; + } else { + package = packages.ItemAt(packageIndex); +#ifdef DEBUG + fprintf(stdout, "will populate package icon for [%s]\n", + package->Name().String()); +#endif + if (_PopulatePackageIcon(package) == B_OK) + countIconsSet++; + + packageIndex++; + } + } +} + + void Model::_PopulateAllPackagesThread(bool fromCacheOnly) { int32 depotIndex = 0; int32 packageIndex = 0; PackageList bulkPackageList; - PackageList packagesWithIconsList; while (!fStopPopulatingAllPackages) { // Obtain PackageInfoRef while keeping the depot and package lists @@ -946,30 +990,15 @@ Model::_PopulateAllPackagesThread(bool fromCacheOnly) //_PopulatePackageInfo(package, fromCacheOnly); bulkPackageList.Add(package); if (bulkPackageList.CountItems() == 50) { - _PopulatePackageInfos(bulkPackageList, fromCacheOnly, - packagesWithIconsList); + _PopulatePackageInfos(bulkPackageList, fromCacheOnly); bulkPackageList.Clear(); } - if (fromCacheOnly) - _PopulatePackageIcon(package, fromCacheOnly); // TODO: Average user rating. It needs to be shown in the // list view, so without the user clicking the package. } if (bulkPackageList.CountItems() > 0) { - _PopulatePackageInfos(bulkPackageList, fromCacheOnly, - packagesWithIconsList); - } - - if (!fromCacheOnly) { - for (int i = packagesWithIconsList.CountItems() - 1; i >= 0; i--) { - if (fStopPopulatingAllPackages) - break; - const PackageInfoRef& package = packagesWithIconsList.ItemAtFast(i); - printf("Getting/Updating native icon for %s\n", - package->Name().String()); - _PopulatePackageIcon(package, fromCacheOnly); - } + _PopulatePackageInfos(bulkPackageList, fromCacheOnly); } } @@ -1012,8 +1041,7 @@ Model::_GetCacheFile(BPath& path, BFile& file, directory_which directory, void -Model::_PopulatePackageInfos(PackageList& packages, bool fromCacheOnly, - PackageList& packagesWithIcons) +Model::_PopulatePackageInfos(PackageList& packages, bool fromCacheOnly) { if (fStopPopulatingAllPackages) return; @@ -1035,8 +1063,6 @@ Model::_PopulatePackageInfos(PackageList& packages, bool fromCacheOnly, BMessage pkgInfo; if (pkgInfo.Unflatten(&file) == B_OK) { _PopulatePackageInfo(package, pkgInfo); - if (_HasNativeIcon(pkgInfo)) - packagesWithIcons.Add(package); packages.Remove(i); } } @@ -1101,8 +1127,6 @@ Model::_PopulatePackageInfos(PackageList& packages, bool fromCacheOnly, const PackageInfoRef& package = packages.ItemAtFast(i); if (pkgName == package->Name()) { _PopulatePackageInfo(package, pkgInfo); - if (_HasNativeIcon(pkgInfo)) - packagesWithIcons.Add(package); // Store in cache BFile file; @@ -1136,8 +1160,8 @@ Model::_PopulatePackageInfos(PackageList& packages, bool fromCacheOnly, for (int i = count / 2; i < count; i++) secondHalf.Add(packages.ItemAtFast(i)); packages.Clear(); - _PopulatePackageInfos(firstHalf, fromCacheOnly, packagesWithIcons); - _PopulatePackageInfos(secondHalf, fromCacheOnly, packagesWithIcons); + _PopulatePackageInfos(firstHalf, fromCacheOnly); + _PopulatePackageInfos(secondHalf, fromCacheOnly); } else { while (packages.CountItems() > 0) { const PackageInfoRef& package = packages.ItemAtFast(0); @@ -1337,41 +1361,33 @@ Model::_PopulatePackageInfo(const PackageInfoRef& package, const BMessage& data) } -void -Model::_PopulatePackageIcon(const PackageInfoRef& package, bool fromCacheOnly) +status_t +Model::_PopulatePackageIcon(const PackageInfoRef& package) { - // See if there is a cached icon file - BFile iconFile; - BPath iconCachePath; - BString iconName(package->Name()); - iconName << ".hvif"; - if (_GetCacheFile(iconCachePath, iconFile, B_USER_CACHE_DIRECTORY, - "HaikuDepot", iconName, fromCacheOnly, 60 * 60)) { - // Cache file is recent enough, just use it and return. - BitmapRef bitmapRef(new(std::nothrow)SharedBitmap(iconFile), true); + BPath bestIconPath; + + if ( fLocalIconStore.TryFindIconPath( + package->Name(), bestIconPath) == B_OK) { + + BFile bestIconFile(bestIconPath.Path(), O_RDONLY); + BitmapRef bitmapRef(new(std::nothrow)SharedBitmap(bestIconFile), true); BAutolock locker(&fLock); package->SetIcon(bitmapRef); - return; + +#ifdef DEBUG + fprintf(stdout, "have set the package icon for [%s] from [%s]\n", + package->Name().String(), bestIconPath.Path()); +#endif + + return B_OK; } - if (fromCacheOnly) - return; +#ifdef DEBUG + fprintf(stdout, "did not set the package icon for [%s]; no data\n", + package->Name().String()); +#endif - // Retrieve icon from web-app - BMallocIO buffer; - - status_t status = fWebAppInterface.RetrievePackageIcon(package->Name(), - &buffer); - if (status == B_OK) { - BitmapRef bitmapRef(new(std::nothrow)SharedBitmap(buffer), true); - BAutolock locker(&fLock); - package->SetIcon(bitmapRef); - locker.Unlock(); - if (iconFile.SetTo(iconCachePath.Path(), - B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE) == B_OK) { - iconFile.Write(buffer.Buffer(), buffer.BufferLength()); - } - } + return B_FILE_NOT_FOUND; } @@ -1439,32 +1455,6 @@ Model::_PopulatePackageScreenshot(const PackageInfoRef& package, } -bool -Model::_HasNativeIcon(const BMessage& message) const -{ - BMessage pkgIcons; - if (message.FindMessage("pkgIcons", &pkgIcons) != B_OK) - return false; - - int32 index = 0; - while (true) { - BString name; - name << index++; - - BMessage typeCodeInfo; - if (pkgIcons.FindMessage(name, &typeCodeInfo) != B_OK) - break; - - BString mediaTypeCode; - if (typeCodeInfo.FindString("mediaTypeCode", &mediaTypeCode) == B_OK - && mediaTypeCode == "application/x-vnd.haiku-icon") { - return true; - } - } - return false; -} - - // #pragma mark - listener notification methods diff --git a/src/apps/haikudepot/model/Model.h b/src/apps/haikudepot/model/Model.h index 253cc430f7..a7d6459f7f 100644 --- a/src/apps/haikudepot/model/Model.h +++ b/src/apps/haikudepot/model/Model.h @@ -1,6 +1,6 @@ /* * Copyright 2013-2014, Stephan Aßmus . - * Copyright 2016, Andrew Lindesay . + * Copyright 2016-2017, Andrew Lindesay . * All rights reserved. Distributed under the terms of the MIT License. */ #ifndef MODEL_H @@ -9,6 +9,7 @@ #include #include +#include "LocalIconStore.h" #include "PackageInfo.h" #include "WebAppInterface.h" @@ -156,6 +157,8 @@ private: static int32 _PopulateAllPackagesEntry(void* cookie); void _PopulateAllPackagesThread(bool fromCacheOnly); + void _PopulateAllPackagesIcons(); + bool _GetCacheFile(BPath& path, BFile& file, directory_which directory, const char* relativeLocation, @@ -169,18 +172,15 @@ private: void _PopulatePackageInfos( PackageList& packages, - bool fromCacheOnly, - PackageList& packagesWithIcons); + bool fromCacheOnly); void _PopulatePackageInfo( const PackageInfoRef& package, bool fromCacheOnly); void _PopulatePackageInfo( const PackageInfoRef& package, const BMessage& data); - void _PopulatePackageIcon( - const PackageInfoRef& package, - bool fromCacheOnly); - bool _HasNativeIcon(const BMessage& message) const; + status_t _PopulatePackageIcon( + const PackageInfoRef& package); void _PopulatePackageScreenshot( const PackageInfoRef& package, const ScreenshotInfo& info, @@ -194,6 +194,8 @@ private: DepotList fDepots; + LocalIconStore fLocalIconStore; + CategoryRef fCategoryAudio; CategoryRef fCategoryBusiness; CategoryRef fCategoryDevelopment; diff --git a/src/apps/haikudepot/server/AbstractServerProcess.cpp b/src/apps/haikudepot/server/AbstractServerProcess.cpp new file mode 100644 index 0000000000..2fb761f3f5 --- /dev/null +++ b/src/apps/haikudepot/server/AbstractServerProcess.cpp @@ -0,0 +1,12 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ +#include "AbstractServerProcess.h" + + +status_t +AbstractServerProcess::Run() +{ + return B_OK; +} diff --git a/src/apps/haikudepot/server/AbstractServerProcess.h b/src/apps/haikudepot/server/AbstractServerProcess.h new file mode 100644 index 0000000000..8264742f98 --- /dev/null +++ b/src/apps/haikudepot/server/AbstractServerProcess.h @@ -0,0 +1,16 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#ifndef ABSTRACT_SERVER_PROCESS_H +#define ABSTRACT_SERVER_PROCESS_H + +#include + +class AbstractServerProcess { +public: + status_t Run(); +}; + +#endif // ABSTRACT_SERVER_PROCESS_H diff --git a/src/apps/haikudepot/server/IconMetaData.cpp b/src/apps/haikudepot/server/IconMetaData.cpp new file mode 100644 index 0000000000..5a4e2f4ff0 --- /dev/null +++ b/src/apps/haikudepot/server/IconMetaData.cpp @@ -0,0 +1,34 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#include "IconMetaData.h" + + +uint64_t +IconMetaData::GetCreateTimestamp() +{ + return fCreateTimestamp; +} + + +void +IconMetaData::SetCreateTimestamp(uint64_t value) +{ + fCreateTimestamp = value; +} + + +uint64_t +IconMetaData::GetDataModifiedTimestamp() +{ + return fDataModifiedTimestamp; +} + + +void +IconMetaData::SetDataModifiedTimestamp(uint64_t value) +{ + fDataModifiedTimestamp = value; +} diff --git a/src/apps/haikudepot/server/IconMetaData.h b/src/apps/haikudepot/server/IconMetaData.h new file mode 100644 index 0000000000..6d9e6d1d40 --- /dev/null +++ b/src/apps/haikudepot/server/IconMetaData.h @@ -0,0 +1,37 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + + +#ifndef ICON_META_DATA_H +#define ICON_META_DATA_H + +#include +#include +#include +#include + + +/* This class models (some of) the meta-data that is bundled into the tar file + * that is downloaded to the HaikuDepot client from the HaikuDepotServer + * application server system. The file is included in the tar-ball data. + */ + + +class IconMetaData { +public: + uint64_t GetCreateTimestamp(); + void SetCreateTimestamp(uint64_t value); + + uint64_t GetDataModifiedTimestamp(); + void SetDataModifiedTimestamp( + uint64_t value); + +private: + uint64_t fCreateTimestamp; + uint64_t fDataModifiedTimestamp; +}; + + +#endif // ICON_META_DATA_H diff --git a/src/apps/haikudepot/server/ServerIconExportUpdateProcess.cpp b/src/apps/haikudepot/server/ServerIconExportUpdateProcess.cpp new file mode 100644 index 0000000000..b1c2d8e3cd --- /dev/null +++ b/src/apps/haikudepot/server/ServerIconExportUpdateProcess.cpp @@ -0,0 +1,294 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#include "ServerIconExportUpdateProcess.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "ServerSettings.h" +#include "StorageUtils.h" +#include "TarArchiveService.h" +#include "ToFileUrlProtocolListener.h" + + +#define MAX_REDIRECTS 3 +#define MAX_FAILURES 2 + +#define HTTP_STATUS_OK 200 +#define HTTP_STATUS_FOUND 302 +#define HTTP_STATUS_NOT_MODIFIED 304 + +#define APP_ERR_NOT_MODIFIED (B_APP_ERROR_BASE + 452) + +// 30 seconds +#define TIMEOUT_MICROSECONDS 3e+7 + + +/*! This constructor will locate the cached data in a standardized location */ + +ServerIconExportUpdateProcess::ServerIconExportUpdateProcess( + const BPath& localStorageDirectoryPath) +{ + fLocalStorageDirectoryPath = localStorageDirectoryPath; +} + + +status_t +ServerIconExportUpdateProcess::Run() +{ + BPath tarGzFilePath(tmpnam(NULL)); + status_t result = B_OK; + + fprintf(stdout, "will start fetching icons\n"); + + result = _Download(tarGzFilePath); + + if (result != APP_ERR_NOT_MODIFIED) { + if (result != B_OK) + return result; + + fprintf(stdout, "delete any existing stored data\n"); + StorageUtils::RemoveDirectoryContents(fLocalStorageDirectoryPath); + + BFile *tarGzFile = new BFile(tarGzFilePath.Path(), O_RDONLY); + BDataIO* tarIn; + + BZlibDecompressionParameters* zlibDecompressionParameters + = new BZlibDecompressionParameters(); + + result = BZlibCompressionAlgorithm() + .CreateDecompressingInputStream(tarGzFile, + zlibDecompressionParameters, tarIn); + + if (result == B_OK) { + result = TarArchiveService::Unpack(*tarIn, + fLocalStorageDirectoryPath); + + if (result == B_OK) { + if (0 != remove(tarGzFilePath.Path())) { + fprintf(stdout, "unable to delete the temporary tgz path; " + "%s", tarGzFilePath.Path()); + } + } + } + + delete tarGzFile; + } + + fprintf(stdout, "did complete fetching icons\n"); + + return result; +} + + +status_t +ServerIconExportUpdateProcess::_IfModifiedSinceHeaderValue(BString& headerValue, + BPath& iconMetaDataPath) const +{ + headerValue.SetTo(""); + struct stat s; + + if (-1 == stat(iconMetaDataPath.Path(), &s)) { + if (ENOENT != errno) + return B_ERROR; + + return B_FILE_NOT_FOUND; + } + + IconMetaData iconMetaData; + status_t result = _PopulateIconMetaData(iconMetaData, iconMetaDataPath); + + if (result == B_OK) { + _TimestampToRfc2822String(iconMetaData.GetDataModifiedTimestamp(), + headerValue); + } + + return result; +} + + +status_t +ServerIconExportUpdateProcess::_IfModifiedSinceHeaderValue(BString& headerValue) + const +{ + BPath iconMetaDataPath(fLocalStorageDirectoryPath); + iconMetaDataPath.Append("hicn/info.json"); + return _IfModifiedSinceHeaderValue(headerValue, iconMetaDataPath); +} + + +status_t +ServerIconExportUpdateProcess::_Download(BPath& tarGzFilePath) +{ + BString urlString = ServerSettings::CreateFullUrl("/__pkgicon/all.tar.gz"); + return _Download(tarGzFilePath, BUrl(urlString), 0, 0); +} + + +status_t +ServerIconExportUpdateProcess::_Download(BPath& tarGzFilePath, const BUrl& url, + uint32 redirects, uint32 failures) +{ + if (redirects > MAX_REDIRECTS) { + fprintf(stdout, "exceeded %d redirects --> failure\n", MAX_REDIRECTS); + return B_IO_ERROR; + } + + if (failures > MAX_FAILURES) { + fprintf(stdout, "exceeded %d failures\n", MAX_FAILURES); + return B_IO_ERROR; + } + + fprintf(stdout, "will stream '%s' to [%s]\n", url.UrlString().String(), + tarGzFilePath.Path()); + + bool isSecure = url.Protocol() == BString("https"); + ToFileUrlProtocolListener listener(tarGzFilePath, "icon-export", + ServerSettings::UrlConnectionTraceLoggingEnabled()); + BUrlContext context; + + BHttpHeaders headers; + ServerSettings::AugmentHeaders(headers); + + BString ifModifiedSinceHeader; + status_t ifModifiedSinceHeaderStatus = _IfModifiedSinceHeaderValue( + ifModifiedSinceHeader); + + if (ifModifiedSinceHeaderStatus == B_OK && + ifModifiedSinceHeader.Length() > 0) { + headers.AddHeader("If-Modified-Since", ifModifiedSinceHeader); + } + + BHttpRequest request(url, isSecure, "HTTP", &listener, &context); + request.SetMethod(B_HTTP_GET); + request.SetHeaders(headers); + request.SetTimeout(TIMEOUT_MICROSECONDS); + + thread_id thread = request.Run(); + wait_for_thread(thread, NULL); + + const BHttpResult& result = dynamic_cast( + request.Result()); + + int32 statusCode = result.StatusCode(); + + switch (statusCode) { + case HTTP_STATUS_OK: + fprintf(stdout, "did complete streaming data\n"); + return B_OK; + + case HTTP_STATUS_NOT_MODIFIED: + fprintf(stdout, "remote data has not changed since [%s]\n", + ifModifiedSinceHeader.String()); + return APP_ERR_NOT_MODIFIED; + + case HTTP_STATUS_FOUND: // redirect + { + const BHttpHeaders responseHeaders = result.Headers(); + const char *locationValue = responseHeaders["Location"]; + + if (NULL != locationValue && 0 != strlen(locationValue)) { + BUrl location(locationValue); + fprintf(stdout, "will redirect to; %s\n", + location.UrlString().String()); + return _Download(tarGzFilePath, location, redirects + 1, 0); + } + + fprintf(stdout, "unable to find 'Location' header for redirect\n"); + return B_IO_ERROR; + } + + default: + if (0 == statusCode || 5 == (statusCode / 100)) { + fprintf(stdout, "error response from server; %zu --> " + "retry...\n", statusCode); + return _Download(tarGzFilePath, url, redirects, failures + 1); + } + + fprintf(stdout, "unexpected response from server; %zu\n", + statusCode); + return B_IO_ERROR; + } +} + + +status_t +ServerIconExportUpdateProcess::_PopulateIconMetaData(IconMetaData& iconMetaData, + BMessage& message) const +{ + status_t result = B_OK; + double value; // numeric resolution issue? + + if (result == B_OK) + result = message.FindDouble("createTimestamp", &value); + + if (result == B_OK) + iconMetaData.SetCreateTimestamp((uint64) value); + + if (result == B_OK) + result = message.FindDouble("dataModifiedTimestamp", &value); + + if (result == B_OK) + iconMetaData.SetDataModifiedTimestamp((uint64) value); + + return result; +} + + +status_t +ServerIconExportUpdateProcess::_PopulateIconMetaData(IconMetaData& iconMetaData, + BString& jsonString) const +{ + BJson parser; + BMessage infoMetaDataMessage; + status_t result = parser.Parse(infoMetaDataMessage, jsonString); + + if (result == B_OK) + return _PopulateIconMetaData(iconMetaData, infoMetaDataMessage); + + return result; +} + + +status_t +ServerIconExportUpdateProcess::_PopulateIconMetaData(IconMetaData& iconMetaData, + BPath& path) const +{ + + BString infoMetaDataStr; + status_t result = StorageUtils::AppendToString(path, infoMetaDataStr); + + if (result == B_OK) + return _PopulateIconMetaData(iconMetaData, infoMetaDataStr); + + return result; +} + + +/* The output format for this is suitable for use in the "If-Modified-Since" + * header. An example of this output would be; + * 'Fri, 24 Oct 2014 19:32:27 +0000' + */ + +void +ServerIconExportUpdateProcess::_TimestampToRfc2822String( + uint64_t timestampMillis, + BString& rfc2822String) const +{ + char output[256]; + output[0] = 0; + time_t t = (timestampMillis / 1000); + strftime(output, 256, "%a, %d %b %Y %T %z", gmtime(&t)); + rfc2822String.SetTo(output); +} + diff --git a/src/apps/haikudepot/server/ServerIconExportUpdateProcess.h b/src/apps/haikudepot/server/ServerIconExportUpdateProcess.h new file mode 100644 index 0000000000..0ead261953 --- /dev/null +++ b/src/apps/haikudepot/server/ServerIconExportUpdateProcess.h @@ -0,0 +1,58 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#ifndef SERVER_ICON_PROCESS_H +#define SERVER_ICON_PROCESS_H + + +#include "AbstractServerProcess.h" + +#include +#include +#include +#include + +#include "IconMetaData.h" + + +class ServerIconExportUpdateProcess : public AbstractServerProcess { +public: + + ServerIconExportUpdateProcess( + const BPath& localStorageDirectoryPath); + + status_t Run(); + +private: + status_t _Download(BPath& tarGzFilePath); + status_t _Download(BPath& tarGzFilePath, const BUrl& url, + uint32 redirects, uint32 failures); + BString _FormFullUrl(const BString& suffix) const; + status_t _IfModifiedSinceHeaderValue( + BString& headerValue) const; + status_t _IfModifiedSinceHeaderValue( + BString& headerValue, + BPath& iconMetaDataPath) const; + + status_t _PopulateIconMetaData( + IconMetaData& iconMetaData, BPath& path) + const; + status_t _PopulateIconMetaData( + IconMetaData& iconMetaData, + BString& jsonString) const; + status_t _PopulateIconMetaData( + IconMetaData& iconMetaData, + BMessage& message) const; + + void _TimestampToRfc2822String( + uint64_t timestampMillis, + BString& rfc2822String) const; + + BString fBaseUrl; + BPath fLocalStorageDirectoryPath; + +}; + +#endif // SERVER_ICON_PROCESS_H diff --git a/src/apps/haikudepot/server/ServerSettings.cpp b/src/apps/haikudepot/server/ServerSettings.cpp new file mode 100644 index 0000000000..d681120138 --- /dev/null +++ b/src/apps/haikudepot/server/ServerSettings.cpp @@ -0,0 +1,128 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#include "ServerSettings.h" + +#include +#include + +#include +#include +#include +#include + +#include "AutoLocker.h" + + +#define BASEURL_DEFAULT "https://depot.haiku-os.org" +#define USERAGENT_FALLBACK_VERSION "0.0.0" + + +BString ServerSettings::fBaseUrl = BString(BASEURL_DEFAULT); +BString ServerSettings::fUserAgent = BString(); +BLocker ServerSettings::fUserAgentLocker; +bool ServerSettings::fUrlConnectionTraceLogging = false; + + +status_t +ServerSettings::SetBaseUrl(const BString& value) +{ + BUrl url(value); + + if (!url.IsValid()) { + fprintf(stderr, "the url is not valid\n"); + return B_BAD_VALUE; + } + + if (url.Protocol() != "http" && url.Protocol() != "https") { + fprintf(stderr, "the url protocol must be 'http' or 'https'\n"); + return B_BAD_VALUE; + } + + fBaseUrl.SetTo(value); + + if (fBaseUrl.EndsWith("/")) { + fprintf(stderr, "will remove trailing '/' character in url base\n"); + fBaseUrl.Remove(fBaseUrl.Length() - 1, 1); + } + + return B_OK; +} + + +BString +ServerSettings::CreateFullUrl(const BString urlPathComponents) +{ + return BString(fBaseUrl) << urlPathComponents; +} + + +const BString +ServerSettings::GetUserAgent() +{ + AutoLocker lock(&fUserAgentLocker); + + if (fUserAgent.IsEmpty()) { + fUserAgent.SetTo("HaikuDepot/"); + fUserAgent.Append(_GetUserAgentVersionString()); + } + + return fUserAgent; +} + + +void +ServerSettings::EnableUrlConnectionTraceLogging() { + fUrlConnectionTraceLogging = true; +} + +bool +ServerSettings::UrlConnectionTraceLoggingEnabled() { + return fUrlConnectionTraceLogging; +} + + +const BString +ServerSettings::_GetUserAgentVersionString() +{ + app_info info; + + if (be_app->GetAppInfo(&info) != B_OK) { + fprintf(stderr, "Unable to get the application info\n"); + be_app->Quit(); + return BString(USERAGENT_FALLBACK_VERSION); + } + + BFile file(&info.ref, B_READ_ONLY); + + if (file.InitCheck() != B_OK) { + fprintf(stderr, "Unable to access the application info file\n"); + be_app->Quit(); + return BString(USERAGENT_FALLBACK_VERSION); + } + + BAppFileInfo appFileInfo(&file); + version_info versionInfo; + + if (appFileInfo.GetVersionInfo( + &versionInfo, B_APP_VERSION_KIND) != B_OK) { + fprintf(stderr, "Unable to establish the application version\n"); + be_app->Quit(); + return BString(USERAGENT_FALLBACK_VERSION); + } + + BString result; + result.SetToFormat("%" B_PRId32 ".%" B_PRId32 ".%" B_PRId32, + versionInfo.major, versionInfo.middle, versionInfo.minor); + return result; +} + +void +ServerSettings::AugmentHeaders(BHttpHeaders& headers) +{ + headers.AddHeader("User-Agent", GetUserAgent()); +} + + diff --git a/src/apps/haikudepot/server/ServerSettings.h b/src/apps/haikudepot/server/ServerSettings.h new file mode 100644 index 0000000000..cac2b6e3b2 --- /dev/null +++ b/src/apps/haikudepot/server/ServerSettings.h @@ -0,0 +1,34 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#ifndef SERVER_SETTINGS_H +#define SERVER_SETTINGS_H + +#include +#include +#include +#include + + +class ServerSettings { +public: + static status_t SetBaseUrl(const BString& baseUrl); + static const BString GetUserAgent(); + static void AugmentHeaders(BHttpHeaders& headers); + static BString CreateFullUrl( + const BString urlPathComponents); + static void EnableUrlConnectionTraceLogging(); + static bool UrlConnectionTraceLoggingEnabled(); + +private: + static const BString _GetUserAgentVersionString(); + + static BString fBaseUrl; + static BString fUserAgent; + static BLocker fUserAgentLocker; + static bool fUrlConnectionTraceLogging; +}; + +#endif // SERVER_SETTINGS_H diff --git a/src/apps/haikudepot/model/WebAppInterface.cpp b/src/apps/haikudepot/server/WebAppInterface.cpp similarity index 76% rename from src/apps/haikudepot/model/WebAppInterface.cpp rename to src/apps/haikudepot/server/WebAppInterface.cpp index d7a1f22782..58f76b1e21 100644 --- a/src/apps/haikudepot/model/WebAppInterface.cpp +++ b/src/apps/haikudepot/server/WebAppInterface.cpp @@ -1,6 +1,6 @@ /* * Copyright 2014, Stephan Aßmus . - * Copyright 2016, Andrew Lindesay . + * Copyright 2016-2017, Andrew Lindesay . * All rights reserved. Distributed under the terms of the MIT License. */ @@ -25,6 +25,7 @@ #include "AutoLocker.h" #include "List.h" #include "PackageInfo.h" +#include "ServerSettings.h" #define BASEURL_DEFAULT "https://depot.haiku-os.org" @@ -186,43 +187,36 @@ private: class ProtocolListener : public BUrlProtocolListener { public: - ProtocolListener() + ProtocolListener(bool traceLogging) : fDownloadIO(NULL), - fDebug(false) + fTraceLogging(traceLogging) + { + } + + virtual ~ProtocolListener() { } virtual void ConnectionOpened(BUrlRequest* caller) { -// printf("ConnectionOpened(%p)\n", caller); } virtual void HostnameResolved(BUrlRequest* caller, const char* ip) { -// printf("HostnameResolved(%p): %s\n", caller, ip); } virtual void ResponseStarted(BUrlRequest* caller) { - if (fDebug) - printf("ResponseStarted(%p)\n", caller); } virtual void HeadersReceived(BUrlRequest* caller) { - if (fDebug) - printf("HeadersReceived(%p)\n", caller); } virtual void DataReceived(BUrlRequest* caller, const char* data, off_t position, ssize_t size) { - if (fDebug) { - printf("DataReceived(%p): %ld bytes\n", caller, size); - printf("%.*s", (int)size, data); - } - if (fDownloadIO != NULL) fDownloadIO->Write(data, size); } @@ -230,27 +224,22 @@ public: virtual void DownloadProgress(BUrlRequest* caller, ssize_t bytesReceived, ssize_t bytesTotal) { -// printf("DownloadProgress(%p): %ld/%ld\n", caller, bytesReceived, -// bytesTotal); } virtual void UploadProgress(BUrlRequest* caller, ssize_t bytesSent, ssize_t bytesTotal) { - if (fDebug) - printf("UploadProgress(%p): %ld/%ld\n", caller, bytesSent, bytesTotal); } virtual void RequestCompleted(BUrlRequest* caller, bool success) { - if (fDebug) - printf("RequestCompleted(%p): %d\n", caller, success); } virtual void DebugMessage(BUrlRequest* caller, BUrlProtocolDebugMessage type, const char* text) { -// printf("DebugMessage(%p): %s\n", caller, text); + if (fTraceLogging) + printf("jrpc: %s\n", text); } void SetDownloadIO(BDataIO* downloadIO) @@ -258,14 +247,9 @@ public: fDownloadIO = downloadIO; } - void SetDebug(bool debug) - { - fDebug = debug; - } - private: BDataIO* fDownloadIO; - bool fDebug; + bool fTraceLogging; }; @@ -275,14 +259,9 @@ WebAppInterface::fRequestIndex = 0; enum { NEEDS_AUTHORIZATION = 1 << 0, - ENABLE_DEBUG = 1 << 1, }; -BString WebAppInterface::fBaseUrl = BString(BASEURL_DEFAULT); -BString WebAppInterface::fUserAgent = BString(); -BLocker WebAppInterface::fUserAgentLocker; - WebAppInterface::WebAppInterface() : fLanguage("en") @@ -327,108 +306,6 @@ WebAppInterface::SetAuthorization(const BString& username, } -static bool -arguments_is_url_valid(const BString& value) -{ - if (value.Length() < 8) { - fprintf(stderr, "the url is less than 8 characters in length\n"); - return false; - } - - int32 schemeEnd = value.FindFirst("://"); - - if (schemeEnd < B_OK) { - fprintf(stderr, "the url does not contain the '://' string\n"); - return false; - } - - BString scheme; - value.CopyInto(scheme, 0, schemeEnd); - - if (scheme != "http" && scheme != "https") { - fprintf(stderr, "the url scheme should be 'http' or 'https'\n"); - return false; - } - - if (value.Length() - 1 == value.FindLast("/")) { - fprintf(stderr, "the url should be be terminated with a '/'\n"); - return false; - } - - return true; -} - - -/*! This method will set the web app base URL, returning a status to - indicate if the URL was acceptable. - \return B_OK if the base URL was valid and B_BAD_VALUE if not. - */ -status_t -WebAppInterface::SetBaseUrl(const BString& url) -{ - if (!arguments_is_url_valid(url)) - return B_BAD_VALUE; - - fBaseUrl.SetTo(url); - - return B_OK; -} - - -const BString -WebAppInterface::_GetUserAgentVersionString() -{ - app_info info; - - if (be_app->GetAppInfo(&info) != B_OK) { - fprintf(stderr, "Unable to get the application info\n"); - be_app->Quit(); - return BString(USERAGENT_FALLBACK_VERSION); - } - - BFile file(&info.ref, B_READ_ONLY); - - if (file.InitCheck() != B_OK) { - fprintf(stderr, "Unable to access the application info file\n"); - be_app->Quit(); - return BString(USERAGENT_FALLBACK_VERSION); - } - - BAppFileInfo appFileInfo(&file); - version_info versionInfo; - - if (appFileInfo.GetVersionInfo( - &versionInfo, B_APP_VERSION_KIND) != B_OK) { - fprintf(stderr, "Unable to establish the application version\n"); - be_app->Quit(); - return BString(USERAGENT_FALLBACK_VERSION); - } - - BString result; - result.SetToFormat("%" B_PRId32 ".%" B_PRId32 ".%" B_PRId32, - versionInfo.major, versionInfo.middle, versionInfo.minor); - return result; -} - - -/*! This method will devise a suitable User-Agent header value that - can be transmitted with HTTP requests to the server in order - to identify this client. - */ -const BString -WebAppInterface::_GetUserAgent() -{ - AutoLocker lock(&fUserAgentLocker); - - if (fUserAgent.IsEmpty()) { - fUserAgent.SetTo("HaikuDepot/"); - fUserAgent.Append(_GetUserAgentVersionString()); - } - - return fUserAgent; -} - - void WebAppInterface::SetPreferredLanguage(const BString& language) { @@ -509,7 +386,6 @@ WebAppInterface::RetrieveBulkPackageInfo(const StringList& packageNames, .AddArray("filter") .AddItem("PKGCATEGORIES") .AddItem("PKGSCREENSHOTS") - .AddItem("PKGICONS") .AddItem("PKGVERSIONLOCALIZATIONDESCRIPTIONS") .AddItem("PKGCHANGELOG") .EndArray() @@ -521,37 +397,6 @@ WebAppInterface::RetrieveBulkPackageInfo(const StringList& packageNames, } -status_t -WebAppInterface::RetrievePackageIcon(const BString& packageName, - BDataIO* stream) -{ - BString urlString = _FormFullUrl(BString("/__pkgicon/") << packageName - << ".hvif"); - bool isSecure = 0 == urlString.FindFirst("https://"); - - BUrl url(urlString); - - ProtocolListener listener; - listener.SetDownloadIO(stream); - - BHttpRequest request(url, isSecure, "HTTP", &listener); - request.SetMethod(B_HTTP_GET); - - thread_id thread = request.Run(); - wait_for_thread(thread, NULL); - - const BHttpResult& result = dynamic_cast( - request.Result()); - - int32 statusCode = result.StatusCode(); - - if (statusCode == 200) - return B_OK; - - return B_ERROR; -} - - status_t WebAppInterface::RetrieveUserRatings(const BString& packageName, const BString& architecture, int resultOffset, int maxResults, @@ -671,17 +516,18 @@ status_t WebAppInterface::RetrieveScreenshot(const BString& code, int32 width, int32 height, BDataIO* stream) { - BString urlString = _FormFullUrl(BString("/__pkgscreenshot/") << code + BString urlString = ServerSettings::CreateFullUrl(BString("/__pkgscreenshot/") << code << ".png" << "?tw=" << width << "&th=" << height); bool isSecure = 0 == urlString.FindFirst("https://"); BUrl url(urlString); - ProtocolListener listener; + ProtocolListener listener( + ServerSettings::UrlConnectionTraceLoggingEnabled()); listener.SetDownloadIO(stream); BHttpHeaders headers; - headers.AddHeader("User-Agent", _GetUserAgent()); + ServerSettings::AugmentHeaders(headers); BHttpRequest request(url, isSecure, "HTTP", &listener); request.SetMethod(B_HTTP_GET); @@ -776,35 +622,24 @@ WebAppInterface::AuthenticateUser(const BString& nickName, // #pragma mark - private -BString -WebAppInterface::_FormFullUrl(const BString& suffix) const -{ - if (fBaseUrl.IsEmpty()) { - fprintf(stderr, "illegal state - missing web app base url\n"); - exit(EXIT_FAILURE); - } - - return BString(fBaseUrl) << suffix; -} - - status_t WebAppInterface::_SendJsonRequest(const char* domain, BString jsonString, uint32 flags, BMessage& reply) const { - if ((flags & ENABLE_DEBUG) != 0) + if (ServerSettings::UrlConnectionTraceLoggingEnabled()) printf("_SendJsonRequest(%s)\n", jsonString.String()); - BString urlString = _FormFullUrl(BString("/__api/v1/") << domain); + BString urlString = ServerSettings::CreateFullUrl(BString("/__api/v1/") << domain); bool isSecure = 0 == urlString.FindFirst("https://"); BUrl url(urlString); - ProtocolListener listener; + ProtocolListener listener( + ServerSettings::UrlConnectionTraceLoggingEnabled()); BUrlContext context; BHttpHeaders headers; headers.AddHeader("Content-Type", "application/json"); - headers.AddHeader("User-Agent", _GetUserAgent()); + ServerSettings::AugmentHeaders(headers); BHttpRequest request(url, isSecure, "HTTP", &listener, &context); request.SetMethod(B_HTTP_POST); @@ -827,7 +662,6 @@ WebAppInterface::_SendJsonRequest(const char* domain, BString jsonString, BMallocIO replyData; listener.SetDownloadIO(&replyData); - listener.SetDebug((flags & ENABLE_DEBUG) != 0); thread_id thread = request.Run(); wait_for_thread(thread, NULL); @@ -848,7 +682,8 @@ WebAppInterface::_SendJsonRequest(const char* domain, BString jsonString, BJson parser; status_t status = parser.Parse(reply, jsonString); - if ((flags & ENABLE_DEBUG) != 0 && status == B_BAD_DATA) { + if (ServerSettings::UrlConnectionTraceLoggingEnabled() && + status == B_BAD_DATA) { printf("Parser choked on JSON:\n%s\n", jsonString.String()); } return status; diff --git a/src/apps/haikudepot/model/WebAppInterface.h b/src/apps/haikudepot/server/WebAppInterface.h similarity index 87% rename from src/apps/haikudepot/model/WebAppInterface.h rename to src/apps/haikudepot/server/WebAppInterface.h index 04cf67f058..377ccf2f21 100644 --- a/src/apps/haikudepot/model/WebAppInterface.h +++ b/src/apps/haikudepot/server/WebAppInterface.h @@ -1,6 +1,6 @@ /* * Copyright 2014, Stephan Aßmus . - * Copyright 2016, Andrew Lindesay . + * Copyright 2016-2017, Andrew Lindesay . * All rights reserved. Distributed under the terms of the MIT License. */ #ifndef WEB_APP_INTERFACE_H @@ -34,7 +34,6 @@ public: const BString& Username() const { return fUsername; } - static status_t SetBaseUrl(const BString& url); void SetPreferredLanguage(const BString& language); void SetArchitecture(const BString& architecture); @@ -54,10 +53,6 @@ public: const StringList& repositoryCodes, BMessage& message); - status_t RetrievePackageIcon( - const BString& packageName, - BDataIO* stream); - status_t RetrieveUserRatings( const BString& packageName, const BString& architecture, @@ -110,17 +105,11 @@ public: BMessage& message); private: - static const BString _GetUserAgentVersionString(); - static const BString _GetUserAgent(); - BString _FormFullUrl(const BString& suffix) const; status_t _SendJsonRequest(const char* domain, BString jsonString, uint32 flags, BMessage& reply) const; private: - static BString fBaseUrl; - static BString fUserAgent; - static BLocker fUserAgentLocker; BString fUsername; BString fPassword; BString fLanguage; diff --git a/src/apps/haikudepot/tar/TarArchiveHeader.cpp b/src/apps/haikudepot/tar/TarArchiveHeader.cpp new file mode 100644 index 0000000000..07bf5b07b0 --- /dev/null +++ b/src/apps/haikudepot/tar/TarArchiveHeader.cpp @@ -0,0 +1,148 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#include "TarArchiveHeader.h" + +#include + + +#define OFFSET_FILENAME 0 +#define OFFSET_LENGTH 124 +#define OFFSET_CHECKSUM 148 +#define OFFSET_FILETYPE 156 + +#define LENGTH_FILENAME 100 +#define LENGTH_LENGTH 12 +#define LENGTH_CHECKSUM 8 +#define LENGTH_BLOCK 512 + + +TarArchiveHeader::TarArchiveHeader(const BString& fileName, uint64 length, + tar_file_type fileType) + : + fFileName(fileName), + fLength(length), + fFileType(fileType) +{ +} + +tar_file_type +TarArchiveHeader::_ReadFileType(unsigned char data) { + switch (data) { + case 0: + case '0': + return TAR_FILE_TYPE_NORMAL; + + default: + return TAR_FILE_TYPE_OTHER; + } +} + + +const BString +TarArchiveHeader::_ReadString(const unsigned char *data, size_t dataLength) +{ + uint32 actualLength = 0; + + while (actualLength < dataLength && 0 != data[actualLength]) + actualLength++; + + return BString((const char *) data, actualLength); +} + +/* + * This is an octal value represented as an ASCII string. + */ + +static bool tar_is_octal_digit(unsigned char c) +{ + switch (c) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + return true; + + default: + return false; + } +} + +uint32 +TarArchiveHeader::_ReadNumeric(const unsigned char *data, size_t dataLength) +{ + uint32 actualLength = 0; + + while (actualLength < dataLength && tar_is_octal_digit(data[actualLength])) + actualLength++; + + uint32 factor = 1; + uint32 result = 0; + + for (uint32 i = 0; i < actualLength; i++) { + result += (data[actualLength - (1 + i)] - '0') * factor; + factor *= 8; + } + + return result; +} + +uint32 +TarArchiveHeader::_CalculateChecksum(const unsigned char* block) +{ + uint32 result = 0; + + for (uint32 i = 0; i < LENGTH_BLOCK; i++) { + if (i >= OFFSET_CHECKSUM && i < OFFSET_CHECKSUM + LENGTH_CHECKSUM) + result += 32; + else + result += block[i]; + } + + return result; +} + +TarArchiveHeader* +TarArchiveHeader::CreateFromBlock(const unsigned char* block) +{ + uint32 actualChecksum = _CalculateChecksum(block); + uint32 expectedChecksum = _ReadNumeric(&block[OFFSET_CHECKSUM], + LENGTH_CHECKSUM); + + if(actualChecksum != expectedChecksum) { + fprintf(stderr, "tar archive header has bad checksum;" + "expected %zu actual %zu\n", expectedChecksum, actualChecksum); + } else { + return new TarArchiveHeader( + _ReadString(&block[OFFSET_FILENAME], LENGTH_FILENAME), + _ReadNumeric(&block[OFFSET_LENGTH], LENGTH_LENGTH), + _ReadFileType(block[OFFSET_FILETYPE])); + } + + return NULL; +} + +const BString& +TarArchiveHeader::GetFileName() const +{ + return fFileName; +} + +size_t +TarArchiveHeader::GetLength() const +{ + return fLength; +} + + +tar_file_type +TarArchiveHeader::GetFileType() const +{ + return fFileType; +} diff --git a/src/apps/haikudepot/tar/TarArchiveHeader.h b/src/apps/haikudepot/tar/TarArchiveHeader.h new file mode 100644 index 0000000000..9c92f545ab --- /dev/null +++ b/src/apps/haikudepot/tar/TarArchiveHeader.h @@ -0,0 +1,48 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#ifndef TAR_ARCHIVE_HEADER_H +#define TAR_ARCHIVE_HEADER_H + +#include + + +enum tar_file_type { + TAR_FILE_TYPE_NORMAL, + TAR_FILE_TYPE_OTHER +}; + + +/* Each file in a tar-archive has a header on it describing the next entry in + * the stream. This class models the data in the header. + */ + +class TarArchiveHeader { +public: + TarArchiveHeader(const BString& fileName, + uint64 length, tar_file_type fileType); + + static TarArchiveHeader* CreateFromBlock(const unsigned char *block); + + const BString& GetFileName() const; + size_t GetLength() const; + tar_file_type GetFileType() const; + +private: + static uint32 _CalculateChecksum( + const unsigned char* data); + static const BString _ReadString(const unsigned char* data, + size_t dataLength); + static uint32 _ReadNumeric(const unsigned char* data, + size_t dataLength); + static tar_file_type _ReadFileType(unsigned char data); + + const BString fFileName; + uint64 fLength; + tar_file_type fFileType; + +}; + +#endif // TAR_ARCHIVE_HEADER_H diff --git a/src/apps/haikudepot/tar/TarArchiveService.cpp b/src/apps/haikudepot/tar/TarArchiveService.cpp new file mode 100644 index 0000000000..249dbd25a3 --- /dev/null +++ b/src/apps/haikudepot/tar/TarArchiveService.cpp @@ -0,0 +1,185 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#include "TarArchiveService.h" +#include "StorageUtils.h" + +#include + +#include +#include +#include + + +#define LENGTH_BLOCK 512 + + +status_t +TarArchiveService::Unpack(BDataIO& tarDataIo, BPath& targetDirectory) +{ + uint8 buffer[LENGTH_BLOCK]; + uint8 zero_buffer[LENGTH_BLOCK]; + status_t result = B_OK; + uint32_t count_items_read = 0; + + fprintf(stdout, "will unpack to [%s]\n", targetDirectory.Path()); + + memset(zero_buffer, 0, sizeof zero_buffer); + + while (B_OK == result && B_OK == (result = tarDataIo.ReadExactly(buffer, + LENGTH_BLOCK))) { + + count_items_read++; + + if (0 == memcmp(zero_buffer, buffer, sizeof zero_buffer)) { + fprintf(stdout, "detected end of tar-ball\n"); + return B_OK; // end of tar-ball. + } else { + TarArchiveHeader* header = TarArchiveHeader::CreateFromBlock( + buffer); + + if (NULL == header) { + fprintf(stderr, "unable to parse a tar header\n"); + result = B_ERROR; + } + + if (B_OK == result) + result = _UnpackItem(tarDataIo, targetDirectory, *header); + + delete header; + } + } + + fprintf(stdout, "did unpack %d tar items\n", count_items_read); + + if (B_OK != result) { + fprintf(stdout, "error occurred unpacking tar items; %s\n", + strerror(result)); + } + + return result; +} + + +status_t +TarArchiveService::_EnsurePathToTarItemFile( + BPath& targetDirectoryPath, BString &tarItemPath) +{ + if (tarItemPath.Length() == 0) + return B_ERROR; + + BStringList components; + tarItemPath.Split("/", false, components); + + for (int32 i = 0; i < components.CountStrings(); i++) { + BString component = components.StringAt(i); + + if (_ValidatePathComponent(component) != B_OK) { + fprintf(stdout, "malformed component; [%s]\n", component.String()); + return B_ERROR; + } + } + + BPath parentPath; + BPath assembledPath(targetDirectoryPath); + + status_t result = assembledPath.Append(tarItemPath); + + if (result == B_OK) + result = assembledPath.GetParent(&parentPath); + + if (result == B_OK) + result = create_directory(parentPath.Path(), 0777); + + return result; +} + + +status_t +TarArchiveService::_UnpackItem(BDataIO& tarDataIo, + BPath& targetDirectoryPath, + TarArchiveHeader& header) +{ + status_t result = B_OK; + BString entryFileName = header.GetFileName(); + uint32 entryLength = header.GetLength(); + + fprintf(stdout, "will unpack item [%s] length [%zu]b\n", + entryFileName.String(), entryLength); + + // if the path ends in "/" then it is a directory and there's no need to + // unpack it although if there is a length, it will need to be skipped. + + if (!entryFileName.EndsWith("/") || + header.GetFileType() != TAR_FILE_TYPE_NORMAL) { + + result = _EnsurePathToTarItemFile(targetDirectoryPath, + entryFileName); + + if (result == B_OK) { + BPath targetFilePath(targetDirectoryPath); + targetFilePath.Append(entryFileName, false); + result = _UnpackItemData(tarDataIo, targetFilePath, entryLength); + } + } else { + off_t blocksToSkip = (entryLength / LENGTH_BLOCK); + + if (entryLength % LENGTH_BLOCK > 0) + blocksToSkip += 1; + + if (0 != blocksToSkip) { + uint8 buffer[LENGTH_BLOCK]; + + for (uint32 i = 0; B_OK == result && i < blocksToSkip; i++) + tarDataIo.ReadExactly(buffer, LENGTH_BLOCK); + } + } + + return result; +} + + +status_t +TarArchiveService::_UnpackItemData(BDataIO& tarDataIo, + BPath& targetFilePath, uint32 length) +{ + uint8 buffer[LENGTH_BLOCK]; + size_t remainingInItem = length; + status_t result = B_OK; + BFile targetFile(targetFilePath.Path(), O_WRONLY | O_CREAT); + + while (remainingInItem > 0 && + B_OK == result && + B_OK == (result = tarDataIo.ReadExactly(buffer, LENGTH_BLOCK))) { + + size_t writeFromBuffer = LENGTH_BLOCK; + + if (remainingInItem < LENGTH_BLOCK) + writeFromBuffer = remainingInItem; + + result = targetFile.WriteExactly(buffer, writeFromBuffer); + remainingInItem -= writeFromBuffer; + } + + if (result != B_OK) + fprintf(stdout, "unable to unpack item data to; [%s]\n", + targetFilePath.Path()); + + return result; +} + + +status_t +TarArchiveService::_ValidatePathComponent(const BString& component) +{ + if (component.Length() == 0) + return B_ERROR; + + if (component == ".." || component == "." || component == "~") + return B_ERROR; + + return B_OK; +} + diff --git a/src/apps/haikudepot/tar/TarArchiveService.h b/src/apps/haikudepot/tar/TarArchiveService.h new file mode 100644 index 0000000000..a8cf706ae3 --- /dev/null +++ b/src/apps/haikudepot/tar/TarArchiveService.h @@ -0,0 +1,36 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#ifndef TAR_ARCHIVE_SERVICE_H +#define TAR_ARCHIVE_SERVICE_H + +#include "AbstractServerProcess.h" +#include "TarArchiveHeader.h" + +#include +#include + + +class TarArchiveService { +public: + static status_t Unpack(BDataIO& tarDataIo, + BPath& targetDirectoryPath); + +private: + static status_t _EnsurePathToTarItemFile( + BPath& targetDirectoryPath, + BString &tarItemPath); + static status_t _ValidatePathComponent( + const BString& component); + static status_t _UnpackItem(BDataIO& tarDataIo, + BPath& targetDirectory, + TarArchiveHeader& header); + static status_t _UnpackItemData(BDataIO& tarDataIo, + BPath& targetFilePath, + uint32 length); + +}; + +#endif // TAR_ARCHIVE_SERVICE_H diff --git a/src/apps/haikudepot/ui/App.cpp b/src/apps/haikudepot/ui/App.cpp index 93758ca998..7578977453 100644 --- a/src/apps/haikudepot/ui/App.cpp +++ b/src/apps/haikudepot/ui/App.cpp @@ -5,6 +5,7 @@ #include "App.h" +#include "ServerSettings.h" #include @@ -117,29 +118,113 @@ App::RefsReceived(BMessage* message) } +enum arg_switch { + UNKNOWN_SWITCH, + NOT_SWITCH, + HELP_SWITCH, + WEB_APP_BASE_URL_SWITCH, + URL_CONNECTION_TRACE_LOGGING_SWITCH, +}; + + +static void +app_print_help() +{ + fprintf(stdout, "HaikuDepot "); + fprintf(stdout, "[-u|--webappbaseurl ] "); + fprintf(stdout, "[-h] "); + fprintf(stdout, "[-t|--urlconnectiontracelogging]\n"); +} + + +static arg_switch +app_resolve_switch(char *arg) +{ + int arglen = strlen(arg); + + if (arglen > 0 && arg[0] == '-') { + + if (arglen > 3 && arg[1] == '-') { // long form + if (0 == strcmp(&arg[2], "webappbaseurl")) + return WEB_APP_BASE_URL_SWITCH; + + if (0 == strcmp(&arg[2], "help")) + return HELP_SWITCH; + + if (0 == strcmp(&arg[2], "urlconnectiontracelogging")) + return URL_CONNECTION_TRACE_LOGGING_SWITCH; + } else { + if (arglen == 2) { // short form + switch (arg[1]) { + case 'u': + return WEB_APP_BASE_URL_SWITCH; + + case 'h': + return HELP_SWITCH; + + case 't': + return URL_CONNECTION_TRACE_LOGGING_SWITCH; + } + } + } + + return UNKNOWN_SWITCH; + } + + return NOT_SWITCH; +} + + void App::ArgvReceived(int32 argc, char* argv[]) { for (int i = 1; i < argc;) { - if (0 == strcmp("--webappbaseurl", argv[i])) { - if (i == argc-1) { - fprintf(stderr, "unexpected end of arguments; missing web app base url\n"); + switch (app_resolve_switch(argv[i])) { + + case URL_CONNECTION_TRACE_LOGGING_SWITCH: + ServerSettings::EnableUrlConnectionTraceLogging(); + break; + + case HELP_SWITCH: + app_print_help(); Quit(); + break; + + case WEB_APP_BASE_URL_SWITCH: + if (i == argc-1) { + fprintf(stdout, "unexpected end of arguments; missing " + "web-app base url\n"); + Quit(); + } + + if (ServerSettings::SetBaseUrl(argv[i + 1]) != B_OK) { + fprintf(stdout, "malformed web app base url; %s\n", + argv[i + 1]); + Quit(); + } + else { + fprintf(stdout, "did configure the web base url; %s\n", + argv[i + 1]); + } + + i++; // also move past the url value + + break; + + case NOT_SWITCH: + { + BEntry entry(argv[i], true); + _Open(entry); + break; } - if (WebAppInterface::SetBaseUrl(argv[i + 1]) != B_OK) { - fprintf(stderr, "malformed web app base url; %s\n", argv[i + 1]); + case UNKNOWN_SWITCH: + fprintf(stdout, "unknown switch; %s\n", argv[i]); Quit(); - } - else - fprintf(stderr, "did configure the web base url; %s\n",argv[i + 1]); - - i += 2; - } else { - BEntry entry(argv[i], true); - _Open(entry); - i++; + break; } + + i++; // move on at least one arg } } diff --git a/src/apps/haikudepot/util/StorageUtils.cpp b/src/apps/haikudepot/util/StorageUtils.cpp new file mode 100644 index 0000000000..35e102b921 --- /dev/null +++ b/src/apps/haikudepot/util/StorageUtils.cpp @@ -0,0 +1,105 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#include "StorageUtils.h" + +#include +#include +#include +#include + +#include +#include + +#define FILE_TO_STRING_BUFFER_LEN 64 + + +/* This method will append the contents of the file at the supplied path to the + * string provided. + */ + +status_t +StorageUtils::AppendToString(BPath& path, BString& result) +{ + BFile file(path.Path(), O_RDONLY); + uint8_t buffer[FILE_TO_STRING_BUFFER_LEN]; + size_t buffer_read; + + while((buffer_read = file.Read(buffer, FILE_TO_STRING_BUFFER_LEN)) > 0) + result.Append((char *) buffer, buffer_read); + + return (status_t) buffer_read; +} + + +/* This method will traverse the directory structure and will remove all of the + * files that are present in the directories as well as the directories + * themselves. + */ + +status_t +StorageUtils::RemoveDirectoryContents(BPath& path) +{ + BDirectory directory(path.Path()); + BEntry directoryEntry; + status_t result = B_OK; + + while (result == B_OK && + directory.GetNextEntry(&directoryEntry) != B_ENTRY_NOT_FOUND) { + + bool exists = false; + bool isDirectory = false; + BPath directroyEntryPath; + + result = directoryEntry.GetPath(&directroyEntryPath); + + if (result == B_OK) + result = ExistsDirectory(directroyEntryPath, &exists, &isDirectory); + + if (result == B_OK) { + if (isDirectory) + RemoveDirectoryContents(directroyEntryPath); + + if (remove(directroyEntryPath.Path()) == 0) { + fprintf(stdout, "did delete [%s]\n", + directroyEntryPath.Path()); + } else { + fprintf(stderr, "unable to delete [%s]\n", + directroyEntryPath.Path()); + result = B_ERROR; + } + } + + } + + return result; +} + + +/* This method checks to see if a file object exists at the path specified. If + * something does exist then the value of the 'exists' pointer is set to true. + * If the object is a directory then this value is also set to true. + */ + +status_t +StorageUtils::ExistsDirectory(BPath& directory, + bool* exists, + bool* isDirectory) +{ + struct stat s; + + *exists = false; + *isDirectory = false; + + if (-1 == stat(directory.Path(), &s)) { + if (ENOENT != errno) + return B_ERROR; + } else { + *exists = true; + *isDirectory = S_ISDIR(s.st_mode); + } + + return B_OK; +} diff --git a/src/apps/haikudepot/util/StorageUtils.h b/src/apps/haikudepot/util/StorageUtils.h new file mode 100644 index 0000000000..e17778350f --- /dev/null +++ b/src/apps/haikudepot/util/StorageUtils.h @@ -0,0 +1,21 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#ifndef PATH_UTILS_H +#define PATH_UTILS_H + +#include + +class StorageUtils { + +public: + static status_t RemoveDirectoryContents(BPath& path); + static status_t AppendToString(BPath& path, BString& result); + static status_t ExistsDirectory(BPath& directory, + bool* exists, + bool* isDirectory); +}; + +#endif // PATH_UTILS_H diff --git a/src/apps/haikudepot/util/ToFileUrlProtocolListener.cpp b/src/apps/haikudepot/util/ToFileUrlProtocolListener.cpp new file mode 100644 index 0000000000..6f7025da07 --- /dev/null +++ b/src/apps/haikudepot/util/ToFileUrlProtocolListener.cpp @@ -0,0 +1,100 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#include "ToFileUrlProtocolListener.h" + +#include + +#include + + +ToFileUrlProtocolListener::ToFileUrlProtocolListener(BPath path, + BString traceLoggingIdentifier, bool traceLogging) +{ + fDownloadIO = new BFile(path.Path(), O_WRONLY | O_CREAT); + fTraceLoggingIdentifier = traceLoggingIdentifier; + fTraceLogging = traceLogging; +} + + +ToFileUrlProtocolListener::~ToFileUrlProtocolListener() +{ + delete fDownloadIO; +} + + +void +ToFileUrlProtocolListener::ConnectionOpened(BUrlRequest* caller) +{ +} + + +void +ToFileUrlProtocolListener::HostnameResolved(BUrlRequest* caller, + const char* ip) +{ +} + + +void +ToFileUrlProtocolListener::ResponseStarted(BUrlRequest* caller) +{ +} + + +void +ToFileUrlProtocolListener::HeadersReceived(BUrlRequest* caller) +{ +} + + +void +ToFileUrlProtocolListener::DataReceived(BUrlRequest* caller, const char* data, + off_t position, ssize_t size) +{ + if (fDownloadIO != NULL && size > 0) { + size_t remaining = size; + size_t written = 0; + + do { + written = fDownloadIO->Write(&data[size - remaining], remaining); + remaining -= written; + } while (remaining > 0 && written > 0); + + if (remaining > 0) + fprintf(stdout, "unable to write all of the data to the file\n"); + } +} + +void +ToFileUrlProtocolListener::DownloadProgress(BUrlRequest* caller, + ssize_t bytesReceived, ssize_t bytesTotal) +{ +} + + +void +ToFileUrlProtocolListener::UploadProgress(BUrlRequest* caller, + ssize_t bytesSent, ssize_t bytesTotal) +{ +} + + +void +ToFileUrlProtocolListener::RequestCompleted(BUrlRequest* caller, bool success) +{ +} + + +void +ToFileUrlProtocolListener::DebugMessage(BUrlRequest* caller, + BUrlProtocolDebugMessage type, const char* text) +{ + if (fTraceLogging) { + fprintf(stdout, "url->file <%s>; %s\n", + fTraceLoggingIdentifier.String(), text); + } +} + diff --git a/src/apps/haikudepot/util/ToFileUrlProtocolListener.h b/src/apps/haikudepot/util/ToFileUrlProtocolListener.h new file mode 100644 index 0000000000..9145dff2a2 --- /dev/null +++ b/src/apps/haikudepot/util/ToFileUrlProtocolListener.h @@ -0,0 +1,40 @@ +/* + * Copyright 2017, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + +#include +#include + +class ToFileUrlProtocolListener : public BUrlProtocolListener { +public: + ToFileUrlProtocolListener(BPath path, + BString traceLoggingIdentifier, + bool traceLogging); + virtual ~ToFileUrlProtocolListener(); + + void ConnectionOpened(BUrlRequest* caller); + void HostnameResolved(BUrlRequest* caller, + const char* ip); + void ResponseStarted(BUrlRequest* caller); + void HeadersReceived(BUrlRequest* caller); + void DataReceived(BUrlRequest* caller, + const char* data, off_t position, + ssize_t size); + void DownloadProgress(BUrlRequest* caller, + ssize_t bytesReceived, ssize_t bytesTotal); + void UploadProgress(BUrlRequest* caller, + ssize_t bytesSent, ssize_t bytesTotal); + void RequestCompleted(BUrlRequest* caller, + bool success); + void DebugMessage(BUrlRequest* caller, + BUrlProtocolDebugMessage type, + const char* text); + +private: + bool fTraceLogging; + BString fTraceLoggingIdentifier; + BDataIO* fDownloadIO; + + +};