HaikuDepot: Use web app bulk transfers.

Collect up to 50 packages and get information about them in a bulk transfer.
If that fails, do it in two smaller transfers, until only one package is
left, fall back to use the getPkg method than.

Remember packages for which there is an icon on the server. After the
first round of reading icons from the cache, only try to fetch icons from
the server for which we know one should be there.

Added quite a bit of command line output to see what is going on.

src/apps/haikudepot/Model.cpp

@@ -17,6 +17,7 @@
 #include <Entry.h>
 #include <FindDirectory.h>
 #include <File.h>
+#include <Message.h>
 #include <Path.h>
 
 #include "WebAppInterface.h"
@@ -598,6 +599,8 @@ Model::_PopulateAllPackagesThread(bool fromCacheOnly)
 {
 	int32 depotIndex = 0;
 	int32 packageIndex = 0;
+	PackageList bulkPackageList;
+	PackageList packagesWithIconsList;
 
 	while (!fStopPopulatingAllPackages) {
 		// Obtain PackageInfoRef while keeping the depot and package lists
@@ -625,11 +628,120 @@ Model::_PopulateAllPackagesThread(bool fromCacheOnly)
 		if (package.Get() == NULL)
 			continue;
 	
-		_PopulatePackageInfo(package, fromCacheOnly);
+		//_PopulatePackageInfo(package, fromCacheOnly);
-		_PopulatePackageIcon(package, fromCacheOnly);
+		bulkPackageList.Add(package);
+		if (bulkPackageList.CountItems() == 50) {
+			_PopulatePackageInfos(bulkPackageList, fromCacheOnly,
+				packagesWithIconsList);
+			bulkPackageList.Clear();
+		}
+		if (fromCacheOnly)
+			_PopulatePackageIcon(package, fromCacheOnly);
 		// TODO: Average user rating. It needs to be shown in the
 		// list view, so without the user clicking the package.
 	}
+
+	if (!fStopPopulatingAllPackages && bulkPackageList.CountItems() > 0) {
+		_PopulatePackageInfos(bulkPackageList, fromCacheOnly,
+			packagesWithIconsList);
+	}
+
+	if (!fromCacheOnly) {
+		for (int i = packagesWithIconsList.CountItems() - 1; i >= 0; i--) {
+			if (fStopPopulatingAllPackages)
+				break;
+			const PackageInfoRef& package = packagesWithIconsList.ItemAtFast(i);
+			printf("Getting/Updating native icon for %s\n",
+				package->Title().String());
+			_PopulatePackageIcon(package, fromCacheOnly);
+		}
+	}
+}
+
+
+void
+Model::_PopulatePackageInfos(PackageList& packages, bool fromCacheOnly,
+	PackageList& packagesWithIcons)
+{
+	if (fromCacheOnly)
+		return;
+	
+	// Retrieve info from web-app
+	WebAppInterface interface;
+	BMessage info;
+
+	StringList packageNames;
+	for (int i = 0; i < packages.CountItems(); i++) {
+		const PackageInfoRef& package = packages.ItemAtFast(i);
+		packageNames.Add(package->Title());
+	}
+
+	status_t status = interface.RetrieveBulkPackageInfo(packageNames, info);
+	if (status == B_OK) {
+		// Parse message
+//		info.PrintToStream();
+		BMessage result;
+		BMessage pkgs;
+		if (info.FindMessage("result", &result) == B_OK
+			&& result.FindMessage("pkgs", &pkgs) == B_OK) {
+			int32 index = 0;
+			while (true) {
+				BString name;
+				name << index++;
+				BMessage pkgInfo;
+				if (pkgs.FindMessage(name, &pkgInfo) != B_OK)
+					break;
+			
+				BString pkgName;
+				if (pkgInfo.FindString("name", &pkgName) != B_OK)
+					continue;
+			
+				// Find the PackageInfoRef
+				bool found = false;
+				for (int i = 0; i < packages.CountItems(); i++) {
+					const PackageInfoRef& package = packages.ItemAtFast(i);
+					if (pkgName == package->Title()) {
+						_PopulatePackageInfo(package, pkgInfo);
+						if (_HasNativeIcon(pkgInfo))
+							packagesWithIcons.Add(package);
+						packages.Remove(i);
+						found = true;
+						break;
+					}
+				}
+				if (!found)
+					printf("No matching package for %s\n", pkgName.String());
+			}
+		}
+	} else {
+		printf("Error sending request: %s\n", strerror(status));
+		int count = packages.CountItems();
+		if (count >= 4) {
+			// Retry in smaller chunks
+			PackageList firstHalf;
+			PackageList secondHalf;
+			for (int i = 0; i < count / 2; i++)
+				firstHalf.Add(packages.ItemAtFast(i));
+			for (int i = count / 2; i < count; i++)
+				secondHalf.Add(packages.ItemAtFast(i));
+			packages.Clear();
+			_PopulatePackageInfos(firstHalf, fromCacheOnly, packagesWithIcons);
+			_PopulatePackageInfos(secondHalf, fromCacheOnly, packagesWithIcons);
+		} else {
+			while (packages.CountItems() > 0) {
+				const PackageInfoRef& package = packages.ItemAtFast(0);
+				_PopulatePackageInfo(package, fromCacheOnly);
+				packages.Remove(0);
+			}
+		}
+	}
+
+	if (packages.CountItems() > 0) {
+		for (int i = 0; i < packages.CountItems(); i++) {
+			const PackageInfoRef& package = packages.ItemAtFast(i);
+			printf("No package info for %s\n", package->Title().String());
+		}
+	}
 }
 
 
@@ -646,61 +758,80 @@ Model::_PopulatePackageInfo(const PackageInfoRef& package, bool fromCacheOnly)
 	status_t status = interface.RetrievePackageInfo(package->Title(), info);
 	if (status == B_OK) {
 		// Parse message
-		info.PrintToStream();
+//		info.PrintToStream();
 		BMessage result;
-		if (info.FindMessage("result", &result) == B_OK) {
+		if (info.FindMessage("result", &result) == B_OK)
-			BMessage categories;
+			_PopulatePackageInfo(package, result);
-			if (result.FindMessage("pkgCategoryCodes", &categories) == B_OK) {
+	}
-				int32 index = 0;
+}
-				while (true) {
-					BString name;
-					name << index++;
-					BString category;
-					if (categories.FindString(name, &category) != B_OK)
-						break;
 
-					if (category == "AUDIO")
+
-						package->AddCategory(CategoryAudio());
+void
-					else if (category == "BUSINESS")
+Model::_PopulatePackageInfo(const PackageInfoRef& package,
-						package->AddCategory(CategoryBusiness());
+	const BMessage& data)
-					else if (category == "DEVELOPMENT")
+{
-						package->AddCategory(CategoryDevelopment());
+	const char* categoriesDebug = "";
-					else if (category == "EDUCATION")
+	const char* ratingDebug = "";
-						package->AddCategory(CategoryEducation());
+
-					else if (category == "GAMES")
+	BMessage categories;
-						package->AddCategory(CategoryGames());
+	if (data.FindMessage("pkgCategoryCodes", &categories) == B_OK) {
-					else if (category == "GRAPHICS")
+		int32 index = 0;
-						package->AddCategory(CategoryGraphics());
+		while (true) {
-					else if (category == "INTERNETANDNETWORK")
+			BString name;
-						package->AddCategory(CategoryInternetAndNetwork());
+			name << index++;
-					else if (category == "PRODUCTIVITY")
+			BString category;
-						package->AddCategory(CategoryProductivity());
+			if (categories.FindString(name, &category) != B_OK)
-					else if (category == "SCIENCEANDMATHEMATICS")
+				break;
-						package->AddCategory(CategoryScienceAndMathematics());
+
-					else if (category == "SYSTEMANDUTILITIES")
+			if (category == "AUDIO")
-						package->AddCategory(CategorySystemAndUtilities());
+				package->AddCategory(CategoryAudio());
-					else if (category == "VIDEO")
+			else if (category == "BUSINESS")
-						package->AddCategory(CategoryVideo());
+				package->AddCategory(CategoryBusiness());
-					// TODO: The server will eventually support an API to
+			else if (category == "DEVELOPMENT")
-					// get the defined categories and their translated names.
+				package->AddCategory(CategoryDevelopment());
-					// This should then be used instead of hard-coded
+			else if (category == "EDUCATION")
-					// categories and translations in the app.
+				package->AddCategory(CategoryEducation());
-				}
+			else if (category == "GAMES")
-			}
+				package->AddCategory(CategoryGames());
-			double derivedRating;
+			else if (category == "GRAPHICS")
-			double derivedRatingSampleSize;
+				package->AddCategory(CategoryGraphics());
-			if (result.FindDouble("derivedRating", &derivedRating) == B_OK
+			else if (category == "INTERNETANDNETWORK")
-				&& result.FindDouble("derivedRatingSampleSize",
+				package->AddCategory(CategoryInternetAndNetwork());
-					&derivedRatingSampleSize) == B_OK) {
+			else if (category == "PRODUCTIVITY")
-				if (derivedRatingSampleSize > 0) {
+				package->AddCategory(CategoryProductivity());
-					RatingSummary summary;
+			else if (category == "SCIENCEANDMATHEMATICS")
-					summary.averageRating = derivedRating;
+				package->AddCategory(CategoryScienceAndMathematics());
-					summary.ratingCount = (int)derivedRatingSampleSize;
+			else if (category == "SYSTEMANDUTILITIES")
-					package->SetRatingSummary(summary);
+				package->AddCategory(CategorySystemAndUtilities());
-				}
+			else if (category == "VIDEO")
-			}
+				package->AddCategory(CategoryVideo());
+			// TODO: The server will eventually support an API to
+			// get the defined categories and their translated names.
+			// This should then be used instead of hard-coded
+			// categories and translations in the app.
+		
+			categoriesDebug = "categories";
 		}
 	}
+	double derivedRating;
+	double derivedRatingSampleSize;
+	if (data.FindDouble("derivedRating", &derivedRating) == B_OK
+		&& data.FindDouble("derivedRatingSampleSize",
+			&derivedRatingSampleSize) == B_OK) {
+		if (derivedRatingSampleSize > 0) {
+			RatingSummary summary;
+			summary.averageRating = derivedRating;
+			summary.ratingCount = (int)derivedRatingSampleSize;
+			package->SetRatingSummary(summary);
+
+			if (strlen(categoriesDebug) > 0)
+				ratingDebug = ", rating";
+			else
+				ratingDebug = "rating";
+		}
+	}
+	printf("Populated package info for %s: %s%s\n",
+		package->Title().String(), categoriesDebug, ratingDebug);
 }
 
 
@@ -753,3 +884,32 @@ Model::_PopulatePackageIcon(const PackageInfoRef& package, bool fromCacheOnly)
 		}
 	}
 }
+
+
+bool
+Model::_HasNativeIcon(const BMessage& message) const
+{
+	BMessage pkgIcons;
+	if (message.FindMessage("pkgIcons", &pkgIcons) != B_OK)
+		return false;
+
+	if (!pkgIcons.IsEmpty())
+		pkgIcons.PrintToStream();
+
+	int32 index = 0;
+	while (true) {
+		BString name;
+		name << index++;
+		
+		BMessage typeCodeInfo;
+		if (pkgIcons.FindMessage(name, &typeCodeInfo) != B_OK)
+			break;
+	
+		BString mediaTypeCode;
+		if (typeCodeInfo.FindString("mediaTypeCode", &mediaTypeCode) == B_OK
+			&& mediaTypeCode == "application/x-vnd.haiku-icon") {
+			return true;
+		}
+	}
+	return false;
+}

src/apps/haikudepot/Model.h

@@ -9,6 +9,8 @@
 
 #include "PackageInfo.h"
 
+class BMessage;
+
 
 class PackageFilter : public BReferenceable {
 public:
@@ -101,12 +103,20 @@ private:
 	static	int32				_PopulateAllPackagesEntry(void* cookie);
 			void				_PopulateAllPackagesThread(bool fromCacheOnly);
 
+			void				_PopulatePackageInfos(
+									PackageList& packages,
+									bool fromCacheOnly,
+									PackageList& packagesWithIcons);
 			void				_PopulatePackageInfo(
 									const PackageInfoRef& package,
 									bool fromCacheOnly);
+			void				_PopulatePackageInfo(
+									const PackageInfoRef& package,
+									const BMessage& data);
 			void				_PopulatePackageIcon(
 									const PackageInfoRef& package,
 									bool fromCacheOnly);
+			bool				_HasNativeIcon(const BMessage& message) const;
 
 private:
 			BLocker				fLock;