-Fingerprint checking now use a simple sum of the hash values, instead of the adler32 algorithm. This allows to match a catalog even if, due to the hashmap instanciation, it is not iterated in the same order. Updated all the existing catkeys files to match this new system.

-As the fingerprint is now fully working, all catkeys fingerprints are now checked, and if they don't match, the build will fail.
-This helped find an error in the Locale Preflet french catalog. It now works again.
-Usual set of cleanups and small style fixes.


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@33340 a95241bf-73f2-0310-859d-f6bbb57e9c96

src/add-ons/locale/catalogs/plaintext/Catalog.cpp

@@ -194,7 +194,7 @@ PlainTextCatalog::ReadFromFile(const char *path)
 
 	if (std::getline(catalogFile, currentItem, '\t').good()) {
 		// Get the language
-		fLanguageName << currentItem.c_str() ;
+		fLanguageName = currentItem.c_str() ;
 	} else {
 		log_team(LOG_DEBUG, "Unable to get language from %s", path);
 		return B_ERROR;
@@ -202,7 +202,7 @@ PlainTextCatalog::ReadFromFile(const char *path)
 
 	if (std::getline(catalogFile, currentItem, '\t').good()) {
 		// Get the signature
-		fSignature << currentItem.c_str() ;
+		fSignature = currentItem.c_str() ;
 	} else {
 		log_team(LOG_DEBUG, "Unable to get signature from %s", path);
 		return B_ERROR;
@@ -219,7 +219,10 @@ PlainTextCatalog::ReadFromFile(const char *path)
 			return B_ERROR;
 		}
 
-		if (fFingerprint!=0 && fFingerprint != foundFingerprint) {
+		if (fFingerprint == 0)
+			fFingerprint = foundFingerprint;
+
+		if (fFingerprint != foundFingerprint) {
 			return B_MISMATCHED_VALUES;
 		}
 	} else {
@@ -237,9 +240,7 @@ PlainTextCatalog::ReadFromFile(const char *path)
 	std::string translated;
 
 	while (std::getline(catalogFile, originalString,'\t').good()) {
-		// Each line is : "original string \t key \t translated string"
+		// Each line is : "original string \t context \t comment \t translation"
-		// However, the original string is also tab-separated because it
-		// also holds the context and comment
 
 		if (!std::getline(catalogFile, context,'\t').good()) {
 			log_team(LOG_DEBUG, "Unable to get context for string %s from %s",
@@ -272,6 +273,17 @@ PlainTextCatalog::ReadFromFile(const char *path)
 
 	catalogFile.close();
 
+	uint32 checkFP = ComputeFingerprint();
+	if (fFingerprint != checkFP) {
+		log_team(LOG_DEBUG, "plaintext-catalog(sig=%s, lang=%s) "
+			"has wrong fingerprint after load (%lX instead of %lX). "
+			"The catalog data may be corrupted, so this catalog is "
+			"skipped.\n",
+			fSignature.String(), fLanguageName.String(), checkFP,
+			fFingerprint);
+		return B_BAD_DATA;
+	}
+
 	// some information living in member variables needs to be copied
 	// to attributes. Although these attributes should have been written
 	// when creating the catalog, we make sure that they exist there:
@@ -296,7 +308,6 @@ PlainTextCatalog::WriteToFile(const char *path)
 	UpdateFingerprint();
 		// make sure we have the correct fingerprint before we flatten it
 
-
 	textContent << kCatArchiveVersion << "\t" << fLanguageName.String() << "\t"
 		<< fSignature.String() << "\t" << fFingerprint << "\n";
 
@@ -409,8 +420,8 @@ extern "C"
 BCatalogAddOn *create_catalog(const char *signature,
 	const char *language)
 {
-	PlainTextCatalog *catalog =
+	PlainTextCatalog *catalog
-		new(std::nothrow) PlainTextCatalog("emptycat", signature, language);
+		= new(std::nothrow) PlainTextCatalog("emptycat", signature, language);
 	return catalog;
 }
 

src/kits/locale/HashMapCatalog.cpp

@@ -221,15 +221,17 @@ BHashMapCatalog::SetString(const CatKey& key, const char *translated)
 
 
 /*
- * computes an adler32-checksum (we call it fingerprint) on all the
+ * computes a checksum (we call it fingerprint) on all the catalog-keys. We do
- * catalog-keys. We do not include the values, since we want catalogs for
+ * not include the values, since we want catalogs for different languages of the
- * different languages of the same app to have the same fingerprint, since we
+ * same app to have the same fingerprint, since we use it to separate different
- * use it to separate different catalog-versions.
+ * catalog-versions. We use a simple sum because there is no well known
+ * checksum algorithm that gives the same result if the string are sorted in the
+ * wrong order, and this does happen, as an hash map is an unsorted container.
  */
 uint32
 BHashMapCatalog::ComputeFingerprint() const
 {
-	uint32 adler = adler32(0, NULL, 0);
+	uint32 checksum = 0;
 
 	int32 hash;
 	CatMap::Iterator iter = fCatMap.GetIterator();
@@ -238,9 +240,9 @@ BHashMapCatalog::ComputeFingerprint() const
 	{
 		entry = iter.Next();
 		hash = B_HOST_TO_LENDIAN_INT32(entry.key.fHashVal);
-		adler = adler32(adler, reinterpret_cast<uint8*>(&hash), sizeof(int32));
+		checksum += hash;
 	}
-	return adler;
+	return checksum;
 }
 
 

src/kits/locale/Jamfile

@@ -5,8 +5,7 @@ UsePublicHeaders locale storage ;
 UseLibraryHeaders icu ;
 
 SharedLibrary liblocale.so
-	: adler32.c
+	: cat.cpp
-	  cat.cpp
 	  Catalog.cpp
 	  Collator.cpp
 	  Country.cpp

src/kits/locale/adler32.c

@@ -1,52 +0,0 @@
-/* adler32.c -- compute the Adler-32 checksum of a data stream
- * Copyright (C) 1995-2002 Mark Adler
- * For conditions of distribution and use, see copyright notice in zlib.h
- */
-
-/*
- * [zooey]:
- *		This file has been adjusted from the original found in zlib
- *		for better conformance to our style-guide.
- */
-
-#include <SupportDefs.h>
-
-uint32 adler32(uint32 adler, const uint8 *buf, uint32 len);
-	// prototype required by mwcc
-
-#define BASE 65521L /* largest prime smaller than 65536 */
-#define NMAX 5552
-/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */
-
-#define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
-#define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
-#define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
-#define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
-#define DO16(buf)   DO8(buf,0); DO8(buf,8);
-
-/* ========================================================================= */
-uint32 adler32(uint32 adler, const uint8 *buf, uint32 len)
-{
-	uint32 s1 = adler & 0xffff;
-	uint32 s2 = (adler >> 16) & 0xffff;
-	int k;
-
-	if (buf == NULL) return 1L;
-
-	while (len > 0) {
-		k = len < NMAX ? len : NMAX;
-		len -= k;
-		while (k >= 16) {
-			DO16(buf);
-			buf += 16;
-			k -= 16;
-		}
-		if (k != 0) do {
-			s1 += *buf++;
-			s2 += s1;
-		} while (--k);
-		s1 %= BASE;
-		s2 %= BASE;
-	}
-	return (s2 << 16) | s1;
-}

src/preferences/appearance/de.catkeys

@@ -1,4 +1,4 @@
-1	german	x-vnd.Haiku-Appearance	4000077587
+1	german	x-vnd.Haiku-Appearance	959674221
 Navigation Pulse	Colors tab		Navigations Puls
 Shadow	Colors tab		Schatten
 On	AntialiasingSettingsView		Ein

src/preferences/appearance/fr.catkeys

@@ -1,4 +1,4 @@
-1	french	x-vnd.Haiku-Appearance	4000077587
+1	french	x-vnd.Haiku-Appearance	959674221
 Navigation Pulse	Colors tab		Pulsation de navigation
 Shadow	Colors tab		Ombre
 On	AntialiasingSettingsView		Activé

src/preferences/bluetooth/fr.catkeys

@@ -1,4 +1,4 @@
-1	french	x-vnd.Haiku-BluetoothPrefs	2346738190
+1	french	x-vnd.Haiku-BluetoothPrefs	1173425682
 Handheld	Settings view		Appareil de poche
 Only from Trusted devices	Settings view		Seulement les appareils de confiance
 Refresh LocalDevicesxE2x80xA6	Window		Rafraîchir LocalDevicexE2x80xA6

src/preferences/cpufrequency/fr.catkeys

@@ -1,4 +1,4 @@
-1	french	x-vnd.Haiku-CPUFrequencyPref	2148935007
+1	french	x-vnd.Haiku-CPUFrequencyPref	1503797596
 Ok	Status view		Ok
 Integration Time [ms]	CPU Frequency View		Temps d'intégration	 [ms]
 High Performance	Status view		Haute performance

src/preferences/locale/fr.catkeys

@@ -1,4 +1,4 @@
-1	french	x-vnd.Haiku-Locale	1616861198
+1	french	x-vnd.Haiku-Locale	2641784784
 day of week (short name)	TimeFormatSettings		jour de la semaine (abrégé)
 Available languages	Locale Preflet Window		Langues disponibles
 Decimal separator	TimeFormatSettings		Séparateur décimal

src/tools/locale/HashMapCatalog.cpp

@@ -219,15 +219,17 @@ BHashMapCatalog::SetString(const CatKey& key, const char *translated)
 
 
 /*
- * computes an adler32-checksum (we call it fingerprint) on all the
+ * computes a checksum (we call it fingerprint) on all the catalog-keys. We do
- * catalog-keys. We do not include the values, since we want catalogs for
+ * not include the values, since we want catalogs for different languages of the
- * different languages of the same app to have the same fingerprint, since we
+ * same app to have the same fingerprint, since we use it to separate different
- * use it to separate different catalog-versions.
+ * catalog-versions. We use a simple sum because there is no well known
+ * checksum algorithm that gives the same result if the string are sorted in the
+ * wrong order, and this does happen, as an hash map is an unsorted container.
  */
 uint32
 BHashMapCatalog::ComputeFingerprint() const
 {
-	uint32 adler = adler32(0, NULL, 0);
+	uint32 checksum = 0;
 
 	int32 hash;
 	CatMap::Iterator iter = fCatMap.GetIterator();
@@ -236,9 +238,9 @@ BHashMapCatalog::ComputeFingerprint() const
 	{
 		entry = iter.Next();
 		hash = B_HOST_TO_LENDIAN_INT32(entry.key.fHashVal);
-		adler = adler32(adler, reinterpret_cast<uint8*>(&hash), sizeof(int32));
+		checksum += hash;
 	}
-	return adler;
+	return checksum;
 }
 
 

src/tools/locale/Jamfile

@@ -31,7 +31,6 @@ BuildPlatformMain <build>collectcatkeys :
 	HashMapCatalog.cpp
 	Catalog.cpp
 	RegExp.cpp
-	adler32.c
 	: $(HOST_LIBBE)  $(HOST_LIBSUPC++) $(HOST_LIBSTDC++) ;
 
 BuildPlatformMain <build>linkcatkeys :
@@ -40,5 +39,4 @@ BuildPlatformMain <build>linkcatkeys :
 	HashMapCatalog.cpp
 	DefaultCatalog.cpp
 	Catalog.cpp
-	adler32.c
 	: $(HOST_LIBBE) $(HOST_LIBSUPC++) $(HOST_LIBSTDC++) ;

src/tools/locale/PlainTextCatalog.cpp

@@ -44,9 +44,6 @@ using std::pair;
  */
 
 
-extern "C" uint32 adler32(uint32 adler, const uint8 *buf, uint32 len);
-	// definition lives in adler32.c
-
 const char *PlainTextCatalog::kCatMimeType
 	= "locale/x-vnd.Be.locale-catalog.plaintext";
 
@@ -235,13 +232,7 @@ PlainTextCatalog::ReadFromFile(const char *path)
 			"skipped.\n",
 			fSignature.String(), fLanguageName.String(), checkFP,
 			fFingerprint);
-		// TODO: This is what should be done if the fingerprint calculation
+		return B_BAD_DATA;
-		// actually worked. Unfortunately, adler32 will not give the same
-		// results if you swap strings, and an HashMap is not an ordered
-		// container so you can get a different result each time you iterate
-		// over it...
-
-		// return B_BAD_DATA;
 	}
 
 	// some information living in member variables needs to be copied

src/tools/locale/adler32.c

@@ -1,52 +0,0 @@
-/* adler32.c -- compute the Adler-32 checksum of a data stream
- * Copyright (C) 1995-2002 Mark Adler
- * For conditions of distribution and use, see copyright notice in zlib.h
- */
-
-/*
- * [zooey]:
- *		This file has been adjusted from the original found in zlib
- *		for better conformance to our style-guide.
- */
-
-#include <SupportDefs.h>
-
-uint32 adler32(uint32 adler, const uint8 *buf, uint32 len);
-	// prototype required by mwcc
-
-#define BASE 65521L /* largest prime smaller than 65536 */
-#define NMAX 5552
-/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */
-
-#define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
-#define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
-#define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
-#define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
-#define DO16(buf)   DO8(buf,0); DO8(buf,8);
-
-/* ========================================================================= */
-uint32 adler32(uint32 adler, const uint8 *buf, uint32 len)
-{
-	uint32 s1 = adler & 0xffff;
-	uint32 s2 = (adler >> 16) & 0xffff;
-	int k;
-
-	if (buf == NULL) return 1L;
-
-	while (len > 0) {
-		k = len < NMAX ? len : NMAX;
-		len -= k;
-		while (k >= 16) {
-			DO16(buf);
-			buf += 16;
-			k -= 16;
-		}
-		if (k != 0) do {
-			s1 += *buf++;
-			s2 += s1;
-		} while (--k);
-		s1 %= BASE;
-		s2 %= BASE;
-	}
-	return (s2 << 16) | s1;
-}