From e6b7213cc231a50bf0419b01fb46f3294c355b7d Mon Sep 17 00:00:00 2001 From: Adrien Destugues Date: Mon, 28 Sep 2009 20:08:53 +0000 Subject: [PATCH] -Fingerprint checking now use a simple sum of the hash values, instead of the adler32 algorithm. This allows to match a catalog even if, due to the hashmap instanciation, it is not iterated in the same order. Updated all the existing catkeys files to match this new system. -As the fingerprint is now fully working, all catkeys fingerprints are now checked, and if they don't match, the build will fail. -This helped find an error in the Locale Preflet french catalog. It now works again. -Usual set of cleanups and small style fixes. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@33340 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- .../locale/catalogs/plaintext/Catalog.cpp | 41 +++++++++------ src/kits/locale/HashMapCatalog.cpp | 16 +++--- src/kits/locale/Jamfile | 3 +- src/kits/locale/adler32.c | 52 ------------------- src/preferences/appearance/de.catkeys | 2 +- src/preferences/appearance/fr.catkeys | 2 +- src/preferences/bluetooth/fr.catkeys | 2 +- src/preferences/cpufrequency/fr.catkeys | 2 +- src/preferences/locale/fr.catkeys | 4 +- src/tools/locale/HashMapCatalog.cpp | 16 +++--- src/tools/locale/Jamfile | 2 - src/tools/locale/PlainTextCatalog.cpp | 11 +--- src/tools/locale/adler32.c | 52 ------------------- 13 files changed, 52 insertions(+), 153 deletions(-) delete mode 100644 src/kits/locale/adler32.c delete mode 100644 src/tools/locale/adler32.c diff --git a/src/add-ons/locale/catalogs/plaintext/Catalog.cpp b/src/add-ons/locale/catalogs/plaintext/Catalog.cpp index 30bc97703e..e64a4e441f 100644 --- a/src/add-ons/locale/catalogs/plaintext/Catalog.cpp +++ b/src/add-ons/locale/catalogs/plaintext/Catalog.cpp @@ -152,7 +152,7 @@ status_t PlainTextCatalog::ReadFromFile(const char *path) { std::fstream catalogFile; - std::string currentItem; + std::string currentItem; if (!path) path = fPath.String(); @@ -173,7 +173,7 @@ PlainTextCatalog::ReadFromFile(const char *path) std::istringstream ss(currentItem); ss >> arcver; if (ss.fail()) { - // can't convert to int + // can't convert to int log_team(LOG_DEBUG, "Unable to extract archive version ( string: %s ) from %s", currentItem.c_str(), path); @@ -181,7 +181,7 @@ PlainTextCatalog::ReadFromFile(const char *path) } if (arcver != kCatArchiveVersion) { - // wrong version + // wrong version log_team(LOG_DEBUG, "Wrong archive version ! Got %d instead of %d from %s", arcver, kCatArchiveVersion, path); @@ -194,7 +194,7 @@ PlainTextCatalog::ReadFromFile(const char *path) if (std::getline(catalogFile, currentItem, '\t').good()) { // Get the language - fLanguageName << currentItem.c_str() ; + fLanguageName = currentItem.c_str() ; } else { log_team(LOG_DEBUG, "Unable to get language from %s", path); return B_ERROR; @@ -202,7 +202,7 @@ PlainTextCatalog::ReadFromFile(const char *path) if (std::getline(catalogFile, currentItem, '\t').good()) { // Get the signature - fSignature << currentItem.c_str() ; + fSignature = currentItem.c_str() ; } else { log_team(LOG_DEBUG, "Unable to get signature from %s", path); return B_ERROR; @@ -219,7 +219,10 @@ PlainTextCatalog::ReadFromFile(const char *path) return B_ERROR; } - if (fFingerprint!=0 && fFingerprint != foundFingerprint) { + if (fFingerprint == 0) + fFingerprint = foundFingerprint; + + if (fFingerprint != foundFingerprint) { return B_MISMATCHED_VALUES; } } else { @@ -237,23 +240,21 @@ PlainTextCatalog::ReadFromFile(const char *path) std::string translated; while (std::getline(catalogFile, originalString,'\t').good()) { - // Each line is : "original string \t key \t translated string" - // However, the original string is also tab-separated because it - // also holds the context and comment + // Each line is : "original string \t context \t comment \t translation" - if (!std::getline(catalogFile,context,'\t').good()) { + if (!std::getline(catalogFile, context,'\t').good()) { log_team(LOG_DEBUG, "Unable to get context for string %s from %s", originalString.c_str(), path); return B_ERROR; } - if (!std::getline(catalogFile,comment,'\t').good()) { + if (!std::getline(catalogFile, comment,'\t').good()) { log_team(LOG_DEBUG, "Unable to get comment for string %s from %s", originalString.c_str(), path); return B_ERROR; } - if (!std::getline(catalogFile,translated).good()) { + if (!std::getline(catalogFile, translated).good()) { log_team(LOG_DEBUG, "Unable to get translated text for string %s from %s", originalString.c_str(), path); @@ -272,6 +273,17 @@ PlainTextCatalog::ReadFromFile(const char *path) catalogFile.close(); + uint32 checkFP = ComputeFingerprint(); + if (fFingerprint != checkFP) { + log_team(LOG_DEBUG, "plaintext-catalog(sig=%s, lang=%s) " + "has wrong fingerprint after load (%lX instead of %lX). " + "The catalog data may be corrupted, so this catalog is " + "skipped.\n", + fSignature.String(), fLanguageName.String(), checkFP, + fFingerprint); + return B_BAD_DATA; + } + // some information living in member variables needs to be copied // to attributes. Although these attributes should have been written // when creating the catalog, we make sure that they exist there: @@ -296,7 +308,6 @@ PlainTextCatalog::WriteToFile(const char *path) UpdateFingerprint(); // make sure we have the correct fingerprint before we flatten it - textContent << kCatArchiveVersion << "\t" << fLanguageName.String() << "\t" << fSignature.String() << "\t" << fFingerprint << "\n"; @@ -409,8 +420,8 @@ extern "C" BCatalogAddOn *create_catalog(const char *signature, const char *language) { - PlainTextCatalog *catalog = - new(std::nothrow) PlainTextCatalog("emptycat", signature, language); + PlainTextCatalog *catalog + = new(std::nothrow) PlainTextCatalog("emptycat", signature, language); return catalog; } diff --git a/src/kits/locale/HashMapCatalog.cpp b/src/kits/locale/HashMapCatalog.cpp index ff82959546..4b52c6ed40 100644 --- a/src/kits/locale/HashMapCatalog.cpp +++ b/src/kits/locale/HashMapCatalog.cpp @@ -221,15 +221,17 @@ BHashMapCatalog::SetString(const CatKey& key, const char *translated) /* - * computes an adler32-checksum (we call it fingerprint) on all the - * catalog-keys. We do not include the values, since we want catalogs for - * different languages of the same app to have the same fingerprint, since we - * use it to separate different catalog-versions. + * computes a checksum (we call it fingerprint) on all the catalog-keys. We do + * not include the values, since we want catalogs for different languages of the + * same app to have the same fingerprint, since we use it to separate different + * catalog-versions. We use a simple sum because there is no well known + * checksum algorithm that gives the same result if the string are sorted in the + * wrong order, and this does happen, as an hash map is an unsorted container. */ uint32 BHashMapCatalog::ComputeFingerprint() const { - uint32 adler = adler32(0, NULL, 0); + uint32 checksum = 0; int32 hash; CatMap::Iterator iter = fCatMap.GetIterator(); @@ -238,9 +240,9 @@ BHashMapCatalog::ComputeFingerprint() const { entry = iter.Next(); hash = B_HOST_TO_LENDIAN_INT32(entry.key.fHashVal); - adler = adler32(adler, reinterpret_cast(&hash), sizeof(int32)); + checksum += hash; } - return adler; + return checksum; } diff --git a/src/kits/locale/Jamfile b/src/kits/locale/Jamfile index 4c32a417ce..6ebc7fdf53 100644 --- a/src/kits/locale/Jamfile +++ b/src/kits/locale/Jamfile @@ -5,8 +5,7 @@ UsePublicHeaders locale storage ; UseLibraryHeaders icu ; SharedLibrary liblocale.so - : adler32.c - cat.cpp + : cat.cpp Catalog.cpp Collator.cpp Country.cpp diff --git a/src/kits/locale/adler32.c b/src/kits/locale/adler32.c deleted file mode 100644 index 524746a713..0000000000 --- a/src/kits/locale/adler32.c +++ /dev/null @@ -1,52 +0,0 @@ -/* adler32.c -- compute the Adler-32 checksum of a data stream - * Copyright (C) 1995-2002 Mark Adler - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -/* - * [zooey]: - * This file has been adjusted from the original found in zlib - * for better conformance to our style-guide. - */ - -#include - -uint32 adler32(uint32 adler, const uint8 *buf, uint32 len); - // prototype required by mwcc - -#define BASE 65521L /* largest prime smaller than 65536 */ -#define NMAX 5552 -/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ - -#define DO1(buf,i) {s1 += buf[i]; s2 += s1;} -#define DO2(buf,i) DO1(buf,i); DO1(buf,i+1); -#define DO4(buf,i) DO2(buf,i); DO2(buf,i+2); -#define DO8(buf,i) DO4(buf,i); DO4(buf,i+4); -#define DO16(buf) DO8(buf,0); DO8(buf,8); - -/* ========================================================================= */ -uint32 adler32(uint32 adler, const uint8 *buf, uint32 len) -{ - uint32 s1 = adler & 0xffff; - uint32 s2 = (adler >> 16) & 0xffff; - int k; - - if (buf == NULL) return 1L; - - while (len > 0) { - k = len < NMAX ? len : NMAX; - len -= k; - while (k >= 16) { - DO16(buf); - buf += 16; - k -= 16; - } - if (k != 0) do { - s1 += *buf++; - s2 += s1; - } while (--k); - s1 %= BASE; - s2 %= BASE; - } - return (s2 << 16) | s1; -} diff --git a/src/preferences/appearance/de.catkeys b/src/preferences/appearance/de.catkeys index 176305e422..eac3682ff9 100644 --- a/src/preferences/appearance/de.catkeys +++ b/src/preferences/appearance/de.catkeys @@ -1,4 +1,4 @@ -1 german x-vnd.Haiku-Appearance 4000077587 +1 german x-vnd.Haiku-Appearance 959674221 Navigation Pulse Colors tab Navigations Puls Shadow Colors tab Schatten On AntialiasingSettingsView Ein diff --git a/src/preferences/appearance/fr.catkeys b/src/preferences/appearance/fr.catkeys index 6ba455433f..f4eb8a283a 100644 --- a/src/preferences/appearance/fr.catkeys +++ b/src/preferences/appearance/fr.catkeys @@ -1,4 +1,4 @@ -1 french x-vnd.Haiku-Appearance 4000077587 +1 french x-vnd.Haiku-Appearance 959674221 Navigation Pulse Colors tab Pulsation de navigation Shadow Colors tab Ombre On AntialiasingSettingsView Activé diff --git a/src/preferences/bluetooth/fr.catkeys b/src/preferences/bluetooth/fr.catkeys index 964bb0f4d2..35222b5091 100644 --- a/src/preferences/bluetooth/fr.catkeys +++ b/src/preferences/bluetooth/fr.catkeys @@ -1,4 +1,4 @@ -1 french x-vnd.Haiku-BluetoothPrefs 2346738190 +1 french x-vnd.Haiku-BluetoothPrefs 1173425682 Handheld Settings view Appareil de poche Only from Trusted devices Settings view Seulement les appareils de confiance Refresh LocalDevicesxE2x80xA6 Window Rafraîchir LocalDevicexE2x80xA6 diff --git a/src/preferences/cpufrequency/fr.catkeys b/src/preferences/cpufrequency/fr.catkeys index 57b2c82b7c..453d9073bd 100644 --- a/src/preferences/cpufrequency/fr.catkeys +++ b/src/preferences/cpufrequency/fr.catkeys @@ -1,4 +1,4 @@ -1 french x-vnd.Haiku-CPUFrequencyPref 2148935007 +1 french x-vnd.Haiku-CPUFrequencyPref 1503797596 Ok Status view Ok Integration Time [ms] CPU Frequency View Temps d'intégration [ms] High Performance Status view Haute performance diff --git a/src/preferences/locale/fr.catkeys b/src/preferences/locale/fr.catkeys index 6668e23b34..37d3b0abef 100644 --- a/src/preferences/locale/fr.catkeys +++ b/src/preferences/locale/fr.catkeys @@ -1,4 +1,4 @@ -1 french x-vnd.Haiku-Locale 1616861198 +1 french x-vnd.Haiku-Locale 2641784784 day of week (short name) TimeFormatSettings jour de la semaine (abrégé) Available languages Locale Preflet Window Langues disponibles Decimal separator TimeFormatSettings Séparateur décimal @@ -19,7 +19,7 @@ Preferred languages Locale Preflet Window Langues préférées Symbol position TimeFormatSettings Position du symbole month number TimeFormatSettings numéro du mois month number (2 digits) TimeFormatSettings numéro du mois (2 chiffres) -Unable to find the available languages!You can't use this preflet! Locale Preflet Window La liste des langages disponibles est introuvable, ce preflet est donc inutilisable ! +Unable to find the available languages! You can't use thispreflet! Locale Preflet Window La liste des langages disponibles est introuvable, ce preflet est donc inutilisable ! Numbers TimeFormatSettings Nombres (unknown format) TimeFormatSettings (format inconnu) Negative marker: TimeFormatSettings Marqueur de nombre négatifs: diff --git a/src/tools/locale/HashMapCatalog.cpp b/src/tools/locale/HashMapCatalog.cpp index 6f82f4cdd3..bf7da83864 100644 --- a/src/tools/locale/HashMapCatalog.cpp +++ b/src/tools/locale/HashMapCatalog.cpp @@ -219,15 +219,17 @@ BHashMapCatalog::SetString(const CatKey& key, const char *translated) /* - * computes an adler32-checksum (we call it fingerprint) on all the - * catalog-keys. We do not include the values, since we want catalogs for - * different languages of the same app to have the same fingerprint, since we - * use it to separate different catalog-versions. + * computes a checksum (we call it fingerprint) on all the catalog-keys. We do + * not include the values, since we want catalogs for different languages of the + * same app to have the same fingerprint, since we use it to separate different + * catalog-versions. We use a simple sum because there is no well known + * checksum algorithm that gives the same result if the string are sorted in the + * wrong order, and this does happen, as an hash map is an unsorted container. */ uint32 BHashMapCatalog::ComputeFingerprint() const { - uint32 adler = adler32(0, NULL, 0); + uint32 checksum = 0; int32 hash; CatMap::Iterator iter = fCatMap.GetIterator(); @@ -236,9 +238,9 @@ BHashMapCatalog::ComputeFingerprint() const { entry = iter.Next(); hash = B_HOST_TO_LENDIAN_INT32(entry.key.fHashVal); - adler = adler32(adler, reinterpret_cast(&hash), sizeof(int32)); + checksum += hash; } - return adler; + return checksum; } diff --git a/src/tools/locale/Jamfile b/src/tools/locale/Jamfile index 7b04305060..d3b6c61f07 100644 --- a/src/tools/locale/Jamfile +++ b/src/tools/locale/Jamfile @@ -31,7 +31,6 @@ BuildPlatformMain collectcatkeys : HashMapCatalog.cpp Catalog.cpp RegExp.cpp - adler32.c : $(HOST_LIBBE) $(HOST_LIBSUPC++) $(HOST_LIBSTDC++) ; BuildPlatformMain linkcatkeys : @@ -40,5 +39,4 @@ BuildPlatformMain linkcatkeys : HashMapCatalog.cpp DefaultCatalog.cpp Catalog.cpp - adler32.c : $(HOST_LIBBE) $(HOST_LIBSUPC++) $(HOST_LIBSTDC++) ; diff --git a/src/tools/locale/PlainTextCatalog.cpp b/src/tools/locale/PlainTextCatalog.cpp index 92dbfb10c3..aae075ae78 100644 --- a/src/tools/locale/PlainTextCatalog.cpp +++ b/src/tools/locale/PlainTextCatalog.cpp @@ -44,9 +44,6 @@ using std::pair; */ -extern "C" uint32 adler32(uint32 adler, const uint8 *buf, uint32 len); - // definition lives in adler32.c - const char *PlainTextCatalog::kCatMimeType = "locale/x-vnd.Be.locale-catalog.plaintext"; @@ -235,13 +232,7 @@ PlainTextCatalog::ReadFromFile(const char *path) "skipped.\n", fSignature.String(), fLanguageName.String(), checkFP, fFingerprint); - // TODO: This is what should be done if the fingerprint calculation - // actually worked. Unfortunately, adler32 will not give the same - // results if you swap strings, and an HashMap is not an ordered - // container so you can get a different result each time you iterate - // over it... - - // return B_BAD_DATA; + return B_BAD_DATA; } // some information living in member variables needs to be copied diff --git a/src/tools/locale/adler32.c b/src/tools/locale/adler32.c deleted file mode 100644 index 524746a713..0000000000 --- a/src/tools/locale/adler32.c +++ /dev/null @@ -1,52 +0,0 @@ -/* adler32.c -- compute the Adler-32 checksum of a data stream - * Copyright (C) 1995-2002 Mark Adler - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -/* - * [zooey]: - * This file has been adjusted from the original found in zlib - * for better conformance to our style-guide. - */ - -#include - -uint32 adler32(uint32 adler, const uint8 *buf, uint32 len); - // prototype required by mwcc - -#define BASE 65521L /* largest prime smaller than 65536 */ -#define NMAX 5552 -/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ - -#define DO1(buf,i) {s1 += buf[i]; s2 += s1;} -#define DO2(buf,i) DO1(buf,i); DO1(buf,i+1); -#define DO4(buf,i) DO2(buf,i); DO2(buf,i+2); -#define DO8(buf,i) DO4(buf,i); DO4(buf,i+4); -#define DO16(buf) DO8(buf,0); DO8(buf,8); - -/* ========================================================================= */ -uint32 adler32(uint32 adler, const uint8 *buf, uint32 len) -{ - uint32 s1 = adler & 0xffff; - uint32 s2 = (adler >> 16) & 0xffff; - int k; - - if (buf == NULL) return 1L; - - while (len > 0) { - k = len < NMAX ? len : NMAX; - len -= k; - while (k >= 16) { - DO16(buf); - buf += 16; - k -= 16; - } - if (k != 0) do { - s1 += *buf++; - s2 += s1; - } while (--k); - s1 %= BASE; - s2 %= BASE; - } - return (s2 << 16) | s1; -}