From 110649a1a4d72f1ce49e32bbb1474df965f35bd6 Mon Sep 17 00:00:00 2001 From: Adrien Destugues Date: Wed, 28 Jul 2010 11:29:14 +0000 Subject: [PATCH] * Collator API now actually uses ICU. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@37783 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/os/locale/Collator.h | 78 +++---- src/kits/locale/Collator.cpp | 419 ++++++----------------------------- 2 files changed, 91 insertions(+), 406 deletions(-) diff --git a/headers/os/locale/Collator.h b/headers/os/locale/Collator.h index d969acdfd7..991a08b49f 100644 --- a/headers/os/locale/Collator.h +++ b/headers/os/locale/Collator.h @@ -1,3 +1,9 @@ +/* + * Copyright 2003-2010, Haiku, Inc. + * Distributed under the terms of the MIT Licence. +*/ + + #ifndef _COLLATOR_H_ #define _COLLATOR_H_ @@ -5,6 +11,12 @@ #include #include + +namespace icu_44 { + class Collator; +}; + + class BString; class BCollatorAddOn; @@ -13,7 +25,8 @@ enum collator_strengths { B_COLLATE_DEFAULT = -1, B_COLLATE_PRIMARY = 1, // e.g.: no diacritical differences, e = é - B_COLLATE_SECONDARY, // diacritics are different from their base characters, a != ä + B_COLLATE_SECONDARY, // diacritics are different from their base + // characters, a != ä B_COLLATE_TERTIARY, // case sensitive comparison B_COLLATE_QUATERNARY, @@ -24,7 +37,7 @@ enum collator_strengths { class BCollator : public BArchivable { public: BCollator(); - BCollator(BCollatorAddOn *collator, int8 strength, bool ignorePunctuation); + BCollator(const char *locale, int8 strength, bool ignorePunctuation); BCollator(BMessage *archive); ~BCollator(); @@ -34,20 +47,24 @@ class BCollator : public BArchivable { void SetIgnorePunctuation(bool ignore); bool IgnorePunctuation() const; - status_t GetSortKey(const char *string, BString *key, int8 strength = B_COLLATE_DEFAULT); + status_t GetSortKey(const char *string, BString *key, + int8 strength = B_COLLATE_DEFAULT); - int Compare(const char *, const char *, int32 len = -1, int8 strength = B_COLLATE_DEFAULT); - bool Equal(const char *, const char *, int32 len = -1, int8 strength = B_COLLATE_DEFAULT); - bool Greater(const char *, const char *, int32 len = -1, int8 strength = B_COLLATE_DEFAULT); - bool GreaterOrEqual(const char *, const char *, int32 len = -1, int8 strength = B_COLLATE_DEFAULT); + int Compare(const char *, const char *, int32 len = -1, + int8 strength = B_COLLATE_DEFAULT); + bool Equal(const char *, const char *, int32 len = -1, + int8 strength = B_COLLATE_DEFAULT); + bool Greater(const char *, const char *, int32 len = -1, + int8 strength = B_COLLATE_DEFAULT); + bool GreaterOrEqual(const char *, const char *, int32 len = -1, + int8 strength = B_COLLATE_DEFAULT); // (un-)archiving API status_t Archive(BMessage *archive, bool deep) const; static BArchivable *Instantiate(BMessage *archive); private: - BCollatorAddOn *fCollator; - image_id fCollatorImage; + icu_44::Collator* fICUCollator; int8 fStrength; bool fIgnorePunctuation; }; @@ -68,50 +85,11 @@ BCollator::Greater(const char *s1, const char *s2, int32 len, int8 strength) inline bool -BCollator::GreaterOrEqual(const char *s1, const char *s2, int32 len, int8 strength) +BCollator::GreaterOrEqual(const char *s1, const char *s2, int32 len, + int8 strength) { return Compare(s1, s2, len, strength) >= 0; } -/************************************************************************/ - -// For BCollator add-on implementations: - -class BCollatorAddOn : public BArchivable { - public: - BCollatorAddOn(); - BCollatorAddOn(BMessage *archive); - virtual ~BCollatorAddOn(); - - virtual status_t GetSortKey(const char *string, BString *key, int8 strength, - bool ignorePunctuation); - virtual int Compare(const char *a, const char *b, int32 length, int8 strength, - bool ignorePunctuation); - - // (un-)archiving API - virtual status_t Archive(BMessage *archive, bool deep) const; - static BArchivable *Instantiate(BMessage *archive); - - protected: - struct input_context { - input_context(bool ignorePunctuation); - - bool ignore_punctuation; - uint32 next_char; - int32 reserved1; - int32 reserved2; - }; - - virtual uint32 GetNextChar(const char **string, input_context &context); - virtual size_t PrimaryKeyLength(size_t length); - virtual char *PutPrimaryKey(const char *string, char *buffer, int32 length, - bool ignorePunctuation); -}; - -// If your add-on should work with the standard tool to create a language, it -// should export that function. However, once the language file has been written -// only the archived collator is used, and that function won't be called anymore. -extern "C" BCollatorAddOn *instantiate_collator(void); - #endif /* _COLLATOR_H_ */ diff --git a/src/kits/locale/Collator.cpp b/src/kits/locale/Collator.cpp index e8d1299ee1..569710fdfa 100644 --- a/src/kits/locale/Collator.cpp +++ b/src/kits/locale/Collator.cpp @@ -1,6 +1,7 @@ /* -** Copyright 2003, Axel Dörfler, axeld@pinc-software.de. All rights reserved. -** Distributed under the terms of the OpenBeOS License. +* Copyright 2003, Axel Dörfler, axeld@pinc-software.de. All rights reserved. +* Copyright 2010, Adrien Destugues +* Distributed under the terms of the MIT License. */ @@ -9,90 +10,41 @@ #include #include +#include #include #include - -// conversion array for character ranges 192 - 223 & 224 - 255 -static const uint8 kNoDiacrits[] = { - 'a','a','a','a','a','a','a', - 'c', - 'e','e','e','e', - 'i','i','i','i', - 240, // eth - 'n', - 'o','o','o','o','o', - 247, // - 'o', - 'u','u','u','u', - 'y', - 254, // thorn - 'y' -}; - - -static inline uint32 -getPrimaryChar(uint32 c) -{ - if (c < 0x80) - return tolower(c); - - // this automatically returns lowercase letters - if (c >= 192 && c < 223) - return kNoDiacrits[c - 192]; - if (c == 223) // ß - return 's'; - if (c >= 224 && c < 256) - return kNoDiacrits[c - 224]; - - return BUnicodeChar::ToLower(c); -} - - -BCollatorAddOn::input_context::input_context(bool ignorePunctuation) - : - ignore_punctuation(ignorePunctuation), - next_char(0), - reserved1(0), - reserved2(0) -{ -} - - -// #pragma mark - +#include BCollator::BCollator() : - fCollatorImage(B_ERROR), fStrength(B_COLLATE_PRIMARY), fIgnorePunctuation(true) { - // ToDo: the collator construction will have to change; the default + // TODO: the collator construction will have to change; the default // collator should be constructed by the Locale/LocaleRoster, so we // only need a constructor where you specify all details - fCollator = new BCollatorAddOn(); + UErrorCode error = U_ZERO_ERROR; + fICUCollator = Collator::createInstance(error); } -BCollator::BCollator(BCollatorAddOn *collator, int8 strength, +BCollator::BCollator(const char *locale, int8 strength, bool ignorePunctuation) : - fCollator(collator), - fCollatorImage(B_ERROR), fStrength(strength), fIgnorePunctuation(ignorePunctuation) { - if (collator == NULL) - fCollator = new BCollatorAddOn(); + UErrorCode error = U_ZERO_ERROR; + fICUCollator = Collator::createInstance(locale, error); } BCollator::BCollator(BMessage *archive) : BArchivable(archive), - fCollator(NULL), - fCollatorImage(B_ERROR), + fICUCollator(NULL), fIgnorePunctuation(true) { #if HAIKU_TARGET_PLATFORM_HAIKU @@ -105,31 +57,14 @@ BCollator::BCollator(BMessage *archive) if (archive->FindBool("loc:punctuation", &fIgnorePunctuation) != B_OK) fIgnorePunctuation = true; - BMessage collatorArchive; - if (archive->FindMessage("loc:collator", &collatorArchive) == B_OK) { - BArchivable *unarchived = - instantiate_object(&collatorArchive, &fCollatorImage); - - // do we really have a BCollatorAddOn here? - fCollator = dynamic_cast(unarchived); - if (fCollator == NULL) - delete unarchived; - } - - if (fCollator == NULL) { - fCollator = new BCollatorAddOn(); - fCollatorImage = B_ERROR; - } + // TODO : ICU collator ? or just store the locale name ? #endif } BCollator::~BCollator() { - delete fCollator; - - if (fCollatorImage >= B_OK) - unload_add_on(fCollatorImage); + delete fICUCollator; } @@ -164,22 +99,61 @@ BCollator::IgnorePunctuation() const status_t BCollator::GetSortKey(const char *string, BString *key, int8 strength) { + // TODO : handle fIgnorePunctuation and strength if (strength == B_COLLATE_DEFAULT) strength = fStrength; - return fCollator->GetSortKey(string, key, strength, fIgnorePunctuation); + int length = strlen(string); + + uint8_t* buffer = (uint8_t*)malloc(length * 2); + // According to ICU documentation this should be enough in "most cases" + if (buffer == NULL) + return B_NO_MEMORY; + + int requiredSize = fICUCollator->getSortKey(UnicodeString(string, length), + buffer, length * 2); + if (requiredSize > length * 2) { + buffer = (uint8_t*)realloc(buffer, requiredSize); + if (buffer == NULL) + return B_NO_MEMORY; + fICUCollator->getSortKey(UnicodeString(string, length), buffer, + requiredSize); + } + + key->SetTo((char*)buffer); + free(buffer); + + return B_OK; } int BCollator::Compare(const char *a, const char *b, int32 length, int8 strength) { - if (length == -1) // match the whole string - length = 0x7fffffff; - - return fCollator->Compare(a, b, length, - strength == B_COLLATE_DEFAULT ? fStrength : strength, - fIgnorePunctuation); + // TODO : handle fIgnorePunctuation + if (strength == B_COLLATE_DEFAULT) + strength = fStrength; + Collator::ECollationStrength icuStrength; + switch(strength) { + case B_COLLATE_PRIMARY: + icuStrength = Collator::PRIMARY; + break; + case B_COLLATE_SECONDARY: + icuStrength = Collator::SECONDARY; + break; + case B_COLLATE_TERTIARY: + default: + icuStrength = Collator::TERTIARY; + break; + case B_COLLATE_QUATERNARY: + icuStrength = Collator::QUATERNARY; + break; + case B_COLLATE_IDENTICAL: + icuStrength = Collator::IDENTICAL; + break; + } + fICUCollator->setStrength(icuStrength); + return fICUCollator->compare(a, b, length); } @@ -195,12 +169,15 @@ BCollator::Archive(BMessage *archive, bool deep) const if (status == B_OK) status = archive->AddBool("loc:punctuation", fIgnorePunctuation); + /* BMessage collatorArchive; if (status == B_OK && deep && typeid(*fCollator) != typeid(BCollatorAddOn) // only archive subclasses from BCollatorAddOn && (status = fCollator->Archive(&collatorArchive, true)) == B_OK) status = archive->AddMessage("loc:collator", &collatorArchive); + */ + //TODO : archive fICUCollator return status; } @@ -216,273 +193,3 @@ BCollator::Instantiate(BMessage *archive) } -// #pragma mark - - - -BCollatorAddOn::BCollatorAddOn() -{ -} - - -BCollatorAddOn::BCollatorAddOn(BMessage *archive) - : BArchivable(archive) -{ -} - - -BCollatorAddOn::~BCollatorAddOn() -{ -} - - -/** This returns the next Unicode character from the UTF-8 encoded - * input string, and bumps it to the next character. - * It will ignore punctuation if specified by the context, and - * might substitute characters if needed. - */ - -uint32 -BCollatorAddOn::GetNextChar(const char **string, input_context &context) -{ - uint32 c = context.next_char; - if (c != 0) { - context.next_char = 0; - return c; - } - - do { - c = BUnicodeChar::FromUTF8(string); - } while (context.ignore_punctuation - && (BUnicodeChar::IsPunctuation(c) || BUnicodeChar::IsSpace(c))); - - if (c == 223) { - context.next_char = 's'; - return 's'; - } - - return c; -} - - -/** Fills the specified buffer with the primary sort key. The buffer - * has to be long enough to hold the key. - * It returns the position in the buffer immediately after the key; - * it does not add a terminating null byte! - */ - -char * -BCollatorAddOn::PutPrimaryKey(const char *string, char *buffer, int32 length, - bool ignorePunctuation) -{ - input_context context(ignorePunctuation); - - uint32 c; - for (int32 i = 0; (c = GetNextChar(&string, context)) != 0 && i < length; - i++) { - if (c < 0x80) - *buffer++ = tolower(c); - else - BUnicodeChar::ToUTF8(getPrimaryChar(c), &buffer); - } - - return buffer; -} - - -size_t -BCollatorAddOn::PrimaryKeyLength(size_t length) -{ - return length * 2; - // the primary key needs to make space for doubled characters (like 'ß') -} - - -status_t -BCollatorAddOn::GetSortKey(const char *string, BString *key, int8 strength, - bool ignorePunctuation) -{ - if (strength >= B_COLLATE_QUATERNARY) { - // the difference between tertiary and quaternary collation strength - // are usually a different handling of punctuation characters - ignorePunctuation = false; - } - - size_t length = strlen(string); - - switch (strength) { - case B_COLLATE_PRIMARY: - { - char *begin = key->LockBuffer(PrimaryKeyLength(length)); - if (begin == NULL) - return B_NO_MEMORY; - - char *end = PutPrimaryKey(string, begin, length, ignorePunctuation); - *end = '\0'; - - key->UnlockBuffer(end - begin); - break; - } - - case B_COLLATE_SECONDARY: - { - char *begin - = key->LockBuffer(PrimaryKeyLength(length) + length + 1); - // the primary key + the secondary key + separator char - if (begin == NULL) - return B_NO_MEMORY; - - char *buffer = PutPrimaryKey(string, begin, length, ignorePunctuation); - *buffer++ = '\01'; - // separator - - input_context context(ignorePunctuation); - uint32 c; - for (uint32 i = 0; - (c = GetNextChar(&string, context)) && i < length; i++) { - if (c < 0x80) - *buffer++ = tolower(c); - else - BUnicodeChar::ToUTF8(BUnicodeChar::ToLower(c), &buffer); - } - *buffer = '\0'; - - key->UnlockBuffer(buffer - begin); - break; - } - - case B_COLLATE_TERTIARY: - case B_COLLATE_QUATERNARY: - { - char *begin - = key->LockBuffer(PrimaryKeyLength(length) + length + 1); - // the primary key + the tertiary key + separator char - if (begin == NULL) - return B_NO_MEMORY; - - char *buffer = PutPrimaryKey(string, begin, length, ignorePunctuation); - *buffer++ = '\01'; - // separator - - input_context context(ignorePunctuation); - uint32 c; - for (uint32 i = 0; - (c = GetNextChar(&string, context)) && i < length; i++) { - BUnicodeChar::ToUTF8(c, &buffer); - } - *buffer = '\0'; - - key->UnlockBuffer(buffer + length - begin); - break; - } - - case B_COLLATE_IDENTICAL: - default: - key->SetTo(string, length); - // is there any way to check if BString::SetTo() actually succeeded? - break; - } - return B_OK; -} - - -int -BCollatorAddOn::Compare(const char *a, const char *b, int32 length, - int8 strength, bool ignorePunctuation) -{ - if (strength >= B_COLLATE_QUATERNARY) { - // the difference between tertiary and quaternary collation strength - // are usually a different handling of punctuation characters - ignorePunctuation = false; - } - - input_context contextA(ignorePunctuation); - input_context contextB(ignorePunctuation); - - switch (strength) { - case B_COLLATE_PRIMARY: - { - for (int32 i = 0; i < length; i++) { - uint32 charA = GetNextChar(&a, contextA); - uint32 charB = GetNextChar(&b, contextB); - if (charA == 0) - return charB == 0 ? 0 : -(int32)charB; - else if (charB == 0) - return (int32)charA; - - charA = getPrimaryChar(charA); - charB = getPrimaryChar(charB); - - if (charA != charB) - return (int32)charA - (int32)charB; - } - return 0; - } - - case B_COLLATE_SECONDARY: - { - // diacriticals can only change the order between equal strings - int32 compare - = Compare(a, b, length, B_COLLATE_PRIMARY, ignorePunctuation); - if (compare != 0) - return compare; - - for (int32 i = 0; i < length; i++) { - uint32 charA = BUnicodeChar::ToLower(GetNextChar(&a, contextA)); - uint32 charB = BUnicodeChar::ToLower(GetNextChar(&b, contextB)); - - // the two strings does have the same size when we get here - if (charA == 0) - return 0; - - if (charA != charB) - return (int32)charA - (int32)charB; - } - return 0; - } - - case B_COLLATE_TERTIARY: - case B_COLLATE_QUATERNARY: - { - // diacriticals can only change the order between equal strings - int32 compare - = Compare(a, b, length, B_COLLATE_PRIMARY, ignorePunctuation); - if (compare != 0) - return compare; - - for (int32 i = 0; i < length; i++) { - uint32 charA = GetNextChar(&a, contextA); - uint32 charB = GetNextChar(&b, contextB); - - // the two strings does have the same size when we get here - if (charA == 0) - return 0; - - if (charA != charB) - return (int32)charA - (int32)charB; - } - return 0; - } - - case B_COLLATE_IDENTICAL: - default: - return strncmp(a, b, length); - } -} - - -status_t -BCollatorAddOn::Archive(BMessage *archive, bool deep) const -{ - return BArchivable::Archive(archive, deep); -} - - -BArchivable * -BCollatorAddOn::Instantiate(BMessage *archive) -{ - if (validate_instantiation(archive, "BCollatorAddOn")) - return new BCollatorAddOn(archive); - - return NULL; -} -