* Collator API now actually uses ICU.

git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@37783 a95241bf-73f2-0310-859d-f6bbb57e9c96
This commit is contained in:
Adrien Destugues
2010-07-28 11:29:14 +00:00
parent fdc1b8cd84
commit 110649a1a4
2 changed files with 91 additions and 406 deletions
+28 -50
View File
@@ -1,3 +1,9 @@
/*
* Copyright 2003-2010, Haiku, Inc.
* Distributed under the terms of the MIT Licence.
*/
#ifndef _COLLATOR_H_
#define _COLLATOR_H_
@@ -5,6 +11,12 @@
#include <SupportDefs.h>
#include <Archivable.h>
namespace icu_44 {
class Collator;
};
class BString;
class BCollatorAddOn;
@@ -13,7 +25,8 @@ enum collator_strengths {
B_COLLATE_DEFAULT = -1,
B_COLLATE_PRIMARY = 1, // e.g.: no diacritical differences, e = é
B_COLLATE_SECONDARY, // diacritics are different from their base characters, a != ä
B_COLLATE_SECONDARY, // diacritics are different from their base
// characters, a != ä
B_COLLATE_TERTIARY, // case sensitive comparison
B_COLLATE_QUATERNARY,
@@ -24,7 +37,7 @@ enum collator_strengths {
class BCollator : public BArchivable {
public:
BCollator();
BCollator(BCollatorAddOn *collator, int8 strength, bool ignorePunctuation);
BCollator(const char *locale, int8 strength, bool ignorePunctuation);
BCollator(BMessage *archive);
~BCollator();
@@ -34,20 +47,24 @@ class BCollator : public BArchivable {
void SetIgnorePunctuation(bool ignore);
bool IgnorePunctuation() const;
status_t GetSortKey(const char *string, BString *key, int8 strength = B_COLLATE_DEFAULT);
status_t GetSortKey(const char *string, BString *key,
int8 strength = B_COLLATE_DEFAULT);
int Compare(const char *, const char *, int32 len = -1, int8 strength = B_COLLATE_DEFAULT);
bool Equal(const char *, const char *, int32 len = -1, int8 strength = B_COLLATE_DEFAULT);
bool Greater(const char *, const char *, int32 len = -1, int8 strength = B_COLLATE_DEFAULT);
bool GreaterOrEqual(const char *, const char *, int32 len = -1, int8 strength = B_COLLATE_DEFAULT);
int Compare(const char *, const char *, int32 len = -1,
int8 strength = B_COLLATE_DEFAULT);
bool Equal(const char *, const char *, int32 len = -1,
int8 strength = B_COLLATE_DEFAULT);
bool Greater(const char *, const char *, int32 len = -1,
int8 strength = B_COLLATE_DEFAULT);
bool GreaterOrEqual(const char *, const char *, int32 len = -1,
int8 strength = B_COLLATE_DEFAULT);
// (un-)archiving API
status_t Archive(BMessage *archive, bool deep) const;
static BArchivable *Instantiate(BMessage *archive);
private:
BCollatorAddOn *fCollator;
image_id fCollatorImage;
icu_44::Collator* fICUCollator;
int8 fStrength;
bool fIgnorePunctuation;
};
@@ -68,50 +85,11 @@ BCollator::Greater(const char *s1, const char *s2, int32 len, int8 strength)
inline bool
BCollator::GreaterOrEqual(const char *s1, const char *s2, int32 len, int8 strength)
BCollator::GreaterOrEqual(const char *s1, const char *s2, int32 len,
int8 strength)
{
return Compare(s1, s2, len, strength) >= 0;
}
/************************************************************************/
// For BCollator add-on implementations:
class BCollatorAddOn : public BArchivable {
public:
BCollatorAddOn();
BCollatorAddOn(BMessage *archive);
virtual ~BCollatorAddOn();
virtual status_t GetSortKey(const char *string, BString *key, int8 strength,
bool ignorePunctuation);
virtual int Compare(const char *a, const char *b, int32 length, int8 strength,
bool ignorePunctuation);
// (un-)archiving API
virtual status_t Archive(BMessage *archive, bool deep) const;
static BArchivable *Instantiate(BMessage *archive);
protected:
struct input_context {
input_context(bool ignorePunctuation);
bool ignore_punctuation;
uint32 next_char;
int32 reserved1;
int32 reserved2;
};
virtual uint32 GetNextChar(const char **string, input_context &context);
virtual size_t PrimaryKeyLength(size_t length);
virtual char *PutPrimaryKey(const char *string, char *buffer, int32 length,
bool ignorePunctuation);
};
// If your add-on should work with the standard tool to create a language, it
// should export that function. However, once the language file has been written
// only the archived collator is used, and that function won't be called anymore.
extern "C" BCollatorAddOn *instantiate_collator(void);
#endif /* _COLLATOR_H_ */
+63 -356
View File
@@ -1,6 +1,7 @@
/*
** Copyright 2003, Axel Dörfler, [email protected]. All rights reserved.
** Distributed under the terms of the OpenBeOS License.
* Copyright 2003, Axel Dörfler, [email protected]. All rights reserved.
* Copyright 2010, Adrien Destugues <[email protected]>
* Distributed under the terms of the MIT License.
*/
@@ -9,90 +10,41 @@
#include <String.h>
#include <Message.h>
#include <stdlib.h>
#include <typeinfo>
#include <ctype.h>
// conversion array for character ranges 192 - 223 & 224 - 255
static const uint8 kNoDiacrits[] = {
'a','a','a','a','a','a','a',
'c',
'e','e','e','e',
'i','i','i','i',
240, // eth
'n',
'o','o','o','o','o',
247, //
'o',
'u','u','u','u',
'y',
254, // thorn
'y'
};
static inline uint32
getPrimaryChar(uint32 c)
{
if (c < 0x80)
return tolower(c);
// this automatically returns lowercase letters
if (c >= 192 && c < 223)
return kNoDiacrits[c - 192];
if (c == 223) // ß
return 's';
if (c >= 224 && c < 256)
return kNoDiacrits[c - 224];
return BUnicodeChar::ToLower(c);
}
BCollatorAddOn::input_context::input_context(bool ignorePunctuation)
:
ignore_punctuation(ignorePunctuation),
next_char(0),
reserved1(0),
reserved2(0)
{
}
// #pragma mark -
#include <unicode/coll.h>
BCollator::BCollator()
:
fCollatorImage(B_ERROR),
fStrength(B_COLLATE_PRIMARY),
fIgnorePunctuation(true)
{
// ToDo: the collator construction will have to change; the default
// TODO: the collator construction will have to change; the default
// collator should be constructed by the Locale/LocaleRoster, so we
// only need a constructor where you specify all details
fCollator = new BCollatorAddOn();
UErrorCode error = U_ZERO_ERROR;
fICUCollator = Collator::createInstance(error);
}
BCollator::BCollator(BCollatorAddOn *collator, int8 strength,
BCollator::BCollator(const char *locale, int8 strength,
bool ignorePunctuation)
:
fCollator(collator),
fCollatorImage(B_ERROR),
fStrength(strength),
fIgnorePunctuation(ignorePunctuation)
{
if (collator == NULL)
fCollator = new BCollatorAddOn();
UErrorCode error = U_ZERO_ERROR;
fICUCollator = Collator::createInstance(locale, error);
}
BCollator::BCollator(BMessage *archive)
: BArchivable(archive),
fCollator(NULL),
fCollatorImage(B_ERROR),
fICUCollator(NULL),
fIgnorePunctuation(true)
{
#if HAIKU_TARGET_PLATFORM_HAIKU
@@ -105,31 +57,14 @@ BCollator::BCollator(BMessage *archive)
if (archive->FindBool("loc:punctuation", &fIgnorePunctuation) != B_OK)
fIgnorePunctuation = true;
BMessage collatorArchive;
if (archive->FindMessage("loc:collator", &collatorArchive) == B_OK) {
BArchivable *unarchived =
instantiate_object(&collatorArchive, &fCollatorImage);
// do we really have a BCollatorAddOn here?
fCollator = dynamic_cast<BCollatorAddOn *>(unarchived);
if (fCollator == NULL)
delete unarchived;
}
if (fCollator == NULL) {
fCollator = new BCollatorAddOn();
fCollatorImage = B_ERROR;
}
// TODO : ICU collator ? or just store the locale name ?
#endif
}
BCollator::~BCollator()
{
delete fCollator;
if (fCollatorImage >= B_OK)
unload_add_on(fCollatorImage);
delete fICUCollator;
}
@@ -164,22 +99,61 @@ BCollator::IgnorePunctuation() const
status_t
BCollator::GetSortKey(const char *string, BString *key, int8 strength)
{
// TODO : handle fIgnorePunctuation and strength
if (strength == B_COLLATE_DEFAULT)
strength = fStrength;
return fCollator->GetSortKey(string, key, strength, fIgnorePunctuation);
int length = strlen(string);
uint8_t* buffer = (uint8_t*)malloc(length * 2);
// According to ICU documentation this should be enough in "most cases"
if (buffer == NULL)
return B_NO_MEMORY;
int requiredSize = fICUCollator->getSortKey(UnicodeString(string, length),
buffer, length * 2);
if (requiredSize > length * 2) {
buffer = (uint8_t*)realloc(buffer, requiredSize);
if (buffer == NULL)
return B_NO_MEMORY;
fICUCollator->getSortKey(UnicodeString(string, length), buffer,
requiredSize);
}
key->SetTo((char*)buffer);
free(buffer);
return B_OK;
}
int
BCollator::Compare(const char *a, const char *b, int32 length, int8 strength)
{
if (length == -1) // match the whole string
length = 0x7fffffff;
return fCollator->Compare(a, b, length,
strength == B_COLLATE_DEFAULT ? fStrength : strength,
fIgnorePunctuation);
// TODO : handle fIgnorePunctuation
if (strength == B_COLLATE_DEFAULT)
strength = fStrength;
Collator::ECollationStrength icuStrength;
switch(strength) {
case B_COLLATE_PRIMARY:
icuStrength = Collator::PRIMARY;
break;
case B_COLLATE_SECONDARY:
icuStrength = Collator::SECONDARY;
break;
case B_COLLATE_TERTIARY:
default:
icuStrength = Collator::TERTIARY;
break;
case B_COLLATE_QUATERNARY:
icuStrength = Collator::QUATERNARY;
break;
case B_COLLATE_IDENTICAL:
icuStrength = Collator::IDENTICAL;
break;
}
fICUCollator->setStrength(icuStrength);
return fICUCollator->compare(a, b, length);
}
@@ -195,12 +169,15 @@ BCollator::Archive(BMessage *archive, bool deep) const
if (status == B_OK)
status = archive->AddBool("loc:punctuation", fIgnorePunctuation);
/*
BMessage collatorArchive;
if (status == B_OK && deep
&& typeid(*fCollator) != typeid(BCollatorAddOn)
// only archive subclasses from BCollatorAddOn
&& (status = fCollator->Archive(&collatorArchive, true)) == B_OK)
status = archive->AddMessage("loc:collator", &collatorArchive);
*/
//TODO : archive fICUCollator
return status;
}
@@ -216,273 +193,3 @@ BCollator::Instantiate(BMessage *archive)
}
// #pragma mark -
BCollatorAddOn::BCollatorAddOn()
{
}
BCollatorAddOn::BCollatorAddOn(BMessage *archive)
: BArchivable(archive)
{
}
BCollatorAddOn::~BCollatorAddOn()
{
}
/** This returns the next Unicode character from the UTF-8 encoded
* input string, and bumps it to the next character.
* It will ignore punctuation if specified by the context, and
* might substitute characters if needed.
*/
uint32
BCollatorAddOn::GetNextChar(const char **string, input_context &context)
{
uint32 c = context.next_char;
if (c != 0) {
context.next_char = 0;
return c;
}
do {
c = BUnicodeChar::FromUTF8(string);
} while (context.ignore_punctuation
&& (BUnicodeChar::IsPunctuation(c) || BUnicodeChar::IsSpace(c)));
if (c == 223) {
context.next_char = 's';
return 's';
}
return c;
}
/** Fills the specified buffer with the primary sort key. The buffer
* has to be long enough to hold the key.
* It returns the position in the buffer immediately after the key;
* it does not add a terminating null byte!
*/
char *
BCollatorAddOn::PutPrimaryKey(const char *string, char *buffer, int32 length,
bool ignorePunctuation)
{
input_context context(ignorePunctuation);
uint32 c;
for (int32 i = 0; (c = GetNextChar(&string, context)) != 0 && i < length;
i++) {
if (c < 0x80)
*buffer++ = tolower(c);
else
BUnicodeChar::ToUTF8(getPrimaryChar(c), &buffer);
}
return buffer;
}
size_t
BCollatorAddOn::PrimaryKeyLength(size_t length)
{
return length * 2;
// the primary key needs to make space for doubled characters (like 'ß')
}
status_t
BCollatorAddOn::GetSortKey(const char *string, BString *key, int8 strength,
bool ignorePunctuation)
{
if (strength >= B_COLLATE_QUATERNARY) {
// the difference between tertiary and quaternary collation strength
// are usually a different handling of punctuation characters
ignorePunctuation = false;
}
size_t length = strlen(string);
switch (strength) {
case B_COLLATE_PRIMARY:
{
char *begin = key->LockBuffer(PrimaryKeyLength(length));
if (begin == NULL)
return B_NO_MEMORY;
char *end = PutPrimaryKey(string, begin, length, ignorePunctuation);
*end = '\0';
key->UnlockBuffer(end - begin);
break;
}
case B_COLLATE_SECONDARY:
{
char *begin
= key->LockBuffer(PrimaryKeyLength(length) + length + 1);
// the primary key + the secondary key + separator char
if (begin == NULL)
return B_NO_MEMORY;
char *buffer = PutPrimaryKey(string, begin, length, ignorePunctuation);
*buffer++ = '\01';
// separator
input_context context(ignorePunctuation);
uint32 c;
for (uint32 i = 0;
(c = GetNextChar(&string, context)) && i < length; i++) {
if (c < 0x80)
*buffer++ = tolower(c);
else
BUnicodeChar::ToUTF8(BUnicodeChar::ToLower(c), &buffer);
}
*buffer = '\0';
key->UnlockBuffer(buffer - begin);
break;
}
case B_COLLATE_TERTIARY:
case B_COLLATE_QUATERNARY:
{
char *begin
= key->LockBuffer(PrimaryKeyLength(length) + length + 1);
// the primary key + the tertiary key + separator char
if (begin == NULL)
return B_NO_MEMORY;
char *buffer = PutPrimaryKey(string, begin, length, ignorePunctuation);
*buffer++ = '\01';
// separator
input_context context(ignorePunctuation);
uint32 c;
for (uint32 i = 0;
(c = GetNextChar(&string, context)) && i < length; i++) {
BUnicodeChar::ToUTF8(c, &buffer);
}
*buffer = '\0';
key->UnlockBuffer(buffer + length - begin);
break;
}
case B_COLLATE_IDENTICAL:
default:
key->SetTo(string, length);
// is there any way to check if BString::SetTo() actually succeeded?
break;
}
return B_OK;
}
int
BCollatorAddOn::Compare(const char *a, const char *b, int32 length,
int8 strength, bool ignorePunctuation)
{
if (strength >= B_COLLATE_QUATERNARY) {
// the difference between tertiary and quaternary collation strength
// are usually a different handling of punctuation characters
ignorePunctuation = false;
}
input_context contextA(ignorePunctuation);
input_context contextB(ignorePunctuation);
switch (strength) {
case B_COLLATE_PRIMARY:
{
for (int32 i = 0; i < length; i++) {
uint32 charA = GetNextChar(&a, contextA);
uint32 charB = GetNextChar(&b, contextB);
if (charA == 0)
return charB == 0 ? 0 : -(int32)charB;
else if (charB == 0)
return (int32)charA;
charA = getPrimaryChar(charA);
charB = getPrimaryChar(charB);
if (charA != charB)
return (int32)charA - (int32)charB;
}
return 0;
}
case B_COLLATE_SECONDARY:
{
// diacriticals can only change the order between equal strings
int32 compare
= Compare(a, b, length, B_COLLATE_PRIMARY, ignorePunctuation);
if (compare != 0)
return compare;
for (int32 i = 0; i < length; i++) {
uint32 charA = BUnicodeChar::ToLower(GetNextChar(&a, contextA));
uint32 charB = BUnicodeChar::ToLower(GetNextChar(&b, contextB));
// the two strings does have the same size when we get here
if (charA == 0)
return 0;
if (charA != charB)
return (int32)charA - (int32)charB;
}
return 0;
}
case B_COLLATE_TERTIARY:
case B_COLLATE_QUATERNARY:
{
// diacriticals can only change the order between equal strings
int32 compare
= Compare(a, b, length, B_COLLATE_PRIMARY, ignorePunctuation);
if (compare != 0)
return compare;
for (int32 i = 0; i < length; i++) {
uint32 charA = GetNextChar(&a, contextA);
uint32 charB = GetNextChar(&b, contextB);
// the two strings does have the same size when we get here
if (charA == 0)
return 0;
if (charA != charB)
return (int32)charA - (int32)charB;
}
return 0;
}
case B_COLLATE_IDENTICAL:
default:
return strncmp(a, b, length);
}
}
status_t
BCollatorAddOn::Archive(BMessage *archive, bool deep) const
{
return BArchivable::Archive(archive, deep);
}
BArchivable *
BCollatorAddOn::Instantiate(BMessage *archive)
{
if (validate_instantiation(archive, "BCollatorAddOn"))
return new BCollatorAddOn(archive);
return NULL;
}