* Collator API now actually uses ICU.

git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@37783 a95241bf-73f2-0310-859d-f6bbb57e9c96

headers/os/locale/Collator.h

@@ -1,3 +1,9 @@
+/*
+ * Copyright 2003-2010, Haiku, Inc.
+ * Distributed under the terms of the MIT Licence.
+*/
+
+
 #ifndef _COLLATOR_H_
 #define _COLLATOR_H_
 
@@ -5,6 +11,12 @@
 #include <SupportDefs.h>
 #include <Archivable.h>
 
+
+namespace icu_44 {
+	class Collator;
+};
+
+
 class BString;
 class BCollatorAddOn;
 
@@ -13,7 +25,8 @@ enum collator_strengths {
 	B_COLLATE_DEFAULT = -1,
 
 	B_COLLATE_PRIMARY = 1,		// e.g.: no diacritical differences, e = é
-	B_COLLATE_SECONDARY,		// diacritics are different from their base characters, a != ä
+	B_COLLATE_SECONDARY,		// diacritics are different from their base
+								// characters, a != ä
 	B_COLLATE_TERTIARY,			// case sensitive comparison
 	B_COLLATE_QUATERNARY,
 
@@ -24,7 +37,7 @@ enum collator_strengths {
 class BCollator : public BArchivable {
 	public:
 		BCollator();
-		BCollator(BCollatorAddOn *collator, int8 strength, bool ignorePunctuation);
+		BCollator(const char *locale, int8 strength, bool ignorePunctuation);
 		BCollator(BMessage *archive);
 		~BCollator();
 
@@ -34,20 +47,24 @@ class BCollator : public BArchivable {
 		void SetIgnorePunctuation(bool ignore);
 		bool IgnorePunctuation() const;
 
-		status_t GetSortKey(const char *string, BString *key, int8 strength = B_COLLATE_DEFAULT);
+		status_t GetSortKey(const char *string, BString *key,
+			int8 strength = B_COLLATE_DEFAULT);
 
-		int Compare(const char *, const char *, int32 len = -1, int8 strength = B_COLLATE_DEFAULT);
+		int Compare(const char *, const char *, int32 len = -1,
-		bool Equal(const char *, const char *, int32 len = -1, int8 strength = B_COLLATE_DEFAULT);
+			int8 strength = B_COLLATE_DEFAULT);
-		bool Greater(const char *, const char *, int32 len = -1, int8 strength = B_COLLATE_DEFAULT);
+		bool Equal(const char *, const char *, int32 len = -1,
-		bool GreaterOrEqual(const char *, const char *, int32 len = -1, int8 strength = B_COLLATE_DEFAULT);
+			int8 strength = B_COLLATE_DEFAULT);
+		bool Greater(const char *, const char *, int32 len = -1,
+			int8 strength = B_COLLATE_DEFAULT);
+		bool GreaterOrEqual(const char *, const char *, int32 len = -1,
+			int8 strength = B_COLLATE_DEFAULT);
 
 		// (un-)archiving API
 		status_t Archive(BMessage *archive, bool deep) const;
 		static BArchivable *Instantiate(BMessage *archive);
 
 	private:
-		BCollatorAddOn	*fCollator;
+		icu_44::Collator* fICUCollator;
-		image_id		fCollatorImage;
 		int8			fStrength;
 		bool			fIgnorePunctuation;
 };
@@ -68,50 +85,11 @@ BCollator::Greater(const char *s1, const char *s2, int32 len, int8 strength)
 
 
 inline bool
-BCollator::GreaterOrEqual(const char *s1, const char *s2, int32 len, int8 strength)
+BCollator::GreaterOrEqual(const char *s1, const char *s2, int32 len,
+	int8 strength)
 {
 	return Compare(s1, s2, len, strength) >= 0;
 }
 
 
-/************************************************************************/
-
-// For BCollator add-on implementations:
-
-class BCollatorAddOn : public BArchivable {
-	public:
-		BCollatorAddOn();
-		BCollatorAddOn(BMessage *archive);
-		virtual ~BCollatorAddOn();
-
-		virtual status_t GetSortKey(const char *string, BString *key, int8 strength,
-						bool ignorePunctuation);
-		virtual int Compare(const char *a, const char *b, int32 length, int8 strength,
-						bool ignorePunctuation);
-
-		// (un-)archiving API
-		virtual status_t Archive(BMessage *archive, bool deep) const;
-		static BArchivable *Instantiate(BMessage *archive);
-
-	protected:
-		struct input_context {
-			input_context(bool ignorePunctuation);
-
-			bool	ignore_punctuation;
-			uint32	next_char;
-			int32	reserved1;
-			int32	reserved2;
-		};
-
-		virtual uint32 GetNextChar(const char **string, input_context &context);
-		virtual size_t PrimaryKeyLength(size_t length);
-		virtual char *PutPrimaryKey(const char *string, char *buffer, int32 length,
-						bool ignorePunctuation);
-};
-
-// If your add-on should work with the standard tool to create a language, it
-// should export that function. However, once the language file has been written
-// only the archived collator is used, and that function won't be called anymore.
-extern "C" BCollatorAddOn *instantiate_collator(void);
-
 #endif	/* _COLLATOR_H_ */

src/kits/locale/Collator.cpp

@@ -1,6 +1,7 @@
 /*
-** Copyright 2003, Axel Dörfler, [email protected]. All rights reserved.
+* Copyright 2003, Axel Dörfler, [email protected]. All rights reserved.
-** Distributed under the terms of the OpenBeOS License.
+* Copyright 2010, Adrien Destugues <[email protected]>
+* Distributed under the terms of the MIT License.
 */
 
 
@@ -9,90 +10,41 @@
 #include <String.h>
 #include <Message.h>
 
+#include <stdlib.h>
 #include <typeinfo>
 #include <ctype.h>
 
-
+#include <unicode/coll.h>
-// conversion array for character ranges 192 - 223 & 224 - 255
-static const uint8 kNoDiacrits[] = {
-	'a','a','a','a','a','a','a',
-	'c',
-	'e','e','e','e',
-	'i','i','i','i',
-	240,	// eth
-	'n',
-	'o','o','o','o','o',
-	247,	//
-	'o',
-	'u','u','u','u',
-	'y',
-	254,	// thorn
-	'y'
-};
-
-
-static inline uint32
-getPrimaryChar(uint32 c)
-{
-	if (c < 0x80)
-		return tolower(c);
-
-	// this automatically returns lowercase letters
-	if (c >= 192 && c < 223)
-		return kNoDiacrits[c - 192];
-	if (c == 223)	// ß
-		return 's';
-	if (c >= 224 && c < 256)
-		return kNoDiacrits[c - 224];
-
-	return BUnicodeChar::ToLower(c);
-}
-
-
-BCollatorAddOn::input_context::input_context(bool ignorePunctuation)
-	:
-	ignore_punctuation(ignorePunctuation),
-	next_char(0),
-	reserved1(0),
-	reserved2(0)
-{
-}
-
-
-//	#pragma mark -
 
 
 BCollator::BCollator()
 	:
-	fCollatorImage(B_ERROR),
 	fStrength(B_COLLATE_PRIMARY),
 	fIgnorePunctuation(true)
 {
-	// ToDo: the collator construction will have to change; the default
+	// TODO: the collator construction will have to change; the default
 	//	collator should be constructed by the Locale/LocaleRoster, so we
 	//	only need a constructor where you specify all details
 
-	fCollator = new BCollatorAddOn();
+	UErrorCode error = U_ZERO_ERROR;
+	fICUCollator = Collator::createInstance(error);
 }
 
 
-BCollator::BCollator(BCollatorAddOn *collator, int8 strength,
+BCollator::BCollator(const char *locale, int8 strength,
 	bool ignorePunctuation)
 	:
-	fCollator(collator),
-	fCollatorImage(B_ERROR),
 	fStrength(strength),
 	fIgnorePunctuation(ignorePunctuation)
 {
-	if (collator == NULL)
+	UErrorCode error = U_ZERO_ERROR;
-		fCollator = new BCollatorAddOn();
+	fICUCollator = Collator::createInstance(locale, error);
 }
 
 
 BCollator::BCollator(BMessage *archive)
 	: BArchivable(archive),
-	fCollator(NULL),
+	fICUCollator(NULL),
-	fCollatorImage(B_ERROR),
 	fIgnorePunctuation(true)
 {
 #if HAIKU_TARGET_PLATFORM_HAIKU
@@ -105,31 +57,14 @@ BCollator::BCollator(BMessage *archive)
 	if (archive->FindBool("loc:punctuation", &fIgnorePunctuation) != B_OK)
 		fIgnorePunctuation = true;
 
-	BMessage collatorArchive;
+	// TODO : ICU collator ? or just store the locale name ?
-	if (archive->FindMessage("loc:collator", &collatorArchive) == B_OK) {
-		BArchivable *unarchived =
-			instantiate_object(&collatorArchive, &fCollatorImage);
-
-		// do we really have a BCollatorAddOn here?
-		fCollator = dynamic_cast<BCollatorAddOn *>(unarchived);
-		if (fCollator == NULL)
-			delete unarchived;
-	}
-
-	if (fCollator == NULL) {
-		fCollator = new BCollatorAddOn();
-		fCollatorImage = B_ERROR;
-	}
 #endif
 }
 
 
 BCollator::~BCollator()
 {
-	delete fCollator;
+	delete fICUCollator;
-
-	if (fCollatorImage >= B_OK)
-		unload_add_on(fCollatorImage);
 }
 
 
@@ -164,22 +99,61 @@ BCollator::IgnorePunctuation() const
 status_t
 BCollator::GetSortKey(const char *string, BString *key, int8 strength)
 {
+	// TODO : handle fIgnorePunctuation and strength
 	if (strength == B_COLLATE_DEFAULT)
 		strength = fStrength;
 
-	return fCollator->GetSortKey(string, key, strength, fIgnorePunctuation);
+	int length = strlen(string);
+
+	uint8_t* buffer = (uint8_t*)malloc(length * 2);
+		// According to ICU documentation this should be enough in "most cases"
+	if (buffer == NULL)
+		return B_NO_MEMORY;
+
+	int requiredSize = fICUCollator->getSortKey(UnicodeString(string, length),
+		buffer, length * 2);
+	if (requiredSize > length * 2) {
+		buffer = (uint8_t*)realloc(buffer, requiredSize);
+		if (buffer == NULL)
+			return B_NO_MEMORY;
+		fICUCollator->getSortKey(UnicodeString(string, length), buffer,
+			requiredSize);
+	}
+
+	key->SetTo((char*)buffer);
+	free(buffer);
+
+	return B_OK;
 }
 
 
 int
 BCollator::Compare(const char *a, const char *b, int32 length, int8 strength)
 {
-	if (length == -1)	// match the whole string
+	// TODO : handle fIgnorePunctuation
-		length = 0x7fffffff;
+	if (strength == B_COLLATE_DEFAULT)
-
+		strength = fStrength;
-	return fCollator->Compare(a, b, length,
+	Collator::ECollationStrength icuStrength;
-				strength == B_COLLATE_DEFAULT ? fStrength : strength,
+	switch(strength) {
-				fIgnorePunctuation);
+		case B_COLLATE_PRIMARY:
+			icuStrength = Collator::PRIMARY;
+			break;
+		case B_COLLATE_SECONDARY:
+			icuStrength = Collator::SECONDARY;
+			break;
+		case B_COLLATE_TERTIARY:
+		default:
+			icuStrength = Collator::TERTIARY;
+			break;
+		case B_COLLATE_QUATERNARY:
+			icuStrength = Collator::QUATERNARY;
+			break;
+		case B_COLLATE_IDENTICAL:
+			icuStrength = Collator::IDENTICAL;
+			break;
+	}
+	fICUCollator->setStrength(icuStrength);
+	return fICUCollator->compare(a, b, length);
 }
 
 
@@ -195,12 +169,15 @@ BCollator::Archive(BMessage *archive, bool deep) const
 	if (status == B_OK)
 		status = archive->AddBool("loc:punctuation", fIgnorePunctuation);
 
+	/*
 	BMessage collatorArchive;
 	if (status == B_OK && deep
 		&& typeid(*fCollator) != typeid(BCollatorAddOn)
 			// only archive subclasses from BCollatorAddOn
 		&& (status = fCollator->Archive(&collatorArchive, true)) == B_OK)
 		status = archive->AddMessage("loc:collator", &collatorArchive);
+	*/
+	//TODO : archive fICUCollator
 
 	return status;
 }
@@ -216,273 +193,3 @@ BCollator::Instantiate(BMessage *archive)
 }
 
 
-//	#pragma mark -
-
-
-BCollatorAddOn::BCollatorAddOn()
-{
-}
-
-
-BCollatorAddOn::BCollatorAddOn(BMessage *archive)
-	: BArchivable(archive)
-{
-}
-
-
-BCollatorAddOn::~BCollatorAddOn()
-{
-}
-
-
-/** This returns the next Unicode character from the UTF-8 encoded
- *	input string, and bumps it to the next character.
- *	It will ignore punctuation if specified by the context, and
- *	might substitute characters if needed.
- */
-
-uint32
-BCollatorAddOn::GetNextChar(const char **string, input_context &context)
-{
-	uint32 c = context.next_char;
-	if (c != 0) {
-		context.next_char = 0;
-		return c;
-	}
-
-	do {
-		c = BUnicodeChar::FromUTF8(string);
-	} while (context.ignore_punctuation
-		&& (BUnicodeChar::IsPunctuation(c) || BUnicodeChar::IsSpace(c)));
-
-	if (c == 223) {
-		context.next_char = 's';
-		return 's';
-	}
-
-	return c;
-}
-
-
-/** Fills the specified buffer with the primary sort key. The buffer
- *	has to be long enough to hold the key.
- *	It returns the position in the buffer immediately after the key;
- *	it does not add a terminating null byte!
- */
-
-char *
-BCollatorAddOn::PutPrimaryKey(const char *string, char *buffer, int32 length,
-	bool ignorePunctuation)
-{
-	input_context context(ignorePunctuation);
-
-	uint32 c;
-	for (int32 i = 0; (c = GetNextChar(&string, context)) != 0 && i < length;
-		i++) {
-		if (c < 0x80)
-			*buffer++ = tolower(c);
-		else
-			BUnicodeChar::ToUTF8(getPrimaryChar(c), &buffer);
-	}
-
-	return buffer;
-}
-
-
-size_t
-BCollatorAddOn::PrimaryKeyLength(size_t length)
-{
-	return length * 2;
-		// the primary key needs to make space for doubled characters (like 'ß')
-}
-
-
-status_t
-BCollatorAddOn::GetSortKey(const char *string, BString *key, int8 strength,
-	bool ignorePunctuation)
-{
-	if (strength >= B_COLLATE_QUATERNARY) {
-		// the difference between tertiary and quaternary collation strength
-		// are usually a different handling of punctuation characters
-		ignorePunctuation = false;
-	}
-
-	size_t length = strlen(string);
-
-	switch (strength) {
-		case B_COLLATE_PRIMARY:
-		{
-			char *begin = key->LockBuffer(PrimaryKeyLength(length));
-			if (begin == NULL)
-				return B_NO_MEMORY;
-
-			char *end = PutPrimaryKey(string, begin, length, ignorePunctuation);
-			*end = '\0';
-
-			key->UnlockBuffer(end - begin);
-			break;
-		}
-
-		case B_COLLATE_SECONDARY:
-		{
-			char *begin
-				= key->LockBuffer(PrimaryKeyLength(length) + length + 1);
-					// the primary key + the secondary key + separator char
-			if (begin == NULL)
-				return B_NO_MEMORY;
-
-			char *buffer = PutPrimaryKey(string, begin, length, ignorePunctuation);
-			*buffer++ = '\01';
-				// separator
-
-			input_context context(ignorePunctuation);
-			uint32 c;
-			for (uint32 i = 0;
-				(c = GetNextChar(&string, context)) && i < length; i++) {
-				if (c < 0x80)
-					*buffer++ = tolower(c);
-				else
-					BUnicodeChar::ToUTF8(BUnicodeChar::ToLower(c), &buffer);
-			}
-			*buffer = '\0';
-
-			key->UnlockBuffer(buffer - begin);
-			break;
-		}
-
-		case B_COLLATE_TERTIARY:
-		case B_COLLATE_QUATERNARY:
-		{
-			char *begin
-				= key->LockBuffer(PrimaryKeyLength(length) + length + 1);
-					// the primary key + the tertiary key + separator char
-			if (begin == NULL)
-				return B_NO_MEMORY;
-
-			char *buffer = PutPrimaryKey(string, begin, length, ignorePunctuation);
-			*buffer++ = '\01';
-				// separator
-
-			input_context context(ignorePunctuation);
-			uint32 c;
-			for (uint32 i = 0;
-				(c = GetNextChar(&string, context)) && i < length; i++) {
-				BUnicodeChar::ToUTF8(c, &buffer);
-			}
-			*buffer = '\0';
-
-			key->UnlockBuffer(buffer + length - begin);
-			break;
-		}
-
-		case B_COLLATE_IDENTICAL:
-		default:
-			key->SetTo(string, length);
-				// is there any way to check if BString::SetTo() actually succeeded?
-			break;
-	}
-	return B_OK;
-}
-
-
-int
-BCollatorAddOn::Compare(const char *a, const char *b, int32 length,
-	int8 strength, bool ignorePunctuation)
-{
-	if (strength >= B_COLLATE_QUATERNARY) {
-		// the difference between tertiary and quaternary collation strength
-		// are usually a different handling of punctuation characters
-		ignorePunctuation = false;
-	}
-
-	input_context contextA(ignorePunctuation);
-	input_context contextB(ignorePunctuation);
-
-	switch (strength) {
-		case B_COLLATE_PRIMARY:
-		{
-			for (int32 i = 0; i < length; i++) {
-				uint32 charA = GetNextChar(&a, contextA);
-				uint32 charB = GetNextChar(&b, contextB);
-				if (charA == 0)
-					return charB == 0 ? 0 : -(int32)charB;
-				else if (charB == 0)
-					return (int32)charA;
-
-				charA = getPrimaryChar(charA);
-				charB = getPrimaryChar(charB);
-
-				if (charA != charB)
-					return (int32)charA - (int32)charB;
-			}
-			return 0;
-		}
-
-		case B_COLLATE_SECONDARY:
-		{
-			// diacriticals can only change the order between equal strings
-			int32 compare
-				= Compare(a, b, length, B_COLLATE_PRIMARY, ignorePunctuation);
-			if (compare != 0)
-				return compare;
-
-			for (int32 i = 0; i < length; i++) {
-				uint32 charA = BUnicodeChar::ToLower(GetNextChar(&a, contextA));
-				uint32 charB = BUnicodeChar::ToLower(GetNextChar(&b, contextB));
-
-				// the two strings does have the same size when we get here
-				if (charA == 0)
-					return 0;
-
-				if (charA != charB)
-					return (int32)charA - (int32)charB;
-			}
-			return 0;
-		}
-
-		case B_COLLATE_TERTIARY:
-		case B_COLLATE_QUATERNARY:
-		{
-			// diacriticals can only change the order between equal strings
-			int32 compare
-				= Compare(a, b, length, B_COLLATE_PRIMARY, ignorePunctuation);
-			if (compare != 0)
-				return compare;
-
-			for (int32 i = 0; i < length; i++) {
-				uint32 charA = GetNextChar(&a, contextA);
-				uint32 charB = GetNextChar(&b, contextB);
-
-				// the two strings does have the same size when we get here
-				if (charA == 0)
-					return 0;
-
-				if (charA != charB)
-					return (int32)charA - (int32)charB;
-			}
-			return 0;
-		}
-
-		case B_COLLATE_IDENTICAL:
-		default:
-			return strncmp(a, b, length);
-	}
-}
-
-
-status_t
-BCollatorAddOn::Archive(BMessage *archive, bool deep) const
-{
-	return BArchivable::Archive(archive, deep);
-}
-
-
-BArchivable *
-BCollatorAddOn::Instantiate(BMessage *archive)
-{
-	if (validate_instantiation(archive, "BCollatorAddOn"))
-		return new BCollatorAddOn(archive);
-
-	return NULL;
-}
-