Improve handling of mbyte chars in UTF8Char::IsSpace

* Switch UTF8Char's IsSpace, IsAlNum, ToLower functions to use
  system-wide BUnicodeChar service routines;
* Switch TermView::CharClassifier to use UTF8Char instead of
  raw char* string ponter. That reduces count of conversions
  and simplify code;

  Fixes #7423.

src/apps/terminal/BasicTerminalBuffer.cpp

@@ -421,7 +421,7 @@ BasicTerminalBuffer::FindWord(const TermPos& pos,
 		x--;
 
 	// get the char type at the given position
-	int type = classifier->Classify(line->cells[x].character.bytes);
+	int type = classifier->Classify(line->cells[x].character);
 
 	// check whether we are supposed to find words only
 	if (type != CHAR_TYPE_WORD_CHAR && !findNonWords)
@@ -444,7 +444,7 @@ BasicTerminalBuffer::FindWord(const TermPos& pos,
 		if (x > 0 && IS_WIDTH(line->cells[x - 1].attributes))
 			x--;
 
-		if (classifier->Classify(line->cells[x].character.bytes) != type)
+		if (classifier->Classify(line->cells[x].character) != type)
 			break;
 
 		start.SetTo(x, y);
@@ -467,7 +467,7 @@ BasicTerminalBuffer::FindWord(const TermPos& pos,
 				break;
 		}
 
-		if (classifier->Classify(line->cells[x].character.bytes) != type)
+		if (classifier->Classify(line->cells[x].character) != type)
 			break;
 
 		x += IS_WIDTH(line->cells[x].attributes) ? 2 : 1;

src/apps/terminal/TermView.cpp

@@ -24,6 +24,7 @@
 
 #include <algorithm>
 #include <new>
+#include <vector>
 
 #include <Alert.h>
 #include <Application.h>
@@ -144,28 +145,34 @@ restrict_value(const Type& value, const Type& min, const Type& max)
 class TermView::CharClassifier : public TerminalCharClassifier {
 public:
 	CharClassifier(const char* specialWordChars)
-		:
-		fSpecialWordChars(specialWordChars)
 	{
+		const char* p = specialWordChars;
+		while (p != NULL && *p) {
+			int count = UTF8Char::ByteCount(*p);
+			if (count <= 0 || count > 4)
+				break;
+			fSpecialWordChars.push_back(UTF8Char(p, count));
+			p += count;
+		}
 	}
 
-	virtual int Classify(const char* character)
+	virtual int Classify(const UTF8Char& character)
 	{
-		// TODO: Deal correctly with non-ASCII chars.
+		if (character.IsSpace())
-		char c = *character;
+			return CHAR_TYPE_SPACE;
-		if (UTF8Char::ByteCount(c) > 1)
+
+		if (character.IsAlNum())
 			return CHAR_TYPE_WORD_CHAR;
 
-		if (isspace(c))
+		if (std::find(fSpecialWordChars.begin(), fSpecialWordChars.end(),
-			return CHAR_TYPE_SPACE;
+			character) != fSpecialWordChars.end())
-		if (isalnum(c) || strchr(fSpecialWordChars, c) != NULL)
 			return CHAR_TYPE_WORD_CHAR;
 
 		return CHAR_TYPE_WORD_DELIMITER;
 	}
 
 private:
-	const char*	fSpecialWordChars;
+	std::vector<UTF8Char> fSpecialWordChars;
 };
 
 

src/apps/terminal/TerminalCharClassifier.h

@@ -13,11 +13,13 @@ enum {
 };
 
 
+class UTF8Char;
+
 class TerminalCharClassifier {
 public:
 	virtual						~TerminalCharClassifier();
 
-	virtual	int					Classify(const char* character) = 0;
+	virtual	int					Classify(const UTF8Char& character) = 0;
 };
 
 

src/apps/terminal/UTF8Char.h

@@ -8,6 +8,8 @@
 #include <ctype.h>
 #include <string.h>
 
+#include <UnicodeChar.h>
+
 
 struct UTF8Char {
 	char	bytes[4];
@@ -63,17 +65,23 @@ struct UTF8Char {
 
 	bool IsSpace() const
 	{
-		// TODO: Support multi-byte chars!
+		return BUnicodeChar::IsSpace(BUnicodeChar::FromUTF8(bytes));
-		return ByteCount() == 1 ? isspace(bytes[0]) : false;
+	}
+
+	bool IsAlNum() const
+	{
+		return BUnicodeChar::IsAlNum(BUnicodeChar::FromUTF8(bytes));
 	}
 
 	UTF8Char ToLower() const
 	{
-		// TODO: Support multi-byte chars!
+		uint32 c = BUnicodeChar::ToLower(BUnicodeChar::FromUTF8(bytes));
-		if (ByteCount() > 1)
-			return *this;
 
-		return UTF8Char((char)tolower(bytes[0]));
+		UTF8Char character;
+		char* utf8 = character.bytes;
+		BUnicodeChar::ToUTF8(c, &utf8);
+
+		return character;
 	}
 
 	bool operator==(const UTF8Char& other) const