i2p/haiku-beta6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Just a few minutes after checking in, I mentioned it to Japanese expert

Browse Source 
Koki and he suggested also including the Japanese comma.  So before I
forget to do it...


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@15195 a95241bf-73f2-0310-859d-f6bbb57e9c96
This commit is contained in:
Alexander G.M. Smith
2005-11-28 02:07:55 +00:00
parent 45bfb95c2a
commit 968748e669
1 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bin/bemail_utils/spamdbm.cpp
+8 -3
View File
@@ -2351,7 +2351,7 @@ static size_t TokenizerPassRemoveHTMLStyle (
/* Convert Japanese periods (a round hollow dot symbol) to spaces so that the /* Convert Japanese periods (a round hollow dot symbol) to spaces so that the
start of the next sentence is recognised at least as the start of a very long start of the next sentence is recognised at least as the start of a very long
word. */ word. The Japanese comma also does the same job. */
static size_t TokenizerPassJapanesePeriodsToSpaces ( static size_t TokenizerPassJapanesePeriodsToSpaces (
char *BufferPntr, char *BufferPntr,
@@ -2359,8 +2359,13 @@ static size_t TokenizerPassJapanesePeriodsToSpaces (
char PrefixCharacter, char PrefixCharacter,
set<string> &WordSet) set<string> &WordSet)
{ {
return TokenizerUtilRemoveStartEndThing (BufferPntr, size_t BytesRemaining = NumberOfBytes;
NumberOfBytes, PrefixCharacter, WordSet, "。", "", true);
BytesRemaining = TokenizerUtilRemoveStartEndThing (BufferPntr,
BytesRemaining, PrefixCharacter, WordSet, "。" /* period */, "", true);
BytesRemaining = TokenizerUtilRemoveStartEndThing (BufferPntr,
BytesRemaining, PrefixCharacter, WordSet, "、" /* comma */, "", true);
return BytesRemaining;
} }