Partial clean-up

git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@35767 a95241bf-73f2-0310-859d-f6bbb57e9c96
This commit is contained in:
Jonas Sundström
2010-03-05 20:50:58 +00:00
parent 80b2e0594f
commit fbc75a5372
4 changed files with 298 additions and 228 deletions
-1
View File
@@ -83,7 +83,6 @@ All rights reserved.
#include "Status.h" #include "Status.h"
#include "String.h" #include "String.h"
#include "Utilities.h" #include "Utilities.h"
#include "Words.h"
#define TR_CONTEXT "Mail" #define TR_CONTEXT "Mail"
+239 -177
View File
@@ -32,69 +32,78 @@ names are registered trademarks or trademarks of their respective holders.
All rights reserved. All rights reserved.
*/ */
#include <stdlib.h>
#include <stdio.h> #include "WIndex.h"
#include <ctype.h>
#include <File.h> #include <File.h>
#include <Node.h>
#include <fs_attr.h> #include <fs_attr.h>
#include <Message.h> #include <Message.h>
#include "WIndex.h" #include <Node.h>
#include <ctype.h>
#include <stdlib.h>
#include <stdio.h>
#define IVERSION 1 #define IVERSION 1
static int32 kCRCTable = 0; static int32 kCRCTable = 0;
int32 cmp_i_entries( const WIndexEntry *e1, const WIndexEntry *e2 );
int32 cmp_i_entries(const WIndexEntry *e1, const WIndexEntry *e2);
void gen_crc_table(); void gen_crc_table();
unsigned long update_crc( unsigned long crc_accum, const char *data_blk_ptr, int data_blk_size ); unsigned long update_crc(unsigned long crc_accum, const char *data_blk_ptr,
int data_blk_size);
FileEntry::FileEntry( void ) FileEntry::FileEntry(void)
{ {
} }
FileEntry::FileEntry( const char *entryStr )
: BString( entryStr ) FileEntry::FileEntry(const char *entryString)
:
BString(entryString)
{ {
} }
status_t WIndex::SetTo( const char *dataPath, const char *indexPath )
status_t
WIndex::SetTo(const char *dataPath, const char *indexPath)
{ {
BFile *dataFile; BFile* dataFile;
BFile indexFile; BFile indexFile;
dataFile = new BFile(); dataFile = new BFile();
if( dataFile->SetTo( dataPath, B_READ_ONLY ) != B_OK ) if (dataFile->SetTo(dataPath, B_READ_ONLY) != B_OK) {
return B_ERROR; return B_ERROR;
else } else {
{
bool buildIndex = true; bool buildIndex = true;
SetTo( dataFile ); SetTo(dataFile);
time_t mtime; time_t mtime;
time_t modified; time_t modified;
dataFile->GetModificationTime( &mtime ); dataFile->GetModificationTime(&mtime);
if( indexFile.SetTo( indexPath, B_READ_ONLY ) == B_OK ) if (indexFile.SetTo(indexPath, B_READ_ONLY) == B_OK) {
{
attr_info info; attr_info info;
if( (indexFile.GetAttrInfo( "WINDEX:version", &info ) == B_NO_ERROR) ) if ((indexFile.GetAttrInfo("WINDEX:version", &info) == B_OK)) {
{
uint32 version = 0; uint32 version = 0;
indexFile.ReadAttr( "WINDEX:version", B_UINT32_TYPE, 0, &version, 4 ); indexFile.ReadAttr("WINDEX:version", B_UINT32_TYPE, 0,
if( IVERSION == version ) &version, 4);
{ if (IVERSION == version) {
if( (indexFile.GetAttrInfo( "WINDEX:modified", &info ) == B_NO_ERROR) ) if ((indexFile.GetAttrInfo("WINDEX:modified", &info)
{ == B_OK)) {
indexFile.ReadAttr( "WINDEX:modified", B_UINT32_TYPE, 0, &modified, 4 ); indexFile.ReadAttr("WINDEX:modified", B_UINT32_TYPE, 0,
if( mtime == modified ) &modified, 4);
{
if (UnflattenIndex( &indexFile ) == B_OK) if (mtime == modified) {
if (UnflattenIndex(&indexFile) == B_OK)
buildIndex = false; buildIndex = false;
} }
} }
@@ -102,169 +111,189 @@ status_t WIndex::SetTo( const char *dataPath, const char *indexPath )
} }
indexFile.Unset(); indexFile.Unset();
} }
if( buildIndex ) if (buildIndex) {
{
// printf( "Building Index...\n" );
InitIndex(); InitIndex();
BuildIndex(); BuildIndex();
if( indexFile.SetTo( indexPath, B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE ) == B_OK ) if (indexFile.SetTo(indexPath,
{ B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE) == B_OK) {
FlattenIndex( &indexFile ); FlattenIndex(&indexFile);
indexFile.WriteAttr( "WINDEX:modified", B_UINT32_TYPE, 0, &mtime, 4 ); indexFile.WriteAttr("WINDEX:modified", B_UINT32_TYPE, 0,
&mtime, 4);
uint32 version = IVERSION; uint32 version = IVERSION;
indexFile.WriteAttr( "WINDEX:version", B_UINT32_TYPE, 0, &version, 4 ); indexFile.WriteAttr("WINDEX:version", B_UINT32_TYPE, 0,
&version, 4);
} }
} }
} }
return B_OK; return B_OK;
} }
FileEntry::~FileEntry( void )
FileEntry::~FileEntry(void)
{ {
} }
WIndex::WIndex( int32 count )
WIndex::WIndex(int32 count)
{ {
entryList = NULL; fEntryList = NULL;
dataFile = NULL; fDataFile = NULL;
ePerB = count; fEntriesPerBlock = count;
entrySize = sizeof( WIndexEntry ); fEntrySize = sizeof(WIndexEntry);
if( !atomic_or( &kCRCTable, 1 ) ) if (!atomic_or(&kCRCTable, 1))
gen_crc_table(); gen_crc_table();
} }
WIndex::WIndex( BPositionIO *dataFile, int32 count )
WIndex::WIndex(BPositionIO *dataFile, int32 count)
{ {
entryList = NULL; fEntryList = NULL;
this->dataFile = dataFile; fDataFile = dataFile;
ePerB = count; fEntriesPerBlock = count;
entrySize = sizeof( WIndexEntry ); fEntrySize = sizeof(WIndexEntry);
if( !atomic_or( &kCRCTable, 1 ) ) if (!atomic_or(&kCRCTable, 1))
gen_crc_table(); gen_crc_table();
} }
WIndex::~WIndex( void )
WIndex::~WIndex(void)
{ {
if( entryList ) if (fEntryList)
free( entryList ); free(fEntryList);
delete dataFile; delete fDataFile;
} }
status_t WIndex::UnflattenIndex( BPositionIO *io )
status_t
WIndex::UnflattenIndex(BPositionIO *io)
{ {
if( entryList ) if (fEntryList)
free( entryList ); free(fEntryList);
WIndexHead head; WIndexHead head;
io->Seek( 0, SEEK_SET ); io->Seek(0, SEEK_SET);
io->Read( &head, sizeof( head ) ); io->Read(&head, sizeof(head));
io->Seek( head.offset, SEEK_SET ); io->Seek(head.offset, SEEK_SET);
entrySize = head.entrySize; fEntrySize = head.entrySize;
entries = head.entries; fEntries = head.entries;
maxEntries = ePerB; fMaxEntries = fEntriesPerBlock;
blockSize = ePerB * entrySize; fBlockSize = fEntriesPerBlock * fEntrySize;
blocks = entries/ePerB+1;; fBlocks = fEntries / fEntriesPerBlock + 1;;
isSorted = true; fIsSorted = true;
int32 size = (head.entries+1) * head.entrySize; int32 size = (head.entries + 1) * head.entrySize;
if( !(entryList = (uint8 *)malloc( size )) ) if (!(fEntryList = (uint8 *)malloc(size)))
return B_ERROR; return B_ERROR;
if( entries ) if (fEntries)
io->Read( entryList, size ); io->Read(fEntryList, size);
return B_OK; return B_OK;
} }
status_t WIndex::FlattenIndex( BPositionIO *io )
status_t
WIndex::FlattenIndex(BPositionIO *io)
{ {
if( entries && !isSorted ) if (fEntries && !fIsSorted)
SortItems(); SortItems();
WIndexHead head; WIndexHead head;
head.entries = entries; head.entries = fEntries;
head.entrySize = entrySize; head.entrySize = fEntrySize;
head.offset = sizeof( WIndexHead ); head.offset = sizeof(WIndexHead);
io->Seek( 0, SEEK_SET ); io->Seek(0, SEEK_SET);
io->Write( &head, sizeof( head ) ); io->Write(&head, sizeof(head));
if( entries ) if (fEntries)
io->Write( entryList, head.entries * head.entrySize ); io->Write(fEntryList, head.entries * head.entrySize);
return B_OK; return B_OK;
} }
int32 WIndex::Lookup( int32 key )
int32
WIndex::Lookup(int32 key)
{ {
if( !entries ) if (!fEntries)
return -1; return -1;
if( !isSorted ) if (!fIsSorted)
SortItems(); SortItems();
// Binary Search // Binary Search
int32 M, Lb, Ub; int32 M, Lb, Ub;
Lb = 0; Lb = 0;
Ub = entries-1; Ub = fEntries - 1;
while( true ) while (true) {
{ M = (Lb + Ub) / 2;
M = (Lb + Ub)/2; if (key < ((WIndexEntry *)(fEntryList + (M * fEntrySize)))->key)
if( key < ((WIndexEntry *)(entryList+(M*entrySize)))->key )
Ub = M - 1; Ub = M - 1;
else if(key > ((WIndexEntry *)(entryList+(M*entrySize)))->key ) else if (key > ((WIndexEntry *)(fEntryList + (M * fEntrySize)))->key)
Lb = M + 1; Lb = M + 1;
else else
return M; return M;
if( Lb > Ub ) if (Lb > Ub)
return -1; return -1;
} }
} }
status_t WIndex::AddItem( WIndexEntry *entry )
status_t
WIndex::AddItem(WIndexEntry *entry)
{ {
if( BlockCheck() == B_ERROR ) if (_BlockCheck() == B_ERROR)
return B_ERROR; return B_ERROR;
memcpy( ((WIndexEntry *)(entryList+(entries*entrySize))), entry, entrySize ); memcpy(((WIndexEntry *)(fEntryList + (fEntries * fEntrySize))), entry,
entries++; fEntrySize);
isSorted = false; fEntries++;
fIsSorted = false;
return B_OK; return B_OK;
} }
void WIndex::SortItems( void )
void
WIndex::SortItems(void)
{ {
qsort( entryList, entries, entrySize, (int (*)(const void *, const void *))cmp_i_entries ); qsort(fEntryList, fEntries, fEntrySize,
isSorted = true; (int(*)(const void *, const void *))cmp_i_entries);
//for( int32 i = 0; i < entries; i++ )
// printf( "Key = %ld\n", entryList[i].key ); fIsSorted = true;
} }
status_t WIndex::BlockCheck( void )
status_t
WIndex::_BlockCheck(void)
{ {
if( entries < maxEntries ) if (fEntries < fMaxEntries)
return B_OK; return B_OK;
blocks = entries/ePerB+1; fBlocks = fEntries / fEntriesPerBlock + 1;
entryList = (uint8 *)realloc( entryList, blockSize*blocks ); fEntryList = (uint8 *)realloc(fEntryList, fBlockSize * fBlocks);
if( !entryList ) if (!fEntryList)
return B_ERROR; return B_ERROR;
return B_OK; return B_OK;
} }
status_t WIndex::InitIndex( void )
status_t
WIndex::InitIndex(void)
{ {
if( entryList ) if (fEntryList)
free( entryList ); free(fEntryList);
isSorted = 0; fIsSorted = 0;
entries = 0; fEntries = 0;
maxEntries = ePerB; fMaxEntries = fEntriesPerBlock;
blockSize = ePerB * entrySize; fBlockSize = fEntriesPerBlock * fEntrySize;
blocks = 1; fBlocks = 1;
entryList = (uint8 *)malloc( blockSize ); fEntryList = (uint8 *)malloc(fBlockSize);
if( !entryList ) if (!fEntryList)
return B_ERROR; return B_ERROR;
return B_OK; return B_OK;
} }
int32 WIndex::GetKey( const char *s )
int32
WIndex::GetKey(const char *s)
{ {
int32 key = 0; int32 key = 0;
@@ -272,61 +301,70 @@ int32 WIndex::GetKey( const char *s )
int32 a = 84589; int32 a = 84589;
int32 b = 45989; int32 b = 45989;
int32 m = 217728; int32 m = 217728;
while( *s ) while (*s) {
{
x = *s++ - 'a'; x = *s++ - 'a';
key ^= (a*x + b) % m; key ^= (a * x + b) % m;
key <<= 1; key <<= 1;
}*/ }*/
key = update_crc( 0, s, strlen(s) ); key = update_crc(0, s, strlen(s));
if( key < 0 ) // No negavite values! if (key < 0) // No negative values!
key = ~key; key = ~key;
return key; return key;
} }
int32 cmp_i_entries( const WIndexEntry *e1, const WIndexEntry *e2 )
int32
cmp_i_entries(const WIndexEntry *e1, const WIndexEntry *e2)
{ {
return e1->key - e2->key; return e1->key - e2->key;
} }
status_t WIndex::SetTo( BPositionIO *dataFile )
status_t
WIndex::SetTo(BPositionIO *dataFile)
{ {
this->dataFile = dataFile; fDataFile = dataFile;
return B_OK; return B_OK;
} }
void WIndex::Unset( void )
void
WIndex::Unset(void)
{ {
dataFile = NULL; fDataFile = NULL;
} }
int32 WIndex::FindFirst( const char *word )
int32
WIndex::FindFirst(const char *word)
{ {
if( !entries ) if (!fEntries)
return -1; return -1;
int32 index; int32 index;
char nword[256]; char nword[256];
int32 key; int32 key;
NormalizeWord( word, nword ); NormalizeWord(word, nword);
key = GetKey( nword ); key = GetKey(nword);
if( (index = Lookup( key )) < 0 ) if ((index = Lookup(key)) < 0)
return -1; return -1;
// Find first instance of key // Find first instance of key
while( (ItemAt( index-1 ))->key == key ) while ((ItemAt(index - 1))->key == key)
index--; index--;
return index; return index;
} }
FileEntry *WIndex::GetEntry( int32 index )
FileEntry*
WIndex::GetEntry(int32 index)
{ {
if( (index >= entries)||(index < 0) ) if ((index >= fEntries)||(index < 0))
return NULL; return NULL;
WIndexEntry *ientry; WIndexEntry *ientry;
FileEntry *dentry; FileEntry *dentry;
@@ -334,34 +372,40 @@ FileEntry *WIndex::GetEntry( int32 index )
dentry = new FileEntry(); dentry = new FileEntry();
ientry = ItemAt( index ); ientry = ItemAt(index);
int32 size; int32 size;
dataFile->Seek( ientry->offset, SEEK_SET ); fDataFile->Seek(ientry->offset, SEEK_SET);
buffer = dentry->LockBuffer( 256 ); buffer = dentry->LockBuffer(256);
dataFile->Read( buffer, 256 ); fDataFile->Read(buffer, 256);
size = GetEntrySize( ientry, buffer ); size = _GetEntrySize(ientry, buffer);
//buffer[256] = 0; //buffer[256] = 0;
//printf( "Entry: = %s\n", buffer ); //printf("Entry: = %s\n", buffer);
dentry->UnlockBuffer( size ); dentry->UnlockBuffer(size);
return dentry; return dentry;
} }
size_t WIndex::GetEntrySize( WIndexEntry *entry, const char *entryData )
size_t
WIndex::_GetEntrySize(WIndexEntry *entry, const char *entryData)
{ {
// eliminate unused parameter warning // eliminate unused parameter warning
(void)entry; (void)entry;
return strcspn( entryData, "\n\r" ); return strcspn(entryData, "\n\r");
} }
FileEntry *WIndex::GetEntry( const char *word )
FileEntry*
WIndex::GetEntry(const char *word)
{ {
return GetEntry( FindFirst( word ) ); return GetEntry(FindFirst(word));
} }
char *WIndex::NormalizeWord( const char *word, char *dest )
char*
WIndex::NormalizeWord(const char *word, char *dest)
{ {
const char *src; const char *src;
char *dst; char *dst;
@@ -369,9 +413,8 @@ char *WIndex::NormalizeWord( const char *word, char *dest )
// remove dots and copy // remove dots and copy
src = word; src = word;
dst = dest; dst = dest;
while( *src ) while (*src) {
{ if (*src != '.')
if( *src != '.' )
*dst++ = *src; *dst++ = *src;
src++; src++;
} }
@@ -379,11 +422,12 @@ char *WIndex::NormalizeWord( const char *word, char *dest )
// convert to lower-case // convert to lower-case
dst = dest; dst = dest;
while( *dst ) while (*dst)
*dst++ = tolower( *dst ); *dst++ = tolower(*dst);
return dest; return dest;
} }
/* crc32h.c -- package to compute 32-bit CRC one byte at a time using */ /* crc32h.c -- package to compute 32-bit CRC one byte at a time using */
/* the high-bit first (Big-Endian) bit ordering convention */ /* the high-bit first (Big-Endian) bit ordering convention */
/* */ /* */
@@ -418,30 +462,48 @@ char *WIndex::NormalizeWord( const char *word, char *dest )
/* The table lookup technique was adapted from the algorithm described */ /* The table lookup technique was adapted from the algorithm described */
/* by Avram Perez, Byte-wise CRC Calculations, IEEE Micro 3, 40 (1983).*/ /* by Avram Perez, Byte-wise CRC Calculations, IEEE Micro 3, 40 (1983).*/
#define POLYNOMIAL 0x04c11db7L #define POLYNOMIAL 0x04c11db7L
static unsigned long crc_table[256]; static unsigned long crc_table[256];
void gen_crc_table()
/* generate the table of CRC remainders for all possible bytes */
{ register int i, j; register unsigned long crc_accum;
for ( i = 0; i < 256; i++ )
{ crc_accum = ( (unsigned long) i << 24 );
for ( j = 0; j < 8; j++ )
{ if ( crc_accum & 0x80000000L )
crc_accum =
( crc_accum << 1 ) ^ POLYNOMIAL;
else
crc_accum =
( crc_accum << 1 ); }
crc_table[i] = crc_accum; }
return; }
unsigned long update_crc( unsigned long crc_accum, const char *data_blk_ptr, int data_blk_size ) void
/* update the CRC on the data block one byte at a time */ gen_crc_table()
{ register int i, j; {
for ( j = 0; j < data_blk_size; j++ ) // generate the table of CRC remainders for all possible bytes
{ i = ( (int) ( crc_accum >> 24) ^ *data_blk_ptr++ ) & 0xff;
crc_accum = ( crc_accum << 8 ) ^ crc_table[i]; } register int i, j;
return crc_accum; } register unsigned long crc_accum;
for (i = 0; i < 256; i++) {
crc_accum = ((unsigned long) i << 24);
for (j = 0; j < 8; j++) {
if (crc_accum & 0x80000000L)
crc_accum = (crc_accum << 1) ^ POLYNOMIAL;
else
crc_accum = (crc_accum << 1);
}
crc_table[i] = crc_accum;
}
return;
}
unsigned long
update_crc(unsigned long crc_accum, const char *data_blk_ptr, int data_blk_size)
{
// update the CRC on the data block one byte at a time
register int i, j;
for (j = 0; j < data_blk_size; j++) {
i = ((int) (crc_accum >> 24) ^ *data_blk_ptr++) & 0xff;
crc_accum = (crc_accum << 8) ^ crc_table[i];
}
return crc_accum;
}
+55 -46
View File
@@ -31,77 +31,86 @@ of Be Incorporated in the United States and other countries. Other brand product
names are registered trademarks or trademarks of their respective holders. names are registered trademarks or trademarks of their respective holders.
All rights reserved. All rights reserved.
*/ */
#ifndef _WORD_INDEX_H #ifndef _WORD_INDEX_H
#define _WORD_INDEX_H #define _WORD_INDEX_H
#include <DataIO.h> #include <DataIO.h>
#include <String.h> #include <String.h>
struct WIndexHead { struct WIndexHead {
int32 entries; int32 entries;
int32 entrySize; int32 entrySize;
int32 offset; int32 offset;
}; };
struct WIndexEntry { struct WIndexEntry {
int32 key; int32 key;
int32 offset; int32 offset;
}; };
class FileEntry : public BString { class FileEntry : public BString {
public: public:
FileEntry(void); FileEntry();
FileEntry(const char *entryStr); FileEntry(const char* entryStr);
virtual ~FileEntry(void); virtual ~FileEntry();
}; };
class WIndex { class WIndex {
public: public:
WIndex(BPositionIO *dataFile, int32 count = 100); WIndex(BPositionIO* dataFile, int32 count = 100);
WIndex(int32 count = 100); WIndex(int32 count = 100);
virtual ~WIndex(void); virtual ~WIndex();
status_t InitIndex(void); status_t InitIndex();
status_t UnflattenIndex(BPositionIO *io); status_t UnflattenIndex(BPositionIO* io);
status_t FlattenIndex(BPositionIO *io); status_t FlattenIndex(BPositionIO* io);
int32 Lookup(int32 key); int32 Lookup(int32 key);
inline WIndexEntry *ItemAt(int32 index) inline WIndexEntry* ItemAt(int32 index) {
{ return (WIndexEntry *)(entryList+(index*entrySize)); } return (WIndexEntry*)
status_t AddItem(WIndexEntry *entry); (fEntryList + (index * fEntrySize));
inline int32 CountItems(void) }
{ return entries; }
void SortItems(void); status_t AddItem(WIndexEntry* entry);
inline int32 CountItems() {
virtual int32 GetKey(const char *s); return fEntries;
virtual char *NormalizeWord(const char *word, char *dest); }
void SortItems();
status_t SetTo(BPositionIO *dataFile);
status_t SetTo(const char *dataPath, const char *indexPath); virtual int32 GetKey(const char* s);
void Unset(void); virtual char* NormalizeWord(const char* word, char* dest);
virtual status_t BuildIndex(void) = 0;
virtual int32 FindFirst(const char *word);
virtual FileEntry *GetEntry(int32 index);
FileEntry *GetEntry(const char *word);
status_t SetTo(BPositionIO* dataFile);
status_t SetTo(const char* dataPath, const char* indexPath);
void Unset();
virtual status_t BuildIndex() = 0;
virtual int32 FindFirst(const char* word);
virtual FileEntry* GetEntry(int32 index);
FileEntry* GetEntry(const char* word);
protected: protected:
status_t BlockCheck(void); status_t _BlockCheck();
virtual size_t GetEntrySize(WIndexEntry *entry, const char *entryData); virtual size_t _GetEntrySize(WIndexEntry* entry,
const char* entryData);
int32 entrySize;
int32 entries; int32 fEntrySize;
int32 maxEntries; int32 fEntries;
int32 ePerB; int32 fMaxEntries;
int32 blockSize; int32 fEntriesPerBlock;
int32 blocks; int32 fBlockSize;
bool isSorted; int32 fBlocks;
uint8 *entryList; bool fIsSorted;
BPositionIO *dataFile; uint8* fEntryList;
BPositionIO* fDataFile;
}; };
#endif // #ifndef _WORD_INDEX_H #endif // _WORD_INDEX_H
+4 -4
View File
@@ -104,7 +104,7 @@ Words::Words(const char* dataPath, const char* indexPath, bool useMetaphone)
fUseMetaphone(useMetaphone) fUseMetaphone(useMetaphone)
{ {
if (!useMetaphone) if (!useMetaphone)
entrySize = sizeof(uint32); fEntrySize = sizeof(uint32);
SetTo(dataPath, indexPath); SetTo(dataPath, indexPath);
} }
@@ -130,14 +130,14 @@ Words::BuildIndex(void)
int32 state = FIND_WORD; int32 state = FIND_WORD;
// Make sure we are at start of file // Make sure we are at start of file
dataFile->Seek(0, SEEK_SET); fDataFile->Seek(0, SEEK_SET);
entry.offset = -1; entry.offset = -1;
// Read blocks from thes until eof // Read blocks from thes until eof
while (true) { while (true) {
// Get next block // Get next block
blockOffset = dataFile->Position(); blockOffset = fDataFile->Position();
if ((blockSize = dataFile->Read(buffer, 16384)) == 0) if ((blockSize = fDataFile->Read(buffer, 16384)) == 0)
break; break;
// parse block // parse block