From 568dabdb5061f8c99137f502d5a0fdaddf38d1b5 Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Sat, 30 Oct 2004 23:21:59 +0000 Subject: [PATCH] Added a server that caches stat data and directory contents. To use this feature run StatCacheServer somewhere in the background. The first run of jam will be slower, but subsequent ones will usually benefit. Only little, if the BeOS FS cache is able to hold the data, but as soon as the number of files and directories jam has to inspect hits the limit at which the cache drops data needed for the next run, the performance difference will be very noticeable. E.g. the time for a "jam -n > /dev/null" in the root of a partially built tree on my machine dropped from >1 min without the server to 13 secs with it (not in the first run, of course). Although tested quite a bit, I would still consider the feature experimental. git-svn-id: file:///srv/svn/repos/haiku/trunk/current@9671 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- src/tools/jam/Jamfile | 34 + src/tools/jam/README.CHANGES | 24 + src/tools/jam/StatCacheServer.cpp | 1390 +++++++++++++++++++++++++++ src/tools/jam/StatCacheServer.h | 35 + src/tools/jam/StatCacheServerImpl.h | 266 +++++ src/tools/jam/beos_stat_cache.c | 221 +++++ src/tools/jam/beos_stat_cache.h | 15 + src/tools/jam/fileunix.c | 9 + 8 files changed, 1994 insertions(+) create mode 100644 src/tools/jam/StatCacheServer.cpp create mode 100644 src/tools/jam/StatCacheServer.h create mode 100644 src/tools/jam/StatCacheServerImpl.h create mode 100644 src/tools/jam/beos_stat_cache.c create mode 100644 src/tools/jam/beos_stat_cache.h diff --git a/src/tools/jam/Jamfile b/src/tools/jam/Jamfile index eb6c8350be..68d898d352 100644 --- a/src/tools/jam/Jamfile +++ b/src/tools/jam/Jamfile @@ -41,6 +41,7 @@ if $(PROFILE) if $(DEBUG) { CCFLAGS += -g ; + C++FLAGS += -g ; LINKFLAGS += -g ; LOCATE_TARGET = $(LOCATE_TARGET)/g ; } @@ -58,6 +59,16 @@ DEFINES += OPT_JAMFILE_CACHE_EXT ; # ### LOCAL CHANGE +### LOCAL CHANGE +# +# Include stat cache server under BeOS. +# +if $(OS) = BEOS { + DEFINES += OPT_STAT_CACHE_SERVER_EXT ; +} +# +### LOCAL CHANGE + # Do we know yacc? if $(YACC) { code += jamgram.y ; } @@ -84,6 +95,16 @@ code += hcache.c ; # ### LOCAL CHANGE +### LOCAL CHANGE +# +# Include stat cache server under BeOS. +# +if $(OS) = BEOS { + code += beos_stat_cache.c ; +} +# +### LOCAL CHANGE + # # How to build the compiled in jambase. # @@ -119,6 +140,19 @@ Library libjam.a : if $(BINDIR) { InstallBin $(BINDIR) : jam ; } +### LOCAL CHANGE +# +# Build stat cache server under BeOS. +# +if $(OS) = BEOS { + LINK on StatCacheServer = g++ ; + LINKLIBS on StatCacheServer + = [ on StatCacheServer return $(LINKLIBS) ] -lbe ; + Main StatCacheServer : StatCacheServer.cpp ; +} +# +### LOCAL CHANGE + # # Distribution making from here on out. # diff --git a/src/tools/jam/README.CHANGES b/src/tools/jam/README.CHANGES index 70518e14e3..7115e67e56 100644 --- a/src/tools/jam/README.CHANGES +++ b/src/tools/jam/README.CHANGES @@ -130,3 +130,27 @@ adjusted more or less to work with 2.5rc3. file is not performed. Setting the SEARCH and LOCATE variables does work as expected. + +* Stat Data and Directory Caching Server (BeOS only) + + Also an optimization for large build systems. Since the BeOS FS cache + is terrible, stat()ing targets to get their timestamp or see if they exist + at all, and reading directories usually happens on disk, since the data + from the previous run are already out of the cache, if the build system + is large enough. + + This change externalizes all stat()ing and directory reading into a + dedicated server process which caches the data, so that they can be + served from memory the next time they are requested. The server uses + the BeOS node monitoring to keep the data up to date. + + The feature particularly leverages the header and jamfile caching, since + after the first run the timestamps of the jamfiles and headers are + cached too, so that reading the jamfiles and performing the header + scanning doesn't require any disk accesses at all (besides reading the + cache files, of course). + + Drawbacks are that the first run of jam will be slower, mainly due to + the communication overhead with the server, and that the server consumes + memory to store the cached data. The server's memory footprint is quite + reasonable, though. diff --git a/src/tools/jam/StatCacheServer.cpp b/src/tools/jam/StatCacheServer.cpp new file mode 100644 index 0000000000..137b765b04 --- /dev/null +++ b/src/tools/jam/StatCacheServer.cpp @@ -0,0 +1,1390 @@ +// StatCacheServer.cpp + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "StatCacheServer.h" +#include "StatCacheServerImpl.h" + +//#define DBG(x) { x; } +#define DBG(x) +#define OUT printf + +static const int32 kMaxSymlinks = 32; + +// node monitor constants +static const int32 kDefaultNodeMonitorLimit = 4096; +static const int32 kNodeMonitorLimitIncrement = 512; + +// private BeOS syscall to set the node monitor slot limits +extern "C" int _kset_mon_limit_(int num); + +// get_dirent_size +static inline +int32 +get_dirent_size(const char *name) +{ + dirent *dummy = NULL; + int32 entrySize = (dummy->d_name + strlen(name) + 1) - (char*)dummy; + return (entrySize + 3) & ~0x3; +} + +// node_ref_hash +static inline +uint32 +node_ref_hash(dev_t device, ino_t node) +{ + uint32 hash = device; + hash = hash * 17 + (uint32)node; + hash = hash * 17 + (uint32)(node >> 32); + return hash; +} + +// string_hash +// +// from the Dragon Book: a slightly modified hashpjw() +static inline +uint32 +string_hash(const char *name) +{ + uint32 h = 0; + if (name) { + for (; *name; name++) { + uint32 g = h & 0xf0000000; + if (g) + h ^= g >> 24; + h = (h << 4) + *name; + } + } + return h; +} + +// NodeRefHash +size_t +NodeRefHash::operator()(const node_ref &nodeRef) const +{ + return node_ref_hash(nodeRef.device, nodeRef.node); +} + +// EntryRefHash +size_t +EntryRefHash::operator()(const entry_ref &entryRef) const +{ + uint32 hash = node_ref_hash(entryRef.device, entryRef.directory); + hash = hash * 17 + string_hash(entryRef.name); + return hash; +} + + +// #pragma mark - + +// constructor +Entry::Entry() + : Referencable(), + fParent(NULL), + fName(), + fNode(NULL), + fPrevious(NULL), + fNext(NULL) +{ +} + +// destructor +Entry::~Entry() +{ + SetNode(NULL); +} + +// SetTo +status_t +Entry::SetTo(Directory *parent, const char *name) +{ + fParent = parent; + fName = name; + return B_OK; +} + +// GetParent +Directory * +Entry::GetParent() const +{ + return fParent; +} + +// GetName +const char * +Entry::GetName() const +{ + return fName.c_str(); +} + +// SetNode +void +Entry::SetNode(Node *node) +{ + if (fNode != node) { + if (fNode) + fNode->RemoveReference(); + fNode = node; + if (fNode) { + fNode->AddReference(); + if (!fNode->GetEntry()) + fNode->SetEntry(this); + } + } +} + +// GetNode +Node * +Entry::GetNode() const +{ + return fNode; +} + +// SetPrevious +void +Entry::SetPrevious(Entry *entry) +{ + fPrevious = entry; +} + +// GetPrevious +Entry * +Entry::GetPrevious() const +{ + return fPrevious; +} + +// SetNext +void +Entry::SetNext(Entry *entry) +{ + fNext = entry; +} + +// GetNext +Entry * +Entry::GetNext() const +{ + return fNext; +} + +// GetEntryRef +entry_ref +Entry::GetEntryRef() const +{ + node_ref dirRef(fParent->GetNodeRef()); + return entry_ref(dirRef.device, dirRef.node, fName.c_str()); +} + +// GetPath +status_t +Entry::GetPath(string& path) +{ + if (!fParent) + return B_ERROR; + + // get directory path + status_t error = fParent->GetPath(path); + if (error != B_OK) + return error; + + // append the entry name + if (path[path.length() - 1] != '/') + path += '/'; + path += fName; + + return B_OK; +} + +// Unreferenced +void +Entry::Unreferenced() +{ + NodeManager::GetDefault()->EntryUnreferenced(this); +} + + +// #pragma mark - + +// constructor +Node::Node(const struct stat &st) + : Referencable(), + fEntry(NULL), + fStat(st), + fStatValid(false) +{ +} + +// destructor +Node::~Node() +{ + // stop watching the node + NodeManager::GetDefault()->StopWatching(this); + + SetEntry(NULL); +} + +// SetTo +status_t +Node::SetTo(Entry *entry) +{ + // start watching the node + status_t error = NodeManager::GetDefault()->StartWatching(this); + if (error != B_OK) + return error; + + SetEntry(entry); + + // update the stat + return UpdateStat(); +} + +// GetPath +status_t +Node::GetPath(string& path) +{ + if (this == NodeManager::GetDefault()->GetRootDirectory()) { + path = "/"; + return B_OK; + } + + if (!fEntry) + return B_ERROR; + return fEntry->GetPath(path); +} + +// GetStat +const struct stat & +Node::GetStat() const +{ + return fStat; +} + +// GetStat +status_t +Node::GetStat(struct stat *st) +{ + if (!fStatValid) { + status_t error = UpdateStat(); + if (error != B_OK) + return error; + } + *st = fStat; + return B_OK; +} + +// UpdateStat +status_t +Node::UpdateStat() +{ + // get path + string path; + status_t error = GetPath(path); + if (error != B_OK) + return error; + +DBG(OUT("disk access: lstat(): %s\n", path.c_str())); + + // read stat + if (lstat(path.c_str(), &fStat) < 0) + return errno; + fStatValid = true; + return B_OK; +} + +// MarkStatInvalid +void +Node::MarkStatInvalid() +{ + fStatValid = false; +} + +// SetEntry +void +Node::SetEntry(Entry *entry) +{ + if (entry != fEntry) { + if (fEntry) + fEntry->RemoveReference(); + fEntry = entry; + if (fEntry) + fEntry->AddReference(); + } +} + +// GetEntry +Entry * +Node::GetEntry() const +{ + return fEntry; +} + +// GetNodeRef +node_ref +Node::GetNodeRef() const +{ + node_ref nodeRef; + nodeRef.device = fStat.st_dev; + nodeRef.node = fStat.st_ino; + return nodeRef; +} + +// Unreferenced +void +Node::Unreferenced() +{ + NodeManager::GetDefault()->NodeUnreferenced(this); +} + + +// #pragma mark - + +// constructor +Directory::Directory(const struct stat &st) + : Node(st), + fFirstEntry(NULL), + fLastEntry(NULL), + fIsComplete(false) +{ +} + +// destructor +Directory::~Directory() +{ + while (Entry *entry = GetFirstEntry()) + RemoveEntry(entry); +} + +// SetTo +status_t +Directory::SetTo(Entry *entry) +{ + return Node::SetTo(entry); +} + +// FindEntry +status_t +Directory::FindEntry(const char *name, Entry **entry) +{ + entry_ref ref(fStat.st_dev, fStat.st_ino, name); + if (!fIsComplete) + return NodeManager::GetDefault()->CreateEntry(ref, entry); + *entry = NodeManager::GetDefault()->GetEntry(ref); + return (*entry ? B_OK : B_ENTRY_NOT_FOUND); +} + +// GetFirstEntry +Entry * +Directory::GetFirstEntry() const +{ + return fFirstEntry; +} + +// GetNextEntry +Entry * +Directory::GetNextEntry(Entry *entry) const +{ + return (entry ? entry->GetNext() : NULL); +} + +// ReadAllEntries +status_t +Directory::ReadAllEntries() +{ + if (fIsComplete) + return B_OK; + + // get the path + string path; + status_t error = GetPath(path); + if (error != B_OK) + return error; + +DBG(OUT("disk access: opendir(): %s\n", path.c_str())); + + // open the directory + DIR *dir = opendir(path.c_str()); + if (!dir) + return errno; + + // read the directory + while (dirent *entry = readdir(dir)) { + Entry *dummy; + FindEntry(entry->d_name, &dummy); + } + closedir(dir); + + fIsComplete = true; + + return B_OK; +} + +// IsComplete +bool +Directory::IsComplete() const +{ + return fIsComplete; +} + +// AddEntry +void +Directory::AddEntry(Entry *entry) +{ + if (fLastEntry) { + entry->SetPrevious(fLastEntry); + entry->SetNext(NULL); + fLastEntry->SetNext(entry); + fLastEntry = entry; + } else { + entry->SetPrevious(NULL); + entry->SetNext(NULL); + fFirstEntry = fLastEntry = entry; + } + entry->AddReference(); + + // the reference the "." entry has, shall be ignored + if (strcmp(entry->GetName(), ".") == 0) + fReferenceBaseCount++; +} + +// RemoveEntry +void +Directory::RemoveEntry(Entry *entry) +{ + if (entry->GetParent() != this) + return; + + // the reference the "." entry has, shall be ignored + if (strcmp(entry->GetName(), ".") == 0) + fReferenceBaseCount--; + + if (entry->GetPrevious()) + entry->GetPrevious()->SetNext(entry->GetNext()); + else + fFirstEntry = entry->GetNext(); + if (entry->GetNext()) + entry->GetNext()->SetPrevious(entry->GetPrevious()); + else + fLastEntry = entry->GetPrevious(); + entry->SetPrevious(NULL); + entry->SetNext(NULL); + entry->RemoveReference(); +} + + +// #pragma mark - + +// constructor +SymLink::SymLink(const struct stat &st) + : Node(st), + fTarget() +{ +} + +// destructor +SymLink::~SymLink() +{ +} + +// SetTo +status_t +SymLink::SetTo(Entry *entry) +{ + // node initialization + status_t error = Node::SetTo(entry); + if (error != B_OK) + return error; + + // get the entry path + string path; + error = entry->GetPath(path); + if (error != B_OK) + return error; + + // read the link + char target[B_PATH_NAME_LENGTH + 1]; + ssize_t bytesRead = readlink(path.c_str(), target, B_PATH_NAME_LENGTH); + if (bytesRead < 0) + return errno; + target[bytesRead] = '\0'; + fTarget = target; + return B_OK; +} + +// GetTarget +const char * +SymLink::GetTarget() const +{ + return fTarget.c_str(); +} + + +// #pragma mark - + +// destructor +NodeMonitor::NodeMonitor() + // higher priority and larger queue, since we must not miss update events + : BLooper("node monitor", B_DISPLAY_PRIORITY, 1000), + fCurrentNodeMonitorLimit(kDefaultNodeMonitorLimit), + fMessageCountSem(-1) +{ +} + +// destructor +NodeMonitor::~NodeMonitor() +{ +} + +// Init +status_t +NodeMonitor::Init() +{ + fMessageCountSem = create_sem(0, "nm message count"); + if (fMessageCountSem < 0) + return fMessageCountSem; + return B_OK; +} + +// MessageReceived +void +NodeMonitor::MessageReceived(BMessage *message) +{ + switch (message->what) { + case B_NODE_MONITOR: + DetachCurrentMessage(); + fMessageQueue.AddMessage(message); + release_sem(fMessageCountSem); + break; + default: + BLooper::MessageReceived(message); + } +} + +// StartWatching +status_t +NodeMonitor::StartWatching(Node *node) +{ + if (!node) + return B_BAD_VALUE; + uint32 flags = B_WATCH_STAT; + if (S_ISDIR(node->GetStat().st_mode)) + flags |= B_WATCH_DIRECTORY; + node_ref ref = node->GetNodeRef(); + status_t error = watch_node(&ref, flags, this); + // If starting to watch the node fail, we allocate more node + // monitoring slots and try again. + if (error != B_OK) { + fCurrentNodeMonitorLimit += kNodeMonitorLimitIncrement; + error = _kset_mon_limit_(fCurrentNodeMonitorLimit); + if (error == B_OK) + error = watch_node(&ref, flags, this); + } + return error; +} + +// StopWatching +status_t +NodeMonitor::StopWatching(Node *node) +{ + if (!node) + return B_BAD_VALUE; + node_ref ref = node->GetNodeRef(); + return watch_node(&ref, B_STOP_WATCHING, this); +} + +// GetNextMonitoringMessage +status_t +NodeMonitor::GetNextMonitoringMessage(BMessage **_message) +{ + // acquire the semaphore + status_t error = B_OK; + do { + error = acquire_sem(fMessageCountSem); + } while (error == B_INTERRUPTED); + if (error != B_OK) + return error; + + // get the message + BMessage *message = fMessageQueue.NextMessage(); + if (!message) + return B_ERROR; + *_message = message; + return B_OK; +} + + +// #pragma mark - + +// constructor +PathResolver::PathResolver() + : fSymLinkCounter(0) +{ +} + +// FindEntry +status_t +PathResolver::FindEntry(const char *path, bool traverse, Entry **_entry) +{ + return FindEntry(NULL, path, traverse, _entry); +} + +// FindEntry +status_t +PathResolver::FindEntry(Entry *entry, const char *path, bool traverse, + Entry **_entry) +{ + // we accept only absolute paths, if no entry was given + if (!path || (!entry && *path != '/')) + return B_BAD_VALUE; + + // get the root directory for absolute paths + if (*path == '/') { + entry = NodeManager::GetDefault()->GetRootDirectory()->GetEntry(); + // skip '/' + while (*path == '/') + path++; + } + + while (*path != '\0') { + // get path component + int componentLen; + if (char *nextSlash = strchr(path, '/')) + componentLen = nextSlash - path; + else + componentLen = strlen(path); + string component(path, componentLen); + path += componentLen; + + // resolve symlink + Node *node = entry->GetNode(); + if (SymLink *symlink = dynamic_cast(node)) { + status_t error = ResolveSymlink(symlink, &node); + if (error != B_OK) + return error; + } + + // find the entry + if (Directory *dir = dynamic_cast(node)) { + status_t error = dir->FindEntry(component.c_str(), &entry); + if (error != B_OK) + return error; + } else + return B_ENTRY_NOT_FOUND; + + // skip '/' + while (*path == '/') + path++; + } + + // traverse leaf symlink, if requested + if (traverse) { + status_t error = ResolveSymlink(entry, &entry); + if (error != B_OK) + return error; + } + + *_entry = entry; + return B_OK; +} + +// FindNode +status_t +PathResolver::FindNode(const char *path, bool traverse, Node **node) +{ + Entry *entry; + status_t error = FindEntry(path, traverse, &entry); + if (error != B_OK) + return error; + if (!entry->GetNode()) + return B_ENTRY_NOT_FOUND; + *node = entry->GetNode(); + return B_OK; +} + +// ResolveSymlink +status_t +PathResolver::ResolveSymlink(Node *node, Node **_node) +{ + Entry *entry; + status_t error = ResolveSymlink(node, &entry); + if (error != B_OK) + return error; + if (!entry->GetNode()) + return B_ENTRY_NOT_FOUND; + *_node = entry->GetNode(); + return B_OK; +} + +// ResolveSymlink +status_t +PathResolver::ResolveSymlink(Node *node, Entry **entry) +{ + return ResolveSymlink(node->GetEntry(), entry); +} + +// ResolveSymlink +status_t +PathResolver::ResolveSymlink(Entry *entry, Entry **_entry) +{ + if (!entry->GetNode()) + return B_ENTRY_NOT_FOUND; + + SymLink *symlink = dynamic_cast(entry->GetNode()); + if (!symlink) { + *_entry = entry; + return B_OK; + } + + if (fSymLinkCounter > kMaxSymlinks) + return B_LINK_LIMIT; + + const char *target = symlink->GetTarget(); + if (!target || !symlink->GetEntry() || !symlink->GetEntry()->GetParent() + || !symlink->GetEntry()->GetParent()->GetEntry()) + return B_ENTRY_NOT_FOUND; + + fSymLinkCounter++; + status_t error = FindEntry(symlink->GetEntry()->GetParent()->GetEntry(), + target, true, _entry); + fSymLinkCounter--; + return error; +} + + +// #pragma mark - + +// constructor +NodeManager::NodeManager() + : BLocker("node manager"), + fRootDirectory(NULL), + fNodeMonitor(NULL), + fNodeMonitoringProcessor(-1) +{ +} + +// destructor +NodeManager::~NodeManager() +{ + if (fNodeMonitor) { + fNodeMonitor->Lock(); + fNodeMonitor->Quit(); + } + if (fNodeMonitoringProcessor >= 0) { + int32 result; + wait_for_thread(fNodeMonitoringProcessor, &result); + } +} + +// GetDefault +NodeManager * +NodeManager::GetDefault() +{ + return &sManager; +} + +// Init +status_t +NodeManager::Init() +{ + // create the node monitor + fNodeMonitor = new NodeMonitor; + status_t error = fNodeMonitor->Init(); + if (error != B_OK) + return error; + fNodeMonitor->Run(); + + // spawn the node monitoring processor + fNodeMonitoringProcessor = spawn_thread(&_NodeMonitoringProcessorEntry, + "node monitoring processor", B_NORMAL_PRIORITY, this); + if (fNodeMonitoringProcessor < 0) + return fNodeMonitoringProcessor; + resume_thread(fNodeMonitoringProcessor); + + // get root dir stat + struct stat st; + if (lstat("/", &st) < 0) + return errno; + + // create the root node + node_ref nodeRef; + nodeRef.device = st.st_dev; + nodeRef.node = st.st_ino; + fRootDirectory = new Directory(st); + fNodes[nodeRef] = fRootDirectory; + + // create an entry pointing to the root node + entry_ref entryRef(st.st_dev, st.st_ino, "."); + Entry *entry = new Entry; + error = entry->SetTo(fRootDirectory, "."); + if (error != B_OK) + return error; + entry->SetNode(fRootDirectory); + fEntries[entryRef] = entry; + + // now we can initialize the root directory + error = fRootDirectory->SetTo(entry); + + return error; +} + +// GetRootDirectory +Directory * +NodeManager::GetRootDirectory() const +{ + return fRootDirectory; +} + +// GetNode +Node * +NodeManager::GetNode(const node_ref &nodeRef) +{ + NodeMap::iterator it = fNodes.find(nodeRef); + if (it == fNodes.end()) + return NULL; + return it->second; +} + +// GetEntry +Entry * +NodeManager::GetEntry(const entry_ref &entryRef) +{ + EntryMap::iterator it = fEntries.find(entryRef); + if (it == fEntries.end()) + return NULL; + return it->second; +} + +// CreateEntry +status_t +NodeManager::CreateEntry(const entry_ref &entryRef, Entry **_entry) +{ + Entry *entry = GetEntry(entryRef); + if (!entry) { + // entry does not yet exist -- create it + + // get the parent directory + node_ref parentDirRef; + parentDirRef.device = entryRef.device; + parentDirRef.node = entryRef.directory; + Directory *dir; + status_t error = CreateDirectory(parentDirRef, &dir); + if (error != B_OK) + return error; + + // if the directory hasn't created it, we need to do that now + entry = GetEntry(entryRef); + if (!entry) { + entry = new Entry; + error = entry->SetTo(dir, entryRef.name); + if (error != B_OK) { + delete entry; + return error; + } + + // get the entry's node + Node *node; + error = NodeManager::GetDefault()->_CreateNode(entry, &node); + if (error != B_OK) { + delete entry; + return error; + } + entry->SetNode(node); + node->RemoveReference(); + + // initialization successful: add the entry to the dir and to the + // entry map + dir->AddEntry(entry); + fEntries[entryRef] = entry; + entry->RemoveReference(); +DBG( +string path; +entry->GetPath(path); +OUT("entry created: `%s'\n", path.c_str()); +) + } + } + + *_entry = entry; + return B_OK; +} + +// CreateDirectory +status_t +NodeManager::CreateDirectory(const node_ref &nodeRef, Directory **_dir) +{ + Node *node = GetNode(nodeRef); + if (!node) { + // node not yet known -- load the directory + // get the full path + entry_ref entryRef(nodeRef.device, nodeRef.node, "."); + BPath path; + status_t error = path.SetTo(&entryRef); + if (error != B_OK) + return error; + + // find the node + error = PathResolver().FindNode(path.Path(), false, &node); + if (error != B_OK) + return error; + } + + // node found -- check, if it is a directory + Directory *dir = dynamic_cast(node); + if (!dir) + return B_NOT_A_DIRECTORY; + + *_dir = dir; + return B_OK; +} + +// RemoveEntry +void +NodeManager::RemoveEntry(Entry *entry) +{ + if (!entry) + return; + +DBG( +string path; +entry->GetPath(path); +OUT("entry removed: `%s'\n", path.c_str()); +) + + // get a temporary reference, so that the entry will not be deleted when + // we unset the node + entry->AddReference(); + + // remove from directory and node + if (entry->GetParent()) + entry->GetParent()->RemoveEntry(entry); + + // detach from node + Node *node = entry->GetNode(); + if (node) { + if (node->GetEntry() == entry) + node->SetEntry(NULL); + entry->SetNode(NULL); + } + + // surrender our temporary reference: now the entry should be unreference + entry->RemoveReference(); +} + +// MoveEntry +void +NodeManager::MoveEntry(Entry *entry, const entry_ref &newRef) +{ + // get the target directory + node_ref newDirRef; + newDirRef.device = newRef.device; + newDirRef.node = newRef.directory; + Directory *newDir = dynamic_cast(GetNode(newDirRef)); + if (!newDir) { + // target directory unknown -- simply remove the entry + RemoveEntry(entry); + return; + } + + // If the directory and/or the name changed, we remove the old entry and + // create a new one. + if (newDir != entry->GetParent() + || strcmp(newRef.name, entry->GetName()) != 0) { + // get a temporary reference to the node, so it won't be unnecessarily + // deleted + Node *node = entry->GetNode(); + if (node) + node->AddReference(); + + RemoveEntry(entry); + CreateEntry(newRef, &entry); + + if (node) + node->RemoveReference(); + } +} + +// EntryUnreferenced +void +NodeManager::EntryUnreferenced(Entry *entry) +{ +DBG(OUT("NodeManager::EntryUnreferenced(%p): (%p, `%s')\n", entry, entry->GetParent(), entry->GetName())); + // remove entry from the map and delete it + if (fEntries.erase(entry->GetEntryRef()) > 0) + delete entry; +} + +// NodeUnreferenced +void +NodeManager::NodeUnreferenced(Node *node) +{ +DBG(OUT("NodeManager::NodeUnreferenced(%p): entry: %p\n", node, node->GetEntry())); + // remove node from the map and delete it + if (fNodes.erase(node->GetNodeRef()) > 0) + delete node; +} + +// StartWatching +status_t +NodeManager::StartWatching(Node *node) +{ + return fNodeMonitor->StartWatching(node); +} + +// StopWatching +status_t +NodeManager::StopWatching(Node *node) +{ + return fNodeMonitor->StopWatching(node); +} + +// _NodeMonitoringProcessorEntry +int32 +NodeManager::_NodeMonitoringProcessorEntry(void *data) +{ + return ((NodeManager*)data)->_NodeMonitoringProcessor(); +} + +// _NodeMonitoringProcessor +int32 +NodeManager::_NodeMonitoringProcessor() +{ + BMessage *message; + while (fNodeMonitor->GetNextMonitoringMessage(&message) == B_OK) { + int32 opcode; + if (message->FindInt32("opcode", &opcode) == B_OK) { + BAutolock _(this); + switch (opcode) { + case B_ENTRY_CREATED: + _EntryCreated(message); + break; + case B_ENTRY_REMOVED: + _EntryRemoved(message); + break; + case B_ENTRY_MOVED: + _EntryMoved(message); + break; + case B_STAT_CHANGED: + _StatChanged(message); + break; + } + } + delete message; + } +} + +// _CreateNode +// +// On success the caller gets a reference to the node, they are required to +// surrender, if done with the node. +status_t +NodeManager::_CreateNode(Entry *entry, Node **_node) +{ + // get the path + string path; + status_t error = entry->GetPath(path); + if (error != B_OK) + return error; + +DBG(OUT("disk access: lstat(): %s\n", path.c_str())); + + // read the stat + struct stat st; + if (lstat(path.c_str(), &st) < 0) + return errno; + + // check, if the node does already exist + node_ref nodeRef; + nodeRef.device = st.st_dev; + nodeRef.node = st.st_ino; + Node *node = GetNode(nodeRef); + + if (node) { + node->AddReference(); + } else { + // node does not yet exist -- create it + if (S_ISLNK(st.st_mode)) + node = new SymLink(st); + else if (S_ISDIR(st.st_mode)) + node = new Directory(st); + else + node = new Node(st); + + error = node->SetTo(entry); + if (error != B_OK) { + delete node; + return error; + } + + fNodes[nodeRef] = node; + } + + *_node = node; + return B_OK; +} + +// _EntryCreated +void +NodeManager::_EntryCreated(BMessage *message) +{ + // get the info + node_ref dirNodeRef; + const char* name; + if (message->FindInt32("device", &dirNodeRef.device) != B_OK + || message->FindInt64("directory", &dirNodeRef.node) != B_OK +// || message->FindInt64("node", &nodeID) != B_OK + || message->FindString("name", &name) != B_OK) { + return; + } + + // get the directory + Node *node = NodeManager::GetDefault()->GetNode(dirNodeRef); + Directory *dir = dynamic_cast(node); + if (!dir) + return; + + // add the entry, if the directory is complete + if (dir->IsComplete()) { + Entry *entry; + if (dir->FindEntry(name, &entry) != B_OK) { + entry_ref ref(dirNodeRef.device, dirNodeRef.node, name); + NodeManager::GetDefault()->CreateEntry(ref, &entry); + } + } +} + +// _EntryRemoved +void +NodeManager::_EntryRemoved(BMessage *message) +{ + // get the info + node_ref nodeRef; + const char* name; + if (message->FindInt32("device", &nodeRef.device) != B_OK +// || message->FindInt64("directory", &nodeRef.node) != B_OK + || message->FindInt64("node", &nodeRef.node) != B_OK) { + return; + } + + // get the node + Node *node = NodeManager::GetDefault()->GetNode(nodeRef); + if (!node) + return; + + // remove it + NodeManager::GetDefault()->RemoveEntry(node->GetEntry()); +} + +// _EntryMoved +void +NodeManager::_EntryMoved(BMessage *message) +{ + // get the info + node_ref nodeRef; + ino_t newDirID; + const char* name; + if (message->FindInt32("device", &nodeRef.device) != B_OK +// || message->FindInt64("from directory", &fromDirectoryID) != B_OK + || message->FindInt64("to directory", &newDirID) != B_OK + || message->FindInt64("node", &nodeRef.node) != B_OK + || message->FindString("name", &name) != B_OK) { + return; + } + + // get the node + Node *node = NodeManager::GetDefault()->GetNode(nodeRef); + if (!node) + return; + + // move it + entry_ref newRef(nodeRef.device, newDirID, name); + NodeManager::GetDefault()->MoveEntry(node->GetEntry(), newRef); +} + +// _StatChanged +void +NodeManager::_StatChanged(BMessage *message) +{ + // get the node ref + node_ref nodeRef; + if (message->FindInt32("device", &nodeRef.device) != B_OK + || message->FindInt64("node", &nodeRef.node)) { + return; + } + + // get the node + Node *node = GetNode(nodeRef); + if (!node) + return; + + node->MarkStatInvalid(); +} + +// sManager +NodeManager NodeManager::sManager; + + +// #pragma mark - + +// read_request +static +status_t +read_request(port_id port, stat_cache_request &request) +{ + status_t error = B_OK; + bool done = false; + do { + int32 code; + ssize_t bytesRead = read_port(port, &code, &request, + sizeof(stat_cache_request)); + if (bytesRead < 0) { + error = bytesRead; + done = (error != B_INTERRUPTED); + } else if (bytesRead + < ((stat_cache_request*)NULL)->path + 2 - (char*)NULL) { +DBG(OUT("request too short: %ld\n", bytesRead)); + error = B_ERROR; + } else { + done = true; + error = B_OK; + } + } while (!done); + return error; +} + +// handle_stat_request +static +status_t +handle_stat_request(stat_cache_request &request) +{ +DBG(OUT("handle_stat_request(): `%s'\n", request.path)); + stat_cache_stat_reply reply; + + // get the node + PathResolver resolver; + Node *node; + reply.error = resolver.FindNode(request.path, true, &node); + + // get the stat + if (reply.error == B_OK) + reply.error = node->GetStat(&reply.st); +DBG(OUT(" -> `%s'\n", strerror(reply.error))); + + // send the reply + return write_port(request.replyPort, 0, &reply, sizeof(reply)); +} + +// handle_read_dir_request +static +status_t +handle_read_dir_request(stat_cache_request &request) +{ +DBG(OUT("handle_read_dir_request(): `%s'\n", request.path)); + // get the directory + PathResolver resolver; + Node *node = NULL; + status_t error = resolver.FindNode(request.path, true, &node); + Directory *dir = dynamic_cast(node); + if (error == B_OK && !dir) + error = B_NOT_A_DIRECTORY; + + // read all entries + if (error == B_OK) + error = dir->ReadAllEntries(); + + // compute the reply size + int32 replySize = sizeof(stat_cache_readdir_reply); + int32 entryCount = 0; + if (error == B_OK) { + for (Entry *entry = dir->GetFirstEntry(); + entry; + entry = dir->GetNextEntry(entry)) { + replySize += get_dirent_size(entry->GetName()); + entryCount++; + } + } + + // allocate a reply + stat_cache_readdir_reply *reply + = (stat_cache_readdir_reply*)new uint8[replySize]; + reply->error = error; + reply->entryCount = entryCount; + + // copy the entries into the reply + if (error == B_OK) { + uint8 *buffer = reply->buffer; + for (Entry *entry = dir->GetFirstEntry(); + entry; + entry = dir->GetNextEntry(entry)) { + // get the required info + int32 entrySize = get_dirent_size(entry->GetName()); + node_ref parentNodeRef(entry->GetParent()->GetNodeRef()); + node_ref nodeRef(entry->GetNode()->GetNodeRef()); + + // fill in the dirent + dirent *ent = (dirent*)buffer; + ent->d_pdev = parentNodeRef.device; + ent->d_pino = parentNodeRef.node; + ent->d_dev = nodeRef.device; + ent->d_ino = nodeRef.node; + ent->d_reclen = entrySize; + strcpy(ent->d_name, entry->GetName()); + + buffer += entrySize; + } + } + +DBG(OUT(" -> entryCount: %ld, error: `%s'\n", reply->entryCount, strerror(reply->error))); + // send the reply + error = write_port(request.replyPort, 0, reply, replySize); + delete[] (uint8*)reply; + return error; +} + +// main +int +main() +{ + // init node manager + status_t error = NodeManager::GetDefault()->Init(); + if (error != B_OK) { + fprintf(stderr, "Failed to init node manager: %s\n", strerror(error)); + return 1; + } + + // create the request port + port_id port = create_port(1, STAT_CACHE_SERVER_PORT_NAME); + if (port < 0) { + fprintf(stderr, "Failed to create request port: %s\n", strerror(port)); + return 1; + } + + stat_cache_request request; + while (read_request(port, request) == B_OK) { + BAutolock _(NodeManager::GetDefault()); + switch (request.command) { + case STAT_CACHE_COMMAND_STAT: + handle_stat_request(request); + break; + case STAT_CACHE_COMMAND_READDIR: + handle_read_dir_request(request); + break; + default: + fprintf(stderr, "Unknown command: %ld\n", request.command); + break; + } + } + + // delete the request port + delete_port(port); + return 0; +} diff --git a/src/tools/jam/StatCacheServer.h b/src/tools/jam/StatCacheServer.h new file mode 100644 index 0000000000..1881829fe4 --- /dev/null +++ b/src/tools/jam/StatCacheServer.h @@ -0,0 +1,35 @@ +// StatCacheServer.h + +#ifndef STAT_CACHE_SERVER_H +#define STAT_CACHE_SERVER_H + +#include + +// common definitions used by server and client + +#define STAT_CACHE_SERVER_PORT_NAME "stat_cache_server_port" + +enum { + STAT_CACHE_COMMAND_STAT = 0, + STAT_CACHE_COMMAND_READDIR = 1, +}; + +typedef struct stat_cache_request { + port_id replyPort; + int32 command; + char path[B_PATH_NAME_LENGTH]; +} stat_cache_request; + +typedef struct stat_cache_stat_reply { + status_t error; + struct stat st; +} stat_cache_stat_reply; + +typedef struct stat_cache_readdir_reply { + status_t error; + int32 entryCount; + void *clientData; // used by the client only + uint8 buffer[1]; +} stat_cache_readdir_reply; + +#endif // STAT_CACHE_SERVER_H diff --git a/src/tools/jam/StatCacheServerImpl.h b/src/tools/jam/StatCacheServerImpl.h new file mode 100644 index 0000000000..e059e8875b --- /dev/null +++ b/src/tools/jam/StatCacheServerImpl.h @@ -0,0 +1,266 @@ +// StatCacheServerImpl.h + +#ifndef STAT_CACHE_SERVER_IMPL_H +#define STAT_CACHE_SERVER_IMPL_H + +#include +#include + +#include +#include +#include +#include +#include +#include + +class Directory; +class Entry; +class Node; +class SymLink; + +// NodeRefHash +struct NodeRefHash +{ + size_t operator()(const node_ref &nodeRef) const; +}; + +// EntryRefHash +struct EntryRefHash +{ + size_t operator()(const entry_ref &entryRef) const; +}; + +// Referencable +class Referencable { +public: + Referencable() + : fReferenceCount(1), + fReferenceBaseCount(0) + { + } + + virtual ~Referencable() + { + } + + void AddReference() + { + fReferenceCount++; + } + + bool RemoveReference() + { + if (--fReferenceCount <= fReferenceBaseCount) { + Unreferenced(); + return true; + } + return false; + } + + int32 CountReferences() const + { + return fReferenceCount; + } + +protected: + virtual void Unreferenced() {}; + +protected: + int32 fReferenceCount; + int32 fReferenceBaseCount; + bool fDeleteWhenUnreferenced; +}; + +// Entry +class Entry : public Referencable { +public: + Entry(); + ~Entry(); + + status_t SetTo(Directory *parent, const char *name); + + Directory *GetParent() const; + + const char *GetName() const; + + void SetNode(Node *node); + Node *GetNode() const; + + void SetPrevious(Entry *entry); + Entry *GetPrevious() const; + void SetNext(Entry *entry); + Entry *GetNext() const; + + entry_ref GetEntryRef() const; + status_t GetPath(string& path); + +protected: + virtual void Unreferenced(); + +private: + Directory *fParent; + string fName; + Node *fNode; + Entry *fPrevious; + Entry *fNext; +}; + +// Node +class Node : public Referencable { +public: + Node(const struct stat &st); + virtual ~Node(); + + virtual status_t SetTo(Entry *entry); + + status_t GetPath(string& path); + + const struct stat &GetStat() const; + status_t GetStat(struct stat *st); + status_t UpdateStat(); + void MarkStatInvalid(); + + void SetEntry(Entry *entry); + Entry *GetEntry() const; + + node_ref GetNodeRef() const; + +protected: + virtual void Unreferenced(); + +protected: + Entry *fEntry; + struct stat fStat; + bool fStatValid; +}; + +// Directory +class Directory : public Node { +public: + Directory(const struct stat &st); + ~Directory(); + + virtual status_t SetTo(Entry *entry); + + status_t FindEntry(const char *name, Entry **entry); + Entry *GetFirstEntry() const; + Entry *GetNextEntry(Entry *entry) const; + + status_t ReadAllEntries(); + + bool IsComplete() const; + + void AddEntry(Entry *entry); + void RemoveEntry(Entry *entry); + +private: + Entry *fFirstEntry; + Entry *fLastEntry; + bool fIsComplete; +}; + +// SymLink +class SymLink : public Node { +public: + SymLink(const struct stat &st); + ~SymLink(); + + virtual status_t SetTo(Entry *entry); + + const char *GetTarget() const; + +private: + string fTarget; +}; + +// NodeMonitor +class NodeMonitor : public BLooper { +public: + NodeMonitor(); + virtual ~NodeMonitor(); + + status_t Init(); + + virtual void MessageReceived(BMessage *message); + + status_t StartWatching(Node *node); + status_t StopWatching(Node *node); + + status_t GetNextMonitoringMessage(BMessage **message); + +private: + int32 fCurrentNodeMonitorLimit; + BMessageQueue fMessageQueue; + sem_id fMessageCountSem; +}; + +// PathResolver +class PathResolver { +public: + PathResolver(); + + status_t FindEntry(const char *path, bool traverse, Entry **_entry); + status_t FindEntry(Entry *entry, const char *path, bool traverse, + Entry **_entry); + status_t FindNode(const char *path, bool traverse, Node **node); + + status_t ResolveSymlink(Node *node, Node **_node); + status_t ResolveSymlink(Node *node, Entry **entry); + status_t ResolveSymlink(Entry *entry, Entry **_entry); + +private: + int32 fSymLinkCounter; +}; + +// NodeManager +class NodeManager : public BLocker { +public: + NodeManager(); + ~NodeManager(); + + static NodeManager *GetDefault(); + + status_t Init(); + + Directory *GetRootDirectory() const; + + Node *GetNode(const node_ref &nodeRef); + Entry *GetEntry(const entry_ref &entryRef); + + status_t CreateEntry(const entry_ref &entryRef, Entry **_entry); + status_t CreateDirectory(const node_ref &nodeRef, Directory **_dir); + + void RemoveEntry(Entry *entry); + void MoveEntry(Entry *entry, const entry_ref &newRef); + + void EntryUnreferenced(Entry *entry); + void NodeUnreferenced(Node *node); + + status_t StartWatching(Node *node); + status_t StopWatching(Node *node); + +private: + static int32 _NodeMonitoringProcessorEntry(void *data); + int32 _NodeMonitoringProcessor(); + + status_t _CreateNode(Entry *entry, Node **_node); + + void _EntryCreated(BMessage *message); + void _EntryRemoved(BMessage *message); + void _EntryMoved(BMessage *message); + void _StatChanged(BMessage *message); + +private: + typedef hash_map EntryMap; + typedef hash_map NodeMap; + + EntryMap fEntries; + NodeMap fNodes; + Directory *fRootDirectory; + NodeMonitor *fNodeMonitor; + thread_id fNodeMonitoringProcessor; + + static NodeManager sManager; +}; + +#endif // STAT_CACHE_SERVER_IMPL_H diff --git a/src/tools/jam/beos_stat_cache.c b/src/tools/jam/beos_stat_cache.c new file mode 100644 index 0000000000..59d908d836 --- /dev/null +++ b/src/tools/jam/beos_stat_cache.c @@ -0,0 +1,221 @@ +// beos_stat_cache.c + +#include +#include +#include + +#include + +#include "beos_stat_cache.h" +#include "pathsys.h" +#include "StatCacheServer.h" + +#define SET_ERRNO_AND_RETURN(error) { \ + if ((error) == B_OK) \ + return 0; \ + errno = (error); \ + return -1; \ +} + +// get_server_port +static +port_id +get_server_port() +{ + static port_id id = -1; + static bool initialized = false; + if (!initialized) { + id = find_port(STAT_CACHE_SERVER_PORT_NAME); + initialized = true; + } + return id; +} + +// get_reply_port +static +port_id +get_reply_port() +{ + static port_id id = -1; + if (id < 0) + id = create_port(1, "stat cache reply port"); + return id; +} + +// send_request +static +status_t +send_request(int32 command, const char *path) +{ + port_id requestPort = get_server_port(); + port_id replyPort = get_reply_port(); + stat_cache_request request; + int requestSize; + + // get request port + if (requestPort < 0) + return requestPort; + // get reply port + if (replyPort < 0) + return replyPort; + // normalize the path + if (!path || !normalize_path(path, request.path, sizeof(request.path))) + return B_BAD_VALUE; + requestSize = (request.path + strlen(request.path) + 1) - (char*)&request; + // send request + request.replyPort = replyPort; + request.command = command; + return write_port(requestPort, 0, &request, requestSize); +} + +// receive_reply +static +status_t +receive_reply(void **_reply, int32 *_replySize, void *buffer, int32 replySize) +{ + port_id replyPort = get_reply_port(); + ssize_t bytesRead; + void *reply; + int32 code; + + // get reply port + if (replyPort < 0) + return replyPort; + + // get the reply size + if (!buffer) { + replySize = port_buffer_size(replyPort); + if (replySize < 0) + return replySize; + } + + // allocate reply + if (buffer) { + reply = buffer; + } else { + reply = malloc(replySize); + if (!reply) + return B_NO_MEMORY; + } + + // read the reply + bytesRead = read_port(replyPort, &code, reply, replySize); + if (bytesRead < 0) { + if (!buffer) + free(reply); + return bytesRead; + } + if (bytesRead != replySize) { + if (!buffer) + free(reply); + return B_ERROR; + } + + if (_reply) + *_reply = reply; + if (_replySize) + *_replySize = replySize; + return B_OK; +} + +// beos_stat_cache_stat +int +beos_stat_cache_stat(const char *filename, struct stat *st) +{ + stat_cache_stat_reply reply; + status_t error; + + // fall back to standard, if there is no server + if (get_server_port() < 0) + return stat(filename, st); + + // send the request + error = send_request(STAT_CACHE_COMMAND_STAT, filename); + if (error != B_OK) + SET_ERRNO_AND_RETURN(error); + + // get the reply + error = receive_reply(NULL, NULL, &reply, sizeof(reply)); + if (error != B_OK) + error = reply.error; + if (error != B_OK) + SET_ERRNO_AND_RETURN(error); + + *st = reply.st; + return 0; +} + +// beos_stat_cache_opendir +DIR * +beos_stat_cache_opendir(const char *dirName) +{ + stat_cache_readdir_reply *reply; + int32 replySize; + status_t error; + + // fall back to standard, if there is no server + if (get_server_port() < 0) + return opendir(dirName); + + // send the request + error = send_request(STAT_CACHE_COMMAND_READDIR, dirName); + if (error != B_OK) { + errno = error; + return NULL; + } + + // get the reply + error = receive_reply((void**)&reply, &replySize, NULL, 0); + if (error != B_OK) + error = reply->error; + if (error != B_OK) { + free(reply); + errno = error; + return NULL; + } + + reply->clientData = reply->buffer; + + // a bit ugly, but anyway... + return (DIR*)reply; +} + +// beos_stat_cache_readdir +struct dirent * +beos_stat_cache_readdir(DIR *dir) +{ + stat_cache_readdir_reply *reply; + struct dirent *entry; + + // fall back to standard, if there is no server + if (get_server_port() < 0) + return readdir(dir); + + reply = (stat_cache_readdir_reply*)dir; + if (reply->entryCount == 0) + return NULL; + + entry = (struct dirent*)reply->clientData; + + // get the next entry + if (--reply->entryCount > 0) + reply->clientData = (uint8*)entry + entry->d_reclen; + + return entry; +} + +// beos_stat_cache_closedir +int +beos_stat_cache_closedir(DIR *dir) +{ + stat_cache_readdir_reply *reply; + + // fall back to standard, if there is no server + if (get_server_port() < 0) + return closedir(dir); + + reply = (stat_cache_readdir_reply*)dir; + free(reply); + return 0; +} + diff --git a/src/tools/jam/beos_stat_cache.h b/src/tools/jam/beos_stat_cache.h new file mode 100644 index 0000000000..b64726c24a --- /dev/null +++ b/src/tools/jam/beos_stat_cache.h @@ -0,0 +1,15 @@ +// beos_stat_cache.h + +#ifndef BEOS_STAT_CACHE_H +#define BEOS_STAT_CACHE_H + +#include +#include + +int beos_stat_cache_stat(const char *filename, struct stat *st); + +DIR* beos_stat_cache_opendir(const char *dirName); +struct dirent *beos_stat_cache_readdir(DIR *dir); +int beos_stat_cache_closedir(DIR *dir); + +#endif BEOS_STAT_CACHE_H diff --git a/src/tools/jam/fileunix.c b/src/tools/jam/fileunix.c index 41127a293a..c310b393af 100644 --- a/src/tools/jam/fileunix.c +++ b/src/tools/jam/fileunix.c @@ -99,6 +99,15 @@ struct ar_hdr /* archive file member header - printable ascii */ # include # endif +# ifdef OPT_STAT_CACHE_SERVER_EXT +# include "beos_stat_cache.h" +inline int stat(const char *filename, struct stat *st) + { return beos_stat_cache_stat(filename, st); } +# define opendir beos_stat_cache_opendir +# define readdir beos_stat_cache_readdir +# define closedir beos_stat_cache_closedir +# endif + /* * file_dirscan() - scan a directory for files */