From 5484890096d122519e722d0a31b478eaa7e8ea1d Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Sun, 3 Jan 2010 02:43:32 +0000 Subject: [PATCH] * Moved the unused vnode management to a new file. Well the few variables used for it that is. * The main cause for the heavy contention of the unused vnodes mutex was that relatively few vnodes are actually used for a longer time. Mainly those are the volume roots, mmap()ed files, and the files opened by programs. A good deal of nodes -- particularly directories -- are just referenced for a very short time, e.g. to resolve a path to a contained entry. This caused those nodes to be added to and removed from the unused vnodes list very frequently, thus resulting in a high contention of the mutex guarding it. To address the problem I've introduced an approximation of a set of "hot" vnodes, i.e. vnodes that have recently been marked unused. They are stored in an array that by means of an r/w locker and atomic operations can most of the time be accessed concurrently. Whenever it gets full, it is flushed to the actual unused vnodes list. * dec_vnode_ref_count(): No longer check the unused vnode count every time. The called new vnode_unused() does only from time to time and returns when the caller is expected to free some of the unused vnodes. As a side effect this also fixes a bug I previously introduced: The unused vnode to be freed was marked busy without being locked first. The -j8 Haiku image test build shows that the changes reduce the contention of the unused vnode list mutex to virtually zero without introducing any significant contention of the new r/w lock. The VMCache lock contention also seems to be decreased somewhat, which is probably not that surprising considering that the page writer acquires/releases vnode references with the cache lock held. The "pages" lock takes over even more contention, now causing more than 100000 waits per second. The total build time reduction is about 4.5%. Kernel time drops more than 10%. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@34866 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- src/system/kernel/fs/Vnode.h | 42 ++++++ src/system/kernel/fs/unused_vnodes.h | 207 +++++++++++++++++++++++++++ src/system/kernel/fs/vfs.cpp | 83 ++++++----- 3 files changed, 289 insertions(+), 43 deletions(-) create mode 100644 src/system/kernel/fs/unused_vnodes.h diff --git a/src/system/kernel/fs/Vnode.h b/src/system/kernel/fs/Vnode.h index c1204e4e34..637036ea9c 100644 --- a/src/system/kernel/fs/Vnode.h +++ b/src/system/kernel/fs/Vnode.h @@ -45,6 +45,12 @@ public: inline bool IsUnpublished() const; inline void SetUnpublished(bool unpublished); + inline bool IsUnused() const; + inline void SetUnused(bool unused); + + inline bool IsHot() const; + inline void SetHot(bool hot); + inline uint32 Type() const; inline void SetType(uint32 type); @@ -59,6 +65,8 @@ private: static const uint32 kFlagsBusy = 0x00000004; static const uint32 kFlagsRemoved = 0x00000008; static const uint32 kFlagsUnpublished = 0x00000010; + static const uint32 kFlagsUnused = 0x00000020; + static const uint32 kFlagsHot = 0x00000040; static const uint32 kFlagsType = 0xfffff000; static const uint32 kBucketCount = 32; @@ -142,6 +150,40 @@ vnode::SetUnpublished(bool unpublished) } +bool +vnode::IsUnused() const +{ + return (fFlags & kFlagsUnused) != 0; +} + + +void +vnode::SetUnused(bool unused) +{ + if (unused) + atomic_or(&fFlags, kFlagsUnused); + else + atomic_and(&fFlags, ~kFlagsUnused); +} + + +bool +vnode::IsHot() const +{ + return (fFlags & kFlagsHot) != 0; +} + + +void +vnode::SetHot(bool hot) +{ + if (hot) + atomic_or(&fFlags, kFlagsHot); + else + atomic_and(&fFlags, ~kFlagsHot); +} + + uint32 vnode::Type() const { diff --git a/src/system/kernel/fs/unused_vnodes.h b/src/system/kernel/fs/unused_vnodes.h new file mode 100644 index 0000000000..ca866debab --- /dev/null +++ b/src/system/kernel/fs/unused_vnodes.h @@ -0,0 +1,207 @@ +/* + * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef UNUSED_VNODES_H +#define UNUSED_VNODES_H + + +#include + +#include +#include + +#include + +#include "Vnode.h" + + +const static uint32 kMaxUnusedVnodes = 8192; + // This is the maximum number of unused vnodes that the system + // will keep around (weak limit, if there is enough memory left, + // they won't get flushed even when hitting that limit). + // It may be chosen with respect to the available memory or enhanced + // by some timestamp/frequency heurism. + + +/*! \brief Guards sUnusedVnodeList and sUnusedVnodes. + + Innermost lock. Must not be held when acquiring any other lock. +*/ +static mutex sUnusedVnodesLock = MUTEX_INITIALIZER("unused vnodes"); +static list sUnusedVnodeList; +static vuint32 sUnusedVnodes = 0; + +static const int32 kMaxHotVnodes = 1024; +static rw_lock sHotVnodesLock = RW_LOCK_INITIALIZER("hot vnodes"); +static Vnode* sHotVnodes[kMaxHotVnodes]; +static vint32 sNextHotVnodeIndex = 0; + +static const int32 kUnusedVnodesCheckInterval = 64; +static vint32 sUnusedVnodesCheckCount = 0; + + +/*! Must be called with sHotVnodesLock write-locked. +*/ +static void +flush_hot_vnodes_locked() +{ + MutexLocker unusedLocker(sUnusedVnodesLock); + + int32 count = std::min((int32)sNextHotVnodeIndex, kMaxHotVnodes); + for (int32 i = 0; i < count; i++) { + Vnode* vnode = sHotVnodes[i]; + if (vnode == NULL) + continue; + + if (vnode->IsHot()) { + if (vnode->IsUnused()) { + list_add_item(&sUnusedVnodeList, vnode); + sUnusedVnodes++; + } + vnode->SetHot(false); + } + + sHotVnodes[i] = NULL; + } + + unusedLocker.Unlock(); + + sNextHotVnodeIndex = 0; +} + + + +/*! To be called when the vnode's ref count drops to 0. + Must be called with sVnodeLock at least read-locked and the vnode locked. + \param vnode The vnode. + \return \c true, if the caller should trigger unused vnode freeing. +*/ +static bool +vnode_unused(Vnode* vnode) +{ + ReadLocker hotReadLocker(sHotVnodesLock); + + vnode->SetUnused(true); + + bool result = false; + int32 checkCount = atomic_add(&sUnusedVnodesCheckCount, 1); + if (checkCount == kUnusedVnodesCheckInterval) { + uint32 unusedCount = sUnusedVnodes; + if (unusedCount > kMaxUnusedVnodes + && low_resource_state( + B_KERNEL_RESOURCE_PAGES | B_KERNEL_RESOURCE_MEMORY) + != B_NO_LOW_RESOURCE) { + // there are too many unused vnodes -- tell the caller to free the + // oldest ones + result = true; + } else { + // nothing urgent -- reset the counter and re-check then + atomic_set(&sUnusedVnodesCheckCount, 0); + } + } + + // nothing to do, if the node is already hot + if (vnode->IsHot()) + return result; + + // no -- enter it + int32 index = atomic_add(&sNextHotVnodeIndex, 1); + if (index < kMaxHotVnodes) { + vnode->SetHot(true); + sHotVnodes[index] = vnode; + return result; + } + + // the array is full -- it has to be emptied + hotReadLocker.Unlock(); + WriteLocker hotWriteLocker(sHotVnodesLock); + + // unless someone was faster than we were, we have to flush the array + if (sNextHotVnodeIndex >= kMaxHotVnodes) + flush_hot_vnodes_locked(); + + // enter the vnode + index = sNextHotVnodeIndex++; + vnode->SetHot(true); + sHotVnodes[index] = vnode; + + return result; +} + + +/*! To be called when the vnode's ref count is changed from 0 to 1. + Must be called with sVnodeLock at least read-locked and the vnode locked. + \param vnode The vnode. +*/ +static void +vnode_used(Vnode* vnode) +{ + ReadLocker hotReadLocker(sHotVnodesLock); + + if (!vnode->IsUnused()) + return; + + vnode->SetUnused(false); + + if (!vnode->IsHot()) { + MutexLocker unusedLocker(sUnusedVnodesLock); + list_remove_item(&sUnusedVnodeList, vnode); + sUnusedVnodes--; + } +} + + +/*! To be called when the vnode's is about to be freed. + Must be called with sVnodeLock at least read-locked and the vnode locked. + \param vnode The vnode. +*/ +static void +vnode_to_be_freed(Vnode* vnode) +{ + ReadLocker hotReadLocker(sHotVnodesLock); + + if (vnode->IsHot()) { + // node is hot -- remove it from the array +// TODO: Maybe better completely flush the array while at it? + int32 count = sNextHotVnodeIndex; + count = std::min(count, kMaxHotVnodes); + for (int32 i = 0; i < count; i++) { + if (sHotVnodes[i] == vnode) { + sHotVnodes[i] = NULL; + break; + } + } + } else if (vnode->IsUnused()) { + MutexLocker unusedLocker(sUnusedVnodesLock); + list_remove_item(&sUnusedVnodeList, vnode); + sUnusedVnodes--; + } + + vnode->SetUnused(false); +} + + +static inline void +flush_hot_vnodes() +{ + WriteLocker hotWriteLocker(sHotVnodesLock); + flush_hot_vnodes_locked(); +} + + +static inline void +unused_vnodes_check_started() +{ + atomic_set(&sUnusedVnodesCheckCount, kUnusedVnodesCheckInterval + 1); +} + + +static inline void +unused_vnodes_check_done() +{ + atomic_set(&sUnusedVnodesCheckCount, 0); +} + + +#endif // UNUSED_VNODES_H diff --git a/src/system/kernel/fs/vfs.cpp b/src/system/kernel/fs/vfs.cpp index 05b4437507..621defc50e 100644 --- a/src/system/kernel/fs/vfs.cpp +++ b/src/system/kernel/fs/vfs.cpp @@ -56,6 +56,7 @@ #include "EntryCache.h" #include "fifo.h" #include "IORequest.h" +#include "unused_vnodes.h" #include "Vnode.h" #include "../cache/vnode_store.h" @@ -104,13 +105,6 @@ #endif -const static uint32 kMaxUnusedVnodes = 8192; - // This is the maximum number of unused vnodes that the system - // will keep around (weak limit, if there is enough memory left, - // they won't get flushed even when hitting that limit). - // It may be chosen with respect to the available memory or enhanced - // by some timestamp/frequency heurism. - const static size_t kMaxPathLength = 65536; // The absolute maximum path length (for getcwd() - this is not depending // on PATH_MAX @@ -263,17 +257,9 @@ static rw_lock sVnodeLock = RW_LOCK_INITIALIZER("vfs_vnode_lock"); */ static mutex sIOContextRootLock = MUTEX_INITIALIZER("io_context::root lock"); -/*! \brief Guards sUnusedVnodeList and sUnusedVnodes. - - Innermost lock. Must not be held when acquiring any other lock. -*/ -static mutex sUnusedVnodesLock = MUTEX_INITIALIZER("unused vnodes"); - #define VNODE_HASH_TABLE_SIZE 1024 static hash_table* sVnodeTable; -static list sUnusedVnodeList; -static uint32 sUnusedVnodes = 0; static struct vnode* sRoot; #define MOUNTS_HASH_TABLE_SIZE 16 @@ -286,6 +272,8 @@ mode_t __gUmask = 022; /* function declarations */ +static void free_unused_vnodes(); + // file descriptor operation prototypes static status_t file_read(struct file_descriptor* descriptor, off_t pos, void* buffer, size_t* _bytes); @@ -1053,33 +1041,24 @@ dec_vnode_ref_count(struct vnode* vnode, bool alwaysFree, bool reenter) panic("dec_vnode_ref_count: called on busy vnode %p\n", vnode); bool freeNode = false; + bool freeUnusedNodes = false; // Just insert the vnode into an unused list if we don't need // to delete it if (vnode->IsRemoved() || alwaysFree) { + vnode_to_be_freed(vnode); vnode->SetBusy(true); freeNode = true; - } else { - MutexLocker unusedVnodesLocker(sUnusedVnodesLock); - list_add_item(&sUnusedVnodeList, vnode); - if (++sUnusedVnodes > kMaxUnusedVnodes - && low_resource_state( - B_KERNEL_RESOURCE_PAGES | B_KERNEL_RESOURCE_MEMORY) - != B_NO_LOW_RESOURCE) { - // there are too many unused vnodes so we free the oldest one - // TODO: evaluate this mechanism - vnode = (struct vnode*)list_remove_head_item(&sUnusedVnodeList); - vnode->SetBusy(true); - freeNode = true; - sUnusedVnodes--; - } - } + } else + freeUnusedNodes = vnode_unused(vnode); nodeLocker.Unlock(); locker.Unlock(); if (freeNode) free_vnode(vnode, reenter); + else if (freeUnusedNodes) + free_unused_vnodes(); return B_OK; } @@ -1178,9 +1157,7 @@ restart: if (vnode) { if (vnode->ref_count == 0) { // this vnode has been unused before - MutexLocker unusedVnodesLocker(sUnusedVnodesLock); - list_remove_item(&sUnusedVnodeList, vnode); - sUnusedVnodes--; + vnode_used(vnode); } inc_vnode_ref_count(vnode); @@ -1269,9 +1246,16 @@ put_vnode(struct vnode* vnode) static void -vnode_low_resource_handler(void* /*data*/, uint32 resources, int32 level) +free_unused_vnodes(int32 level) { - TRACE(("vnode_low_resource_handler(level = %ld)\n", level)); + unused_vnodes_check_started(); + + if (level == B_NO_LOW_RESOURCE) { + unused_vnodes_check_done(); + return; + } + + flush_hot_vnodes(); // determine how many nodes to free uint32 count = 1; @@ -1279,8 +1263,6 @@ vnode_low_resource_handler(void* /*data*/, uint32 resources, int32 level) MutexLocker unusedVnodesLocker(sUnusedVnodesLock); switch (level) { - case B_NO_LOW_RESOURCE: - return; case B_LOW_RESOURCE_NOTE: count = sUnusedVnodes / 100; break; @@ -1336,6 +1318,25 @@ vnode_low_resource_handler(void* /*data*/, uint32 resources, int32 level) dec_vnode_ref_count(vnode, true, false); // this should free the vnode when it's still unused } + + unused_vnodes_check_done(); +} + + +static void +free_unused_vnodes() +{ + free_unused_vnodes( + low_resource_state(B_KERNEL_RESOURCE_PAGES | B_KERNEL_RESOURCE_MEMORY)); +} + + +static void +vnode_low_resource_handler(void* /*data*/, uint32 resources, int32 level) +{ + TRACE(("vnode_low_resource_handler(level = %ld)\n", level)); + + free_unused_vnodes(level); } @@ -7292,9 +7293,7 @@ fs_unmount(char* path, dev_t mountID, uint32 flags, bool kernel) if (vnode->ref_count == 0) { // this vnode has been unused before - MutexLocker unusedVnodesLocker(sUnusedVnodesLock); - list_remove_item(&sUnusedVnodeList, vnode); - sUnusedVnodes--; + vnode_used(vnode); } } @@ -7402,9 +7401,7 @@ fs_sync(dev_t device) if (vnode->ref_count == 0) { // this vnode has been unused before - MutexLocker unusedVnodesLocker(sUnusedVnodesLock); - list_remove_item(&sUnusedVnodeList, vnode); - sUnusedVnodes--; + vnode_used(vnode); } inc_vnode_ref_count(vnode);