From 522c2f19d458245474fcaf5b0254e0099906b117 Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Mon, 7 Dec 2009 15:42:08 +0000 Subject: [PATCH] * Added a simple mechanism to wait for events to VMCache. WaitForPageEvents() waits for certain events on a given page, NotifyPageEvents() wakes up waiting threads respectively. * Used the new feature instead of condition variables for waiting on busy pages. We save publishing and unpublishing of a condition variable whenever a page is marked busy. There's only something to do, if there's at least one thread waiting in the list of the respective cache. The general assumption is that this is only rarely the case and even if it happens, there should be only very few threads. * Added an apparently missing notification in cache_io(). At least I didn't see the reason for it not being there. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@34537 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/private/kernel/vm/VMCache.h | 17 ++++++ src/system/kernel/cache/file_cache.cpp | 32 ++++------- src/system/kernel/vm/VMCache.cpp | 73 ++++++++++++++++++++++---- src/system/kernel/vm/vm.cpp | 23 ++++---- src/system/kernel/vm/vm_page.cpp | 6 +-- 5 files changed, 101 insertions(+), 50 deletions(-) diff --git a/headers/private/kernel/vm/VMCache.h b/headers/private/kernel/vm/VMCache.h index 1a4682565a..15870b9352 100644 --- a/headers/private/kernel/vm/VMCache.h +++ b/headers/private/kernel/vm/VMCache.h @@ -27,6 +27,11 @@ enum { CACHE_TYPE_NULL }; +enum { + PAGE_EVENT_NOT_BUSY = 0x01 // page not busy anymore +}; + + struct VMCachePagesTreeDefinition { typedef page_num_t KeyType; typedef vm_page NodeType; @@ -82,6 +87,12 @@ public: void ReleaseRefAndUnlock() { ReleaseRefLocked(); Unlock(); } + void WaitForPageEvents(vm_page* page, uint32 events, + bool relock); + void NotifyPageEvents(vm_page* page, uint32 events) + { if (fPageEventWaiters != NULL) + _NotifyPageEvents(page, events); } + vm_page* LookupPage(off_t offset); void InsertPage(vm_page* page, off_t offset); void RemovePage(vm_page* page); @@ -153,6 +164,11 @@ public: #endif private: + struct PageEventWaiter; + +private: + void _NotifyPageEvents(vm_page* page, uint32 events); + inline bool _IsMergeable() const; void _MergeWithOnlyConsumer(); @@ -161,6 +177,7 @@ private: private: int32 fRefCount; mutex fLock; + PageEventWaiter* fPageEventWaiters; }; diff --git a/src/system/kernel/cache/file_cache.cpp b/src/system/kernel/cache/file_cache.cpp index 784298d3f2..21b7754d44 100644 --- a/src/system/kernel/cache/file_cache.cpp +++ b/src/system/kernel/cache/file_cache.cpp @@ -114,7 +114,6 @@ PrecacheIO::PrecacheIO(file_cache_ref* ref, off_t offset, size_t size) fRef(ref), fCache(ref->cache), fPages(NULL), - fBusyConditions(NULL), fVecs(NULL), fOffset(offset), fVecCount(0), @@ -128,7 +127,6 @@ PrecacheIO::PrecacheIO(file_cache_ref* ref, off_t offset, size_t size) PrecacheIO::~PrecacheIO() { delete[] fPages; - delete[] fBusyConditions; delete[] fVecs; fCache->ReleaseRefLocked(); } @@ -144,10 +142,6 @@ PrecacheIO::Prepare() if (fPages == NULL) return B_NO_MEMORY; - fBusyConditions = new(std::nothrow) ConditionVariable[fPageCount]; - if (fBusyConditions == NULL) - return B_NO_MEMORY; - fVecs = new(std::nothrow) iovec[fPageCount]; if (fVecs == NULL) return B_NO_MEMORY; @@ -159,7 +153,6 @@ PrecacheIO::Prepare() if (page == NULL) break; - fBusyConditions[i].Publish(page, "page"); fCache->InsertPage(page, fOffset + pos); add_to_iovec(fVecs, fVecCount, fPageCount, @@ -170,7 +163,7 @@ PrecacheIO::Prepare() if (i != fPageCount) { // allocating pages failed while (i-- > 0) { - fBusyConditions[i].Unpublish(); + fCache->NotifyPageEvents(fPages[i], PAGE_EVENT_NOT_BUSY); fCache->RemovePage(fPages[i]); vm_page_set_state(fPages[i], PAGE_STATE_FREE); } @@ -215,12 +208,12 @@ PrecacheIO::IOFinished(status_t status, bool partialTransfer, } fPages[i]->state = PAGE_STATE_ACTIVE; - fBusyConditions[i].Unpublish(); + fCache->NotifyPageEvents(fPages[i], PAGE_EVENT_NOT_BUSY); } // Free pages after failed I/O for (uint32 i = pagesTransferred; i < fPageCount; i++) { - fBusyConditions[i].Unpublish(); + fCache->NotifyPageEvents(fPages[i], PAGE_EVENT_NOT_BUSY); fCache->RemovePage(fPages[i]); vm_page_set_state(fPages[i], PAGE_STATE_FREE); } @@ -383,7 +376,6 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, size_t numBytes = PAGE_ALIGN(pageOffset + bufferSize); vm_page* pages[MAX_IO_VECS]; - ConditionVariable busyConditions[MAX_IO_VECS]; int32 pageIndex = 0; // allocate pages for the cache and mark them busy @@ -393,8 +385,6 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, if (page == NULL) panic("no more pages!"); - busyConditions[pageIndex - 1].Publish(page, "page"); - cache->InsertPage(page, offset + pos); add_to_iovec(vecs, vecCount, MAX_IO_VECS, @@ -417,7 +407,7 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, cache->Lock(); for (int32 i = 0; i < pageIndex; i++) { - busyConditions[i].Unpublish(); + cache->NotifyPageEvents(pages[i], PAGE_EVENT_NOT_BUSY); cache->RemovePage(pages[i]); vm_page_set_state(pages[i], PAGE_STATE_FREE); } @@ -448,7 +438,7 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, for (int32 i = pageIndex; i-- > 0;) { pages[i]->state = PAGE_STATE_ACTIVE; - busyConditions[i].Unpublish(); + cache->NotifyPageEvents(pages[i], PAGE_EVENT_NOT_BUSY); } return B_OK; @@ -504,7 +494,6 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, vm_page* pages[MAX_IO_VECS]; int32 pageIndex = 0; status_t status = B_OK; - ConditionVariable busyConditions[MAX_IO_VECS]; // ToDo: this should be settable somewhere bool writeThrough = false; @@ -518,7 +507,6 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, // in cache_io() vm_page* page = pages[pageIndex++] = vm_page_allocate_page( PAGE_STATE_FREE, true); - busyConditions[pageIndex - 1].Publish(page, "page"); ref->cache->InsertPage(page, offset + pos); @@ -616,7 +604,7 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, // make the pages accessible in the cache for (int32 i = pageIndex; i-- > 0;) { - busyConditions[i].Unpublish(); + ref->cache->NotifyPageEvents(pages[i], PAGE_EVENT_NOT_BUSY); if (writeThrough) pages[i]->state = PAGE_STATE_ACTIVE; @@ -796,11 +784,7 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, return status; if (page->state == PAGE_STATE_BUSY) { - ConditionVariableEntry entry; - entry.Add(page); - locker.Unlock(); - entry.Wait(); - locker.Lock(); + cache->WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, true); continue; } } @@ -845,6 +829,8 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, page->state = oldPageState; if (doWrite && page->state != PAGE_STATE_MODIFIED) vm_page_set_state(page, PAGE_STATE_MODIFIED); + + cache->NotifyPageEvents(page, PAGE_EVENT_NOT_BUSY); } if (bytesLeft <= bytesInPage) { diff --git a/src/system/kernel/vm/VMCache.cpp b/src/system/kernel/vm/VMCache.cpp index 1ae013c4a9..9266c7a3d1 100644 --- a/src/system/kernel/vm/VMCache.cpp +++ b/src/system/kernel/vm/VMCache.cpp @@ -7,6 +7,7 @@ * Distributed under the terms of the NewOS License. */ + #include #include @@ -45,6 +46,14 @@ static mutex sCacheListLock = MUTEX_INITIALIZER("global VMCache list"); // The lock is also needed when the debug feature is disabled. +struct VMCache::PageEventWaiter { + struct thread* thread; + PageEventWaiter* next; + vm_page* page; + uint32 events; +}; + + #if VM_CACHE_TRACING namespace VMCacheTracing { @@ -406,6 +415,7 @@ VMCache::Init(uint32 cacheType) scan_skip = 0; page_count = 0; type = cacheType; + fPageEventWaiters = NULL; #if DEBUG_CACHE_LIST mutex_lock(&sCacheListLock); @@ -622,6 +632,37 @@ VMCache::RemovePage(vm_page* page) } +/*! Waits until one or more events happened for a given page which belongs to + this cache. + The cache must be locked. It will be unlocked by the method. \a relock + specifies whether the method shall re-lock the cache before returning. + \param page The page for which to wait. + \param events The mask of events the caller is interested in. + \param relock If \c true, the cache will be locked when returning, + otherwise it won't be locked. +*/ +void +VMCache::WaitForPageEvents(vm_page* page, uint32 events, bool relock) +{ + PageEventWaiter waiter; + waiter.thread = thread_get_current_thread(); + waiter.next = fPageEventWaiters; + waiter.page = page; + waiter.events = events; + + fPageEventWaiters = &waiter; + + thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_OTHER, + "cache page events"); + + Unlock(); + thread_block(); + + if (relock) + Lock(); +} + + /*! Makes this case the source of the \a consumer cache, and adds the \a consumer to its list. This also grabs a reference to the source cache. @@ -778,11 +819,7 @@ VMCache::Resize(off_t newSize) // this will notify the writer to free the page } else { // wait for page to become unbusy - ConditionVariableEntry entry; - entry.Add(page); - Unlock(); - entry.Wait(); - Lock(); + WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, true); // restart from the start of the list it = pages.GetIterator(newPageCount, true, true); @@ -824,11 +861,7 @@ VMCache::FlushAndRemoveAllPages() vm_page* page = it.Next();) { if (page->state == PAGE_STATE_BUSY) { // wait for page to become unbusy - ConditionVariableEntry entry; - entry.Add(page); - Unlock(); - entry.Wait(); - Lock(); + WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, true); // restart from the start of the list it = pages.GetIterator(); @@ -969,6 +1002,26 @@ VMCache::ReleaseStoreRef() } +/*! Wakes up threads waiting for page events. + \param page The page for which events occurred. + \param events The mask of events that occurred. +*/ +void +VMCache::_NotifyPageEvents(vm_page* page, uint32 events) +{ + PageEventWaiter** it = &fPageEventWaiters; + while (PageEventWaiter* waiter = *it) { + if (waiter->page == page && (waiter->events & events) != 0) { + // remove from list and unblock + *it = waiter->next; + InterruptsSpinLocker threadsLocker(gThreadSpinlock); + thread_unblock_locked(waiter->thread, B_OK); + } else + it = &waiter->next; + } +} + + /*! Merges the given cache with its only consumer. The caller must hold both the cache's and the consumer's lock. The method will unlock the consumer lock. diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index f4a76cbc9e..d0a38b03c8 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -3574,7 +3574,7 @@ public: return fBottomCache; } - void Unlock() + void Unlock(VMCache* exceptCache = NULL) { if (fTopCache == NULL) return; @@ -3582,7 +3582,8 @@ public: VMCache* cache = fTopCache; while (cache != NULL) { VMCache* nextCache = cache->source; - cache->ReleaseRefAndUnlock(); + if (cache != exceptCache) + cache->ReleaseRefAndUnlock(); if (cache == fBottomCache) break; @@ -3637,11 +3638,11 @@ struct PageFaultContext { cacheChainLocker.SetTo(topCache); } - void UnlockAll() + void UnlockAll(VMCache* exceptCache = NULL) { topCache = NULL; addressSpaceLocker.Unlock(); - cacheChainLocker.Unlock(); + cacheChainLocker.Unlock(exceptCache); } }; @@ -3678,10 +3679,9 @@ fault_get_page(PageFaultContext& context) } // page must be busy -- wait for it to become unbusy - ConditionVariableEntry entry; - entry.Add(page); - context.UnlockAll(); - entry.Wait(); + context.UnlockAll(cache); + cache->ReleaseRefLocked(); + cache->WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, false); // restart the whole process context.restart = true; @@ -3699,9 +3699,6 @@ fault_get_page(PageFaultContext& context) page = vm_page_allocate_page(PAGE_STATE_FREE, true); cache->InsertPage(page, context.cacheOffset); - ConditionVariable busyCondition; - busyCondition.Publish(page, "page"); - // We need to unlock all caches and the address space while reading // the page in. Keep a reference to the cache around. cache->AcquireRefLocked(); @@ -3722,7 +3719,7 @@ fault_get_page(PageFaultContext& context) dprintf("reading page from cache %p returned: %s!\n", cache, strerror(status)); - busyCondition.Unpublish(); + cache->NotifyPageEvents(page, PAGE_EVENT_NOT_BUSY); cache->RemovePage(page); vm_page_set_state(page, PAGE_STATE_FREE); @@ -3732,7 +3729,7 @@ fault_get_page(PageFaultContext& context) // mark the page unbusy again page->state = PAGE_STATE_ACTIVE; - busyCondition.Unpublish(); + cache->NotifyPageEvents(page, PAGE_EVENT_NOT_BUSY); // Since we needed to unlock everything temporarily, the area // situation might have changed. So we need to restart the whole diff --git a/src/system/kernel/vm/vm_page.cpp b/src/system/kernel/vm/vm_page.cpp index eebb0816ce..6cf9b81393 100644 --- a/src/system/kernel/vm/vm_page.cpp +++ b/src/system/kernel/vm/vm_page.cpp @@ -1038,7 +1038,6 @@ private: bool fDequeuedPage; bool fIsActive; int fOldPageState; - ConditionVariable fBusyCondition; }; @@ -1073,8 +1072,6 @@ PageWriteWrapper::SetTo(vm_page* page, bool dequeuedPage) fOldPageState = fPage->state; fPage->state = PAGE_STATE_BUSY; fPage->busy_writing = true; - - fBusyCondition.Publish(fPage, "page"); } @@ -1143,7 +1140,8 @@ PageWriteWrapper::Done(status_t result) fPage->busy_writing = false; } - fBusyCondition.Unpublish(); + + fCache->NotifyPageEvents(fPage, PAGE_EVENT_NOT_BUSY); fIsActive = false; }