diff --git a/headers/private/kernel/vm/VMCache.h b/headers/private/kernel/vm/VMCache.h index 1a4682565a..15870b9352 100644 --- a/headers/private/kernel/vm/VMCache.h +++ b/headers/private/kernel/vm/VMCache.h @@ -27,6 +27,11 @@ enum { CACHE_TYPE_NULL }; +enum { + PAGE_EVENT_NOT_BUSY = 0x01 // page not busy anymore +}; + + struct VMCachePagesTreeDefinition { typedef page_num_t KeyType; typedef vm_page NodeType; @@ -82,6 +87,12 @@ public: void ReleaseRefAndUnlock() { ReleaseRefLocked(); Unlock(); } + void WaitForPageEvents(vm_page* page, uint32 events, + bool relock); + void NotifyPageEvents(vm_page* page, uint32 events) + { if (fPageEventWaiters != NULL) + _NotifyPageEvents(page, events); } + vm_page* LookupPage(off_t offset); void InsertPage(vm_page* page, off_t offset); void RemovePage(vm_page* page); @@ -153,6 +164,11 @@ public: #endif private: + struct PageEventWaiter; + +private: + void _NotifyPageEvents(vm_page* page, uint32 events); + inline bool _IsMergeable() const; void _MergeWithOnlyConsumer(); @@ -161,6 +177,7 @@ private: private: int32 fRefCount; mutex fLock; + PageEventWaiter* fPageEventWaiters; }; diff --git a/src/system/kernel/cache/file_cache.cpp b/src/system/kernel/cache/file_cache.cpp index 784298d3f2..21b7754d44 100644 --- a/src/system/kernel/cache/file_cache.cpp +++ b/src/system/kernel/cache/file_cache.cpp @@ -114,7 +114,6 @@ PrecacheIO::PrecacheIO(file_cache_ref* ref, off_t offset, size_t size) fRef(ref), fCache(ref->cache), fPages(NULL), - fBusyConditions(NULL), fVecs(NULL), fOffset(offset), fVecCount(0), @@ -128,7 +127,6 @@ PrecacheIO::PrecacheIO(file_cache_ref* ref, off_t offset, size_t size) PrecacheIO::~PrecacheIO() { delete[] fPages; - delete[] fBusyConditions; delete[] fVecs; fCache->ReleaseRefLocked(); } @@ -144,10 +142,6 @@ PrecacheIO::Prepare() if (fPages == NULL) return B_NO_MEMORY; - fBusyConditions = new(std::nothrow) ConditionVariable[fPageCount]; - if (fBusyConditions == NULL) - return B_NO_MEMORY; - fVecs = new(std::nothrow) iovec[fPageCount]; if (fVecs == NULL) return B_NO_MEMORY; @@ -159,7 +153,6 @@ PrecacheIO::Prepare() if (page == NULL) break; - fBusyConditions[i].Publish(page, "page"); fCache->InsertPage(page, fOffset + pos); add_to_iovec(fVecs, fVecCount, fPageCount, @@ -170,7 +163,7 @@ PrecacheIO::Prepare() if (i != fPageCount) { // allocating pages failed while (i-- > 0) { - fBusyConditions[i].Unpublish(); + fCache->NotifyPageEvents(fPages[i], PAGE_EVENT_NOT_BUSY); fCache->RemovePage(fPages[i]); vm_page_set_state(fPages[i], PAGE_STATE_FREE); } @@ -215,12 +208,12 @@ PrecacheIO::IOFinished(status_t status, bool partialTransfer, } fPages[i]->state = PAGE_STATE_ACTIVE; - fBusyConditions[i].Unpublish(); + fCache->NotifyPageEvents(fPages[i], PAGE_EVENT_NOT_BUSY); } // Free pages after failed I/O for (uint32 i = pagesTransferred; i < fPageCount; i++) { - fBusyConditions[i].Unpublish(); + fCache->NotifyPageEvents(fPages[i], PAGE_EVENT_NOT_BUSY); fCache->RemovePage(fPages[i]); vm_page_set_state(fPages[i], PAGE_STATE_FREE); } @@ -383,7 +376,6 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, size_t numBytes = PAGE_ALIGN(pageOffset + bufferSize); vm_page* pages[MAX_IO_VECS]; - ConditionVariable busyConditions[MAX_IO_VECS]; int32 pageIndex = 0; // allocate pages for the cache and mark them busy @@ -393,8 +385,6 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, if (page == NULL) panic("no more pages!"); - busyConditions[pageIndex - 1].Publish(page, "page"); - cache->InsertPage(page, offset + pos); add_to_iovec(vecs, vecCount, MAX_IO_VECS, @@ -417,7 +407,7 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, cache->Lock(); for (int32 i = 0; i < pageIndex; i++) { - busyConditions[i].Unpublish(); + cache->NotifyPageEvents(pages[i], PAGE_EVENT_NOT_BUSY); cache->RemovePage(pages[i]); vm_page_set_state(pages[i], PAGE_STATE_FREE); } @@ -448,7 +438,7 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, for (int32 i = pageIndex; i-- > 0;) { pages[i]->state = PAGE_STATE_ACTIVE; - busyConditions[i].Unpublish(); + cache->NotifyPageEvents(pages[i], PAGE_EVENT_NOT_BUSY); } return B_OK; @@ -504,7 +494,6 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, vm_page* pages[MAX_IO_VECS]; int32 pageIndex = 0; status_t status = B_OK; - ConditionVariable busyConditions[MAX_IO_VECS]; // ToDo: this should be settable somewhere bool writeThrough = false; @@ -518,7 +507,6 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, // in cache_io() vm_page* page = pages[pageIndex++] = vm_page_allocate_page( PAGE_STATE_FREE, true); - busyConditions[pageIndex - 1].Publish(page, "page"); ref->cache->InsertPage(page, offset + pos); @@ -616,7 +604,7 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, // make the pages accessible in the cache for (int32 i = pageIndex; i-- > 0;) { - busyConditions[i].Unpublish(); + ref->cache->NotifyPageEvents(pages[i], PAGE_EVENT_NOT_BUSY); if (writeThrough) pages[i]->state = PAGE_STATE_ACTIVE; @@ -796,11 +784,7 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, return status; if (page->state == PAGE_STATE_BUSY) { - ConditionVariableEntry entry; - entry.Add(page); - locker.Unlock(); - entry.Wait(); - locker.Lock(); + cache->WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, true); continue; } } @@ -845,6 +829,8 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, page->state = oldPageState; if (doWrite && page->state != PAGE_STATE_MODIFIED) vm_page_set_state(page, PAGE_STATE_MODIFIED); + + cache->NotifyPageEvents(page, PAGE_EVENT_NOT_BUSY); } if (bytesLeft <= bytesInPage) { diff --git a/src/system/kernel/vm/VMCache.cpp b/src/system/kernel/vm/VMCache.cpp index 1ae013c4a9..9266c7a3d1 100644 --- a/src/system/kernel/vm/VMCache.cpp +++ b/src/system/kernel/vm/VMCache.cpp @@ -7,6 +7,7 @@ * Distributed under the terms of the NewOS License. */ + #include #include @@ -45,6 +46,14 @@ static mutex sCacheListLock = MUTEX_INITIALIZER("global VMCache list"); // The lock is also needed when the debug feature is disabled. +struct VMCache::PageEventWaiter { + struct thread* thread; + PageEventWaiter* next; + vm_page* page; + uint32 events; +}; + + #if VM_CACHE_TRACING namespace VMCacheTracing { @@ -406,6 +415,7 @@ VMCache::Init(uint32 cacheType) scan_skip = 0; page_count = 0; type = cacheType; + fPageEventWaiters = NULL; #if DEBUG_CACHE_LIST mutex_lock(&sCacheListLock); @@ -622,6 +632,37 @@ VMCache::RemovePage(vm_page* page) } +/*! Waits until one or more events happened for a given page which belongs to + this cache. + The cache must be locked. It will be unlocked by the method. \a relock + specifies whether the method shall re-lock the cache before returning. + \param page The page for which to wait. + \param events The mask of events the caller is interested in. + \param relock If \c true, the cache will be locked when returning, + otherwise it won't be locked. +*/ +void +VMCache::WaitForPageEvents(vm_page* page, uint32 events, bool relock) +{ + PageEventWaiter waiter; + waiter.thread = thread_get_current_thread(); + waiter.next = fPageEventWaiters; + waiter.page = page; + waiter.events = events; + + fPageEventWaiters = &waiter; + + thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_OTHER, + "cache page events"); + + Unlock(); + thread_block(); + + if (relock) + Lock(); +} + + /*! Makes this case the source of the \a consumer cache, and adds the \a consumer to its list. This also grabs a reference to the source cache. @@ -778,11 +819,7 @@ VMCache::Resize(off_t newSize) // this will notify the writer to free the page } else { // wait for page to become unbusy - ConditionVariableEntry entry; - entry.Add(page); - Unlock(); - entry.Wait(); - Lock(); + WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, true); // restart from the start of the list it = pages.GetIterator(newPageCount, true, true); @@ -824,11 +861,7 @@ VMCache::FlushAndRemoveAllPages() vm_page* page = it.Next();) { if (page->state == PAGE_STATE_BUSY) { // wait for page to become unbusy - ConditionVariableEntry entry; - entry.Add(page); - Unlock(); - entry.Wait(); - Lock(); + WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, true); // restart from the start of the list it = pages.GetIterator(); @@ -969,6 +1002,26 @@ VMCache::ReleaseStoreRef() } +/*! Wakes up threads waiting for page events. + \param page The page for which events occurred. + \param events The mask of events that occurred. +*/ +void +VMCache::_NotifyPageEvents(vm_page* page, uint32 events) +{ + PageEventWaiter** it = &fPageEventWaiters; + while (PageEventWaiter* waiter = *it) { + if (waiter->page == page && (waiter->events & events) != 0) { + // remove from list and unblock + *it = waiter->next; + InterruptsSpinLocker threadsLocker(gThreadSpinlock); + thread_unblock_locked(waiter->thread, B_OK); + } else + it = &waiter->next; + } +} + + /*! Merges the given cache with its only consumer. The caller must hold both the cache's and the consumer's lock. The method will unlock the consumer lock. diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index f4a76cbc9e..d0a38b03c8 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -3574,7 +3574,7 @@ public: return fBottomCache; } - void Unlock() + void Unlock(VMCache* exceptCache = NULL) { if (fTopCache == NULL) return; @@ -3582,7 +3582,8 @@ public: VMCache* cache = fTopCache; while (cache != NULL) { VMCache* nextCache = cache->source; - cache->ReleaseRefAndUnlock(); + if (cache != exceptCache) + cache->ReleaseRefAndUnlock(); if (cache == fBottomCache) break; @@ -3637,11 +3638,11 @@ struct PageFaultContext { cacheChainLocker.SetTo(topCache); } - void UnlockAll() + void UnlockAll(VMCache* exceptCache = NULL) { topCache = NULL; addressSpaceLocker.Unlock(); - cacheChainLocker.Unlock(); + cacheChainLocker.Unlock(exceptCache); } }; @@ -3678,10 +3679,9 @@ fault_get_page(PageFaultContext& context) } // page must be busy -- wait for it to become unbusy - ConditionVariableEntry entry; - entry.Add(page); - context.UnlockAll(); - entry.Wait(); + context.UnlockAll(cache); + cache->ReleaseRefLocked(); + cache->WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, false); // restart the whole process context.restart = true; @@ -3699,9 +3699,6 @@ fault_get_page(PageFaultContext& context) page = vm_page_allocate_page(PAGE_STATE_FREE, true); cache->InsertPage(page, context.cacheOffset); - ConditionVariable busyCondition; - busyCondition.Publish(page, "page"); - // We need to unlock all caches and the address space while reading // the page in. Keep a reference to the cache around. cache->AcquireRefLocked(); @@ -3722,7 +3719,7 @@ fault_get_page(PageFaultContext& context) dprintf("reading page from cache %p returned: %s!\n", cache, strerror(status)); - busyCondition.Unpublish(); + cache->NotifyPageEvents(page, PAGE_EVENT_NOT_BUSY); cache->RemovePage(page); vm_page_set_state(page, PAGE_STATE_FREE); @@ -3732,7 +3729,7 @@ fault_get_page(PageFaultContext& context) // mark the page unbusy again page->state = PAGE_STATE_ACTIVE; - busyCondition.Unpublish(); + cache->NotifyPageEvents(page, PAGE_EVENT_NOT_BUSY); // Since we needed to unlock everything temporarily, the area // situation might have changed. So we need to restart the whole diff --git a/src/system/kernel/vm/vm_page.cpp b/src/system/kernel/vm/vm_page.cpp index eebb0816ce..6cf9b81393 100644 --- a/src/system/kernel/vm/vm_page.cpp +++ b/src/system/kernel/vm/vm_page.cpp @@ -1038,7 +1038,6 @@ private: bool fDequeuedPage; bool fIsActive; int fOldPageState; - ConditionVariable fBusyCondition; }; @@ -1073,8 +1072,6 @@ PageWriteWrapper::SetTo(vm_page* page, bool dequeuedPage) fOldPageState = fPage->state; fPage->state = PAGE_STATE_BUSY; fPage->busy_writing = true; - - fBusyCondition.Publish(fPage, "page"); } @@ -1143,7 +1140,8 @@ PageWriteWrapper::Done(status_t result) fPage->busy_writing = false; } - fBusyCondition.Unpublish(); + + fCache->NotifyPageEvents(fPage, PAGE_EVENT_NOT_BUSY); fIsActive = false; }