diff --git a/headers/private/kernel/vm/vm_page.h b/headers/private/kernel/vm/vm_page.h index 00e892d6f0..8f7d474ec8 100644 --- a/headers/private/kernel/vm/vm_page.h +++ b/headers/private/kernel/vm/vm_page.h @@ -46,7 +46,7 @@ void vm_page_free_etc(VMCache* cache, vm_page* page, vm_page_reservation* reservation); void vm_page_set_state(struct vm_page *page, int state); -void vm_page_requeue(struct vm_page *page, bool tail); +void vm_page_requeue(struct vm_page *page, bool tail, struct VMPageQueue** _queue); // get some data about the number of pages in the system page_num_t vm_page_num_pages(void); diff --git a/headers/private/kernel/vm/vm_priv.h b/headers/private/kernel/vm/vm_priv.h index 643d9ae497..fbcebf010f 100644 --- a/headers/private/kernel/vm/vm_priv.h +++ b/headers/private/kernel/vm/vm_priv.h @@ -39,6 +39,8 @@ void vm_debug_init(); void vm_kernel_args_init_post_area(kernel_args* args); status_t vm_daemon_init(void); +bool vm_page_should_do_active_paging(); + const char *page_state_to_string(int state); // for debugging purposes only diff --git a/src/system/kernel/cache/file_cache.cpp b/src/system/kernel/cache/file_cache.cpp index c5ec475944..d846ffedd8 100644 --- a/src/system/kernel/cache/file_cache.cpp +++ b/src/system/kernel/cache/file_cache.cpp @@ -890,7 +890,7 @@ do_cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, if (page->State() == PAGE_STATE_CACHED || page->State() == PAGE_STATE_MODIFIED) { DEBUG_PAGE_ACCESS_START(page); - vm_page_requeue(page, true); + vm_page_requeue(page, true, NULL); DEBUG_PAGE_ACCESS_END(page); } diff --git a/src/system/kernel/vm/Jamfile b/src/system/kernel/vm/Jamfile index 1468c45d7c..bb5fef7776 100644 --- a/src/system/kernel/vm/Jamfile +++ b/src/system/kernel/vm/Jamfile @@ -12,6 +12,7 @@ KernelMergeObject kernel_vm.o : vm_debug.cpp vm_init.cpp vm_page.cpp + vm_page_writer.cpp VMAddressSpace.cpp VMAddressSpaceLocking.cpp VMAnonymousCache.cpp diff --git a/src/system/kernel/vm/ModifiedPageQueue.h b/src/system/kernel/vm/ModifiedPageQueue.h new file mode 100644 index 0000000000..019f47343d --- /dev/null +++ b/src/system/kernel/vm/ModifiedPageQueue.h @@ -0,0 +1,29 @@ +/* + * Copyright 2026, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + */ +#ifndef MODIFIED_PAGE_QUEUE_H +#define MODIFIED_PAGE_QUEUE_H + + +#include + +#include "VMPageQueue.h" + + +struct ModifiedPageQueue : public VMPageQueue { +public: + status_t StartWriter(); + void NotifyWriter() { fPageWriterCondition.WakeUp(); } + +private: + static status_t _WriterThreadEntry(void* _this); + status_t _PageWriter(); + +private: + thread_id fWriterThread; + BinarySemaphore fPageWriterCondition; +}; + + +#endif // MODIFIED_PAGE_QUEUE_H diff --git a/src/system/kernel/vm/vm_page.cpp b/src/system/kernel/vm/vm_page.cpp index 799cd89d98..835e73ada3 100644 --- a/src/system/kernel/vm/vm_page.cpp +++ b/src/system/kernel/vm/vm_page.cpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -39,10 +38,9 @@ #include #include -#include "IORequest.h" -#include "PageCacheLocker.h" #include "VMAnonymousCache.h" #include "VMPageQueue.h" +#include "ModifiedPageQueue.h" //#define TRACE_VM_PAGE @@ -67,12 +65,6 @@ #define SCRUB_SIZE 32 // this many pages will be cleared at once in the page scrubber thread -#define MAX_PAGE_WRITER_IO_PRIORITY B_URGENT_DISPLAY_PRIORITY - // maximum I/O priority of the page writer -#define MAX_PAGE_WRITER_IO_PRIORITY_THRESHOLD 10000 - // the maximum I/O priority shall be reached when this many pages need to - // be written - // The page reserve an allocation of the certain priority must not touch. static const size_t kPageReserveForPriority[] = { @@ -103,14 +95,12 @@ static const int32 kPageUsageDecline = 1; int32 gMappedPagesCount; -static VMPageQueue sPageQueues[PAGE_STATE_FIRST_UNQUEUED]; - -static VMPageQueue& sFreePageQueue = sPageQueues[PAGE_STATE_FREE]; -static VMPageQueue& sClearPageQueue = sPageQueues[PAGE_STATE_CLEAR]; -static VMPageQueue& sModifiedPageQueue = sPageQueues[PAGE_STATE_MODIFIED]; -static VMPageQueue& sInactivePageQueue = sPageQueues[PAGE_STATE_INACTIVE]; -static VMPageQueue& sActivePageQueue = sPageQueues[PAGE_STATE_ACTIVE]; -static VMPageQueue& sCachedPageQueue = sPageQueues[PAGE_STATE_CACHED]; +static VMPageQueue sFreePageQueue; +static VMPageQueue sClearPageQueue; +static ModifiedPageQueue sModifiedPageQueue; +static VMPageQueue sInactivePageQueue; +static VMPageQueue sActivePageQueue; +static VMPageQueue sCachedPageQueue; static vm_page *sPages; static page_num_t sPhysicalPageOffset; @@ -197,7 +187,6 @@ typedef DoublyLinkedList PageReservationWaiterList; static PageReservationWaiterList sPageReservationWaiters; -static BinarySemaphore sPageWriterCondition; static BinarySemaphore sPageDaemonCondition; @@ -444,39 +433,6 @@ class FreedPageSwap : public AbstractTraceEntry { #endif // PAGE_DAEMON_TRACING -#if PAGE_WRITER_TRACING - -namespace PageWriterTracing { - -class WritePage : public AbstractTraceEntry { - public: - WritePage(vm_page* page) - : - fCache(page->Cache()), - fPage(page) - { - Initialized(); - } - - virtual void AddDump(TraceOutput& out) - { - out.Print("page write: %p, cache: %p", fPage, fCache); - } - - private: - VMCache* fCache; - vm_page* fPage; -}; - -} // namespace PageWriterTracing - -# define TPW(x) new(std::nothrow) PageWriterTracing::x - -#else -# define TPW(x) -#endif // PAGE_WRITER_TRACING - - #if PAGE_STATE_TRACING namespace PageStateTracing { @@ -1431,6 +1387,15 @@ do_active_paging(const page_stats& pageStats) } +bool +vm_page_should_do_active_paging() +{ + page_stats pageStats; + get_page_stats(pageStats); + return do_active_paging(pageStats); +} + + /*! Reserves as many pages as possible from \c sUnreservedFreePages up to \a count. Doesn't touch the last \a dontTouch pages of \c sUnreservedFreePages, though. @@ -1495,42 +1460,45 @@ unreserve_pages(uint32 count) } +static VMPageQueue* +page_queue_for(vm_page* page, uint8 state) +{ + switch (state) { + case PAGE_STATE_ACTIVE: + return &sActivePageQueue; + case PAGE_STATE_INACTIVE: + return &sInactivePageQueue; + case PAGE_STATE_MODIFIED: + return &sModifiedPageQueue; + case PAGE_STATE_CACHED: + return &sCachedPageQueue; + case PAGE_STATE_FREE: + return &sFreePageQueue; + case PAGE_STATE_CLEAR: + return &sClearPageQueue; + case PAGE_STATE_WIRED: + case PAGE_STATE_UNUSED: + return NULL; + default: + panic("page %p in invalid state %d", page, page->State()); + return NULL; + } +} + + static void free_page(vm_page* page, bool clear) { DEBUG_PAGE_ACCESS_CHECK(page); - PAGE_ASSERT(page, !page->IsMapped()); - VMPageQueue* fromQueue; - - switch (page->State()) { - case PAGE_STATE_ACTIVE: - fromQueue = &sActivePageQueue; - break; - case PAGE_STATE_INACTIVE: - fromQueue = &sInactivePageQueue; - break; - case PAGE_STATE_MODIFIED: - fromQueue = &sModifiedPageQueue; - break; - case PAGE_STATE_CACHED: - fromQueue = &sCachedPageQueue; - break; - case PAGE_STATE_FREE: - case PAGE_STATE_CLEAR: - panic("free_page(): page %p already free", page); - return; - case PAGE_STATE_WIRED: - case PAGE_STATE_UNUSED: - fromQueue = NULL; - break; - default: - panic("free_page(): page %p in invalid state %d", - page, page->State()); - return; + if (page->State() == PAGE_STATE_FREE || page->State() == PAGE_STATE_CLEAR) { + panic("free_page(): page %p already free", page); + return; } + VMPageQueue* fromQueue = page_queue_for(page, page->State()); + if (page->CacheRef() != NULL) panic("to be freed page %p has cache", page); if (page->IsMapped()) @@ -1574,64 +1542,21 @@ set_page_state(vm_page *page, int pageState) if (pageState == page->State()) return; - VMPageQueue* fromQueue; - - switch (page->State()) { - case PAGE_STATE_ACTIVE: - fromQueue = &sActivePageQueue; - break; - case PAGE_STATE_INACTIVE: - fromQueue = &sInactivePageQueue; - break; - case PAGE_STATE_MODIFIED: - fromQueue = &sModifiedPageQueue; - break; - case PAGE_STATE_CACHED: - fromQueue = &sCachedPageQueue; - break; - case PAGE_STATE_FREE: - case PAGE_STATE_CLEAR: - panic("set_page_state(): page %p is free/clear", page); - return; - case PAGE_STATE_WIRED: - case PAGE_STATE_UNUSED: - fromQueue = NULL; - break; - default: - panic("set_page_state(): page %p in invalid state %d", - page, page->State()); - return; + if (page->State() == PAGE_STATE_FREE || page->State() == PAGE_STATE_CLEAR) { + panic("set_page_state(): page %p is free/clear", page); + return; + } + if (pageState == PAGE_STATE_FREE || pageState == PAGE_STATE_CLEAR) { + panic("set_page_state(): target state is free/clear"); + return; + } + if (pageState == PAGE_STATE_CACHED) { + PAGE_ASSERT(page, !page->IsMapped()); + PAGE_ASSERT(page, !page->modified); } - VMPageQueue* toQueue; - - switch (pageState) { - case PAGE_STATE_ACTIVE: - toQueue = &sActivePageQueue; - break; - case PAGE_STATE_INACTIVE: - toQueue = &sInactivePageQueue; - break; - case PAGE_STATE_MODIFIED: - toQueue = &sModifiedPageQueue; - break; - case PAGE_STATE_CACHED: - PAGE_ASSERT(page, !page->IsMapped()); - PAGE_ASSERT(page, !page->modified); - toQueue = &sCachedPageQueue; - break; - case PAGE_STATE_FREE: - case PAGE_STATE_CLEAR: - panic("set_page_state(): target state is free/clear"); - return; - case PAGE_STATE_WIRED: - case PAGE_STATE_UNUSED: - toQueue = NULL; - break; - default: - panic("set_page_state(): invalid target state %d", pageState); - return; - } + VMPageQueue* fromQueue = page_queue_for(page, page->State()); + VMPageQueue* toQueue = page_queue_for(page, pageState); VMCache* cache = page->Cache(); if (cache != NULL && cache->temporary) { @@ -1664,29 +1589,6 @@ set_page_state(vm_page *page, int pageState) } -/*! Moves a previously modified page into a now appropriate queue. - The page queues must not be locked. -*/ -static void -move_page_to_appropriate_queue(vm_page *page) -{ - DEBUG_PAGE_ACCESS_CHECK(page); - - // Note, this logic must be in sync with what the page daemon does. - int32 state; - if (page->IsMapped()) - state = PAGE_STATE_ACTIVE; - else if (page->modified) - state = PAGE_STATE_MODIFIED; - else - state = PAGE_STATE_CACHED; - -// TODO: If free + cached pages are low, we might directly want to free the -// page. - set_page_state(page, state); -} - - static void clear_page(struct vm_page *page) { @@ -1870,640 +1772,12 @@ remove_page_marker(struct vm_page &marker) DEBUG_PAGE_ACCESS_CHECK(&marker); if (marker.State() < PAGE_STATE_FIRST_UNQUEUED) - sPageQueues[marker.State()].RemoveUnlocked(&marker); + page_queue_for(NULL, marker.State())->RemoveUnlocked(&marker); marker.SetState(PAGE_STATE_UNUSED); } -static vm_page* -next_modified_page(page_num_t& maxPagesToSee) -{ - InterruptsSpinLocker locker(sModifiedPageQueue.GetLock()); - - while (maxPagesToSee > 0) { - vm_page* page = sModifiedPageQueue.Head(); - if (page == NULL) - return NULL; - - sModifiedPageQueue.Requeue(page, true); - - maxPagesToSee--; - - if (!page->busy) - return page; - } - - return NULL; -} - - -// #pragma mark - - - -class PageWriteTransfer; -class PageWriteWrapper; - - -class PageWriterRun { -public: - status_t Init(uint32 maxPages); - - void PrepareNextRun(); - void AddPage(vm_page* page); - uint32 Go(); - - void PageWritten(PageWriteTransfer* transfer, status_t status, - bool partialTransfer, size_t bytesTransferred); - -private: - uint32 fMaxPages; - uint32 fWrapperCount; - uint32 fTransferCount; - int32 fPendingTransfers; - PageWriteWrapper* fWrappers; - PageWriteTransfer* fTransfers; - ConditionVariable fAllFinishedCondition; -}; - - -class PageWriteTransfer : public AsyncIOCallback { -public: - void SetTo(PageWriterRun* run, vm_page* page, int32 maxPages); - bool AddPage(vm_page* page); - - status_t Schedule(uint32 flags); - - void SetStatus(status_t status, size_t transferred); - - status_t Status() const { return fStatus; } - struct VMCache* Cache() const { return fCache; } - uint32 PageCount() const { return fPageCount; } - - virtual void IOFinished(status_t status, bool partialTransfer, - generic_size_t bytesTransferred); - -private: - PageWriterRun* fRun; - struct VMCache* fCache; - off_t fOffset; - uint32 fPageCount; - int32 fMaxPages; - status_t fStatus; - uint32 fVecCount; - generic_io_vec fVecs[32]; // TODO: make dynamic/configurable -}; - - -class PageWriteWrapper { -public: - PageWriteWrapper(); - ~PageWriteWrapper(); - void SetTo(vm_page* page); - bool Done(status_t result); - -private: - vm_page* fPage; - struct VMCache* fCache; - bool fIsActive; -}; - - -PageWriteWrapper::PageWriteWrapper() - : - fIsActive(false) -{ -} - - -PageWriteWrapper::~PageWriteWrapper() -{ - if (fIsActive) - panic("page write wrapper going out of scope but isn't completed"); -} - - -/*! The page's cache must be locked. -*/ -void -PageWriteWrapper::SetTo(vm_page* page) -{ - DEBUG_PAGE_ACCESS_CHECK(page); - - if (page->busy) - panic("setting page write wrapper to busy page"); - - if (fIsActive) - panic("re-setting page write wrapper that isn't completed"); - - fPage = page; - fCache = page->Cache(); - fIsActive = true; - - fPage->busy = true; - fPage->busy_io = true; - - // We have a modified page -- however, while we're writing it back, - // the page might still be mapped. In order not to lose any changes to the - // page, we mark it clean before actually writing it back; if - // writing the page fails for some reason, we'll just keep it in the - // modified page list, but that should happen only rarely. - - // If the page is changed after we cleared the dirty flag, but before we - // had the chance to write it back, then we'll write it again later -- that - // will probably not happen that often, though. - - vm_clear_map_flags(fPage, PAGE_MODIFIED); -} - - -/*! The page's cache must be locked. - The page queues must not be locked. - \return \c true if the page was written successfully respectively could be - handled somehow, \c false otherwise. -*/ -bool -PageWriteWrapper::Done(status_t result) -{ - if (!fIsActive) - panic("completing page write wrapper that is not active"); - - DEBUG_PAGE_ACCESS_START(fPage); - - fPage->busy = false; - // Set unbusy and notify later by hand. - - bool success = true; - - if (!fPage->busy_io) { - // The busy_io flag was cleared. That means the cache tried to remove - // the page while we were trying to write it. Let the cache handle the rest. - fCache->FreeRemovedPage(fPage); - } else if (result == B_OK) { - // put it into the active/inactive queue - move_page_to_appropriate_queue(fPage); - fPage->busy_io = false; - DEBUG_PAGE_ACCESS_END(fPage); - - fCache->NotifyPageEvents(fPage, PAGE_EVENT_NOT_BUSY); - } else { - // Writing the page failed -- mark the page modified and move it to - // an appropriate queue other than the modified queue, so we don't - // keep trying to write it over and over again. We keep - // non-temporary pages in the modified queue, though, so they don't - // get lost in the inactive queue. - dprintf("PageWriteWrapper: Failed to write page %p: %s\n", fPage, - strerror(result)); - - fPage->modified = true; - if (!fCache->temporary) - set_page_state(fPage, PAGE_STATE_MODIFIED); - else if (fPage->IsMapped()) - set_page_state(fPage, PAGE_STATE_ACTIVE); - else - set_page_state(fPage, PAGE_STATE_INACTIVE); - - fPage->busy_io = false; - DEBUG_PAGE_ACCESS_END(fPage); - - fCache->NotifyPageEvents(fPage, PAGE_EVENT_NOT_BUSY); - success = false; - } - - fIsActive = false; - - return success; -} - - -/*! The page's cache must be locked. -*/ -void -PageWriteTransfer::SetTo(PageWriterRun* run, vm_page* page, int32 maxPages) -{ - fRun = run; - fCache = page->Cache(); - fOffset = page->cache_offset; - fPageCount = 1; - fMaxPages = maxPages; - fStatus = B_OK; - - fVecs[0].base = (phys_addr_t)page->physical_page_number << PAGE_SHIFT; - fVecs[0].length = B_PAGE_SIZE; - fVecCount = 1; -} - - -/*! The page's cache must be locked. -*/ -bool -PageWriteTransfer::AddPage(vm_page* page) -{ - if (page->Cache() != fCache - || (fMaxPages >= 0 && fPageCount >= (uint32)fMaxPages)) - return false; - - phys_addr_t nextBase = fVecs[fVecCount - 1].base - + fVecs[fVecCount - 1].length; - - if ((phys_addr_t)page->physical_page_number << PAGE_SHIFT == nextBase - && (off_t)page->cache_offset == fOffset + fPageCount) { - // append to last iovec - fVecs[fVecCount - 1].length += B_PAGE_SIZE; - fPageCount++; - return true; - } - - nextBase = fVecs[0].base - B_PAGE_SIZE; - if ((phys_addr_t)page->physical_page_number << PAGE_SHIFT == nextBase - && (off_t)page->cache_offset == fOffset - 1) { - // prepend to first iovec and adjust offset - fVecs[0].base = nextBase; - fVecs[0].length += B_PAGE_SIZE; - fOffset = page->cache_offset; - fPageCount++; - return true; - } - - if (((off_t)page->cache_offset == fOffset + fPageCount - || (off_t)page->cache_offset == fOffset - 1) - && fVecCount < sizeof(fVecs) / sizeof(fVecs[0])) { - // not physically contiguous or not in the right order - uint32 vectorIndex; - if ((off_t)page->cache_offset < fOffset) { - // we are pre-pending another vector, move the other vecs - for (uint32 i = fVecCount; i > 0; i--) - fVecs[i] = fVecs[i - 1]; - - fOffset = page->cache_offset; - vectorIndex = 0; - } else - vectorIndex = fVecCount; - - fVecs[vectorIndex].base - = (phys_addr_t)page->physical_page_number << PAGE_SHIFT; - fVecs[vectorIndex].length = B_PAGE_SIZE; - - fVecCount++; - fPageCount++; - return true; - } - - return false; -} - - -status_t -PageWriteTransfer::Schedule(uint32 flags) -{ - off_t writeOffset = (off_t)fOffset << PAGE_SHIFT; - generic_size_t writeLength = (phys_size_t)fPageCount << PAGE_SHIFT; - - if (fRun != NULL) { - return fCache->WriteAsync(writeOffset, fVecs, fVecCount, writeLength, - flags | B_PHYSICAL_IO_REQUEST, this); - } - - status_t status = fCache->Write(writeOffset, fVecs, fVecCount, - flags | B_PHYSICAL_IO_REQUEST, &writeLength); - - SetStatus(status, writeLength); - return fStatus; -} - - -void -PageWriteTransfer::SetStatus(status_t status, size_t transferred) -{ - // only succeed if all pages up to the last one have been written fully - // and the last page has at least been written partially - if (status == B_OK && transferred <= (fPageCount - 1) * B_PAGE_SIZE) - status = B_ERROR; - - fStatus = status; -} - - -void -PageWriteTransfer::IOFinished(status_t status, bool partialTransfer, - generic_size_t bytesTransferred) -{ - SetStatus(status, bytesTransferred); - fRun->PageWritten(this, fStatus, partialTransfer, bytesTransferred); -} - - -status_t -PageWriterRun::Init(uint32 maxPages) -{ - fMaxPages = maxPages; - fWrapperCount = 0; - fTransferCount = 0; - fPendingTransfers = 0; - - fWrappers = new(std::nothrow) PageWriteWrapper[maxPages]; - fTransfers = new(std::nothrow) PageWriteTransfer[maxPages]; - if (fWrappers == NULL || fTransfers == NULL) - return B_NO_MEMORY; - - return B_OK; -} - - -void -PageWriterRun::PrepareNextRun() -{ - fWrapperCount = 0; - fTransferCount = 0; - fPendingTransfers = 0; -} - - -/*! The page's cache must be locked. -*/ -void -PageWriterRun::AddPage(vm_page* page) -{ - fWrappers[fWrapperCount++].SetTo(page); - - if (fTransferCount == 0 || !fTransfers[fTransferCount - 1].AddPage(page)) { - fTransfers[fTransferCount++].SetTo(this, page, - page->Cache()->MaxPagesPerAsyncWrite()); - } -} - - -/*! Writes all pages previously added. - \return The number of pages that could not be written or otherwise handled. -*/ -uint32 -PageWriterRun::Go() -{ - atomic_set(&fPendingTransfers, fTransferCount); - - fAllFinishedCondition.Init(this, "page writer wait for I/O"); - ConditionVariableEntry waitEntry; - fAllFinishedCondition.Add(&waitEntry); - - // schedule writes - for (uint32 i = 0; i < fTransferCount; i++) - fTransfers[i].Schedule(B_VIP_IO_REQUEST); - - // wait until all pages have been written - waitEntry.Wait(); - - // mark pages depending on whether they could be written or not - - uint32 failedPages = 0; - uint32 wrapperIndex = 0; - for (uint32 i = 0; i < fTransferCount; i++) { - PageWriteTransfer& transfer = fTransfers[i]; - transfer.Cache()->Lock(); - - for (uint32 j = 0; j < transfer.PageCount(); j++) { - if (!fWrappers[wrapperIndex++].Done(transfer.Status())) - failedPages++; - } - - transfer.Cache()->Unlock(); - } - - ASSERT(wrapperIndex == fWrapperCount); - - for (uint32 i = 0; i < fTransferCount; i++) { - PageWriteTransfer& transfer = fTransfers[i]; - struct VMCache* cache = transfer.Cache(); - - // We've acquired a references for each page - for (uint32 j = 0; j < transfer.PageCount(); j++) { - // We release the cache references after all pages were made - // unbusy again - otherwise releasing a vnode could deadlock. - cache->ReleaseStoreRef(); - cache->ReleaseRef(); - } - } - - return failedPages; -} - - -void -PageWriterRun::PageWritten(PageWriteTransfer* transfer, status_t status, - bool partialTransfer, size_t bytesTransferred) -{ - if (atomic_add(&fPendingTransfers, -1) == 1) - fAllFinishedCondition.NotifyAll(); -} - - -/*! The page writer continuously takes some pages from the modified - queue, writes them back, and moves them back to the active queue. - It runs in its own thread, and is only there to keep the number - of modified pages low, so that more pages can be reused with - fewer costs. -*/ -status_t -page_writer(void* /*unused*/) -{ - const uint32 kNumPages = 256; -#ifdef TRACE_VM_PAGE - uint32 writtenPages = 0; - bigtime_t lastWrittenTime = 0; - bigtime_t pageCollectionTime = 0; - bigtime_t pageWritingTime = 0; -#endif - - PageWriterRun run; - if (run.Init(kNumPages) != B_OK) { - panic("page writer: Failed to init PageWriterRun!"); - return B_ERROR; - } - - page_num_t pagesSinceLastSuccessfulWrite = 0; - - while (true) { -// TODO: Maybe wait shorter when memory is low! - if (sModifiedPageQueue.Count() < kNumPages) { - sPageWriterCondition.Wait(3000000, true); - // all 3 seconds when no one triggers us - } - - page_num_t modifiedPages = sModifiedPageQueue.Count(); - if (modifiedPages == 0) - continue; - - if (modifiedPages <= pagesSinceLastSuccessfulWrite) { - // We ran through the whole queue without being able to write a - // single page. Take a break. - snooze(500000); - pagesSinceLastSuccessfulWrite = 0; - } - -#if ENABLE_SWAP_SUPPORT - page_stats pageStats; - get_page_stats(pageStats); - const bool activePaging = do_active_paging(pageStats); -#endif - - // depending on how urgent it becomes to get pages to disk, we adjust - // our I/O priority - uint32 lowPagesState = low_resource_state(B_KERNEL_RESOURCE_PAGES); - int32 ioPriority = B_IDLE_PRIORITY; - if (lowPagesState >= B_LOW_RESOURCE_CRITICAL - || modifiedPages > MAX_PAGE_WRITER_IO_PRIORITY_THRESHOLD) { - ioPriority = MAX_PAGE_WRITER_IO_PRIORITY; - } else { - ioPriority = (uint64)MAX_PAGE_WRITER_IO_PRIORITY * modifiedPages - / MAX_PAGE_WRITER_IO_PRIORITY_THRESHOLD; - } - - thread_set_io_priority(ioPriority); - - uint32 numPages = 0; - run.PrepareNextRun(); - - // TODO: make this laptop friendly, too (ie. only start doing - // something if someone else did something or there is really - // enough to do). - - // collect pages to be written -#ifdef TRACE_VM_PAGE - pageCollectionTime -= system_time(); -#endif - - page_num_t maxPagesToSee = modifiedPages; - - while (numPages < kNumPages && maxPagesToSee > 0) { - vm_page *page = next_modified_page(maxPagesToSee); - if (page == NULL) - break; - - PageCacheLocker cacheLocker(page, false); - if (!cacheLocker.IsLocked()) - continue; - - VMCache *cache = page->Cache(); - - // If the page is busy or its state has changed while we were - // locking the cache, just ignore it. - if (page->busy || page->State() != PAGE_STATE_MODIFIED) - continue; - - DEBUG_PAGE_ACCESS_START(page); - - // Don't write back wired (locked) pages. - if (page->WiredCount() > 0) { - set_page_state(page, PAGE_STATE_ACTIVE); - DEBUG_PAGE_ACCESS_END(page); - continue; - } - - // Write back temporary pages only when we're actively paging. - if (cache->temporary -#if ENABLE_SWAP_SUPPORT - && (!activePaging - || !cache->CanWritePage( - (off_t)page->cache_offset << PAGE_SHIFT)) -#endif - ) { - // We can't/don't want to do anything with this page, so move it - // to one of the other queues. - if (page->mappings.IsEmpty()) - set_page_state(page, PAGE_STATE_INACTIVE); - else - set_page_state(page, PAGE_STATE_ACTIVE); - - DEBUG_PAGE_ACCESS_END(page); - continue; - } - - // We need our own reference to the store, as it might currently be - // destroyed. - if (cache->AcquireUnreferencedStoreRef() != B_OK) { - DEBUG_PAGE_ACCESS_END(page); - cacheLocker.Unlock(); - thread_yield(); - continue; - } - - run.AddPage(page); - // TODO: We're possibly adding pages of different caches and - // thus maybe of different underlying file systems here. This - // is a potential problem for loop file systems/devices, since - // we could mark a page busy that would need to be accessed - // when writing back another page, thus causing a deadlock. - - DEBUG_PAGE_ACCESS_END(page); - - //dprintf("write page %p, cache %p (%ld)\n", page, page->cache, page->cache->ref_count); - TPW(WritePage(page)); - - cache->AcquireRefLocked(); - numPages++; - - // Write adjacent pages at the same time, if they're also modified. - if (cache->temporary) - continue; - while (page->cache_next != NULL && numPages < kNumPages) { - page = page->cache_next; - if (page->busy || page->State() != PAGE_STATE_MODIFIED) - break; - if (page->WiredCount() > 0) - break; - - DEBUG_PAGE_ACCESS_START(page); - sModifiedPageQueue.RequeueUnlocked(page, true); - run.AddPage(page); - DEBUG_PAGE_ACCESS_END(page); - - cache->AcquireStoreRef(); - cache->AcquireRefLocked(); - numPages++; - if (maxPagesToSee > 0) - maxPagesToSee--; - } - } - -#ifdef TRACE_VM_PAGE - pageCollectionTime += system_time(); -#endif - if (numPages == 0) - continue; - - // write pages to disk and do all the cleanup -#ifdef TRACE_VM_PAGE - pageWritingTime -= system_time(); -#endif - uint32 failedPages = run.Go(); -#ifdef TRACE_VM_PAGE - pageWritingTime += system_time(); - - // debug output only... - writtenPages += numPages; - if (writtenPages >= 1024) { - bigtime_t now = system_time(); - TRACE(("page writer: wrote 1024 pages (total: %" B_PRIu64 " ms, " - "collect: %" B_PRIu64 " ms, write: %" B_PRIu64 " ms)\n", - (now - lastWrittenTime) / 1000, - pageCollectionTime / 1000, pageWritingTime / 1000)); - lastWrittenTime = now; - - writtenPages -= 1024; - pageCollectionTime = 0; - pageWritingTime = 0; - } -#endif - - if (failedPages == numPages) - pagesSinceLastSuccessfulWrite += modifiedPages - maxPagesToSee; - else - pagesSinceLastSuccessfulWrite = 0; - } - - return B_OK; -} - - // #pragma mark - @@ -2667,7 +1941,7 @@ idle_scan_active_pages(page_stats& pageStats) if (page->busy) { // page is busy -- requeue at the end - vm_page_requeue(page, true); + vm_page_requeue(page, true, NULL); cache->ReleaseRefAndUnlock(); continue; } @@ -2808,9 +2082,9 @@ full_scan_inactive_pages(page_stats& pageStats, int32 despairLevel) set_page_state(page, PAGE_STATE_ACTIVE); pagesToActive++; } else - vm_page_requeue(page, true); + vm_page_requeue(page, true, NULL); } else if (isMapped) { - vm_page_requeue(page, true); + vm_page_requeue(page, true, NULL); } else if (!page->modified) { set_page_state(page, PAGE_STATE_CACHED); pagesToFree--; @@ -2820,7 +2094,7 @@ full_scan_inactive_pages(page_stats& pageStats, int32 despairLevel) maxToFlush--; pagesToModified++; } else - vm_page_requeue(page, true); + vm_page_requeue(page, true, NULL); DEBUG_PAGE_ACCESS_END(page); @@ -2842,7 +2116,7 @@ full_scan_inactive_pages(page_stats& pageStats, int32 despairLevel) // wake up the page writer, if we tossed it some pages if (pagesToModified > 0) - sPageWriterCondition.WakeUp(); + sModifiedPageQueue.NotifyWriter(); } @@ -3117,175 +2391,6 @@ reserve_pages(uint32 missing, int priority, bool dontWait) } -// #pragma mark - private kernel API - - -/*! Writes a range of modified pages of a cache to disk. - You need to hold the VMCache lock when calling this function. - Note that the cache lock is released in this function. - \param cache The cache. - \param firstPage Offset (in page size units) of the first page in the range. - \param endPage End offset (in page size units) of the page range. The page - at this offset is not included. -*/ -status_t -vm_page_write_modified_page_range(struct VMCache* cache, uint32 firstPage, - uint32 endPage) -{ - static const int32 kMaxPages = 256; - int32 maxPages = cache->MaxPagesPerWrite(); - if (maxPages < 0 || maxPages > kMaxPages) - maxPages = kMaxPages; - - const uint32 allocationFlags = HEAP_DONT_WAIT_FOR_MEMORY - | HEAP_DONT_LOCK_KERNEL_SPACE; - - PageWriteWrapper stackWrappersPool[2]; - PageWriteWrapper* stackWrappers[1]; - PageWriteWrapper* wrapperPool - = new(malloc_flags(allocationFlags)) PageWriteWrapper[maxPages + 1]; - PageWriteWrapper** wrappers - = new(malloc_flags(allocationFlags)) PageWriteWrapper*[maxPages]; - if (wrapperPool == NULL || wrappers == NULL) { - // don't fail, just limit our capabilities - delete[] wrapperPool; - delete[] wrappers; - wrapperPool = stackWrappersPool; - wrappers = stackWrappers; - maxPages = 1; - } - - int32 nextWrapper = 0; - int32 usedWrappers = 0; - - PageWriteTransfer transfer; - bool transferEmpty = true; - - VMCachePagesTree::Iterator it - = cache->pages.GetIterator(firstPage, true, true); - - while (true) { - vm_page* page = it.Next(); - if (page == NULL || page->cache_offset >= endPage) { - if (transferEmpty) - break; - - page = NULL; - } - - if (page != NULL) { - if (page->busy - || (page->State() != PAGE_STATE_MODIFIED - && !vm_test_map_modification(page))) { - page = NULL; - } - } - - PageWriteWrapper* wrapper = NULL; - if (page != NULL) { - wrapper = &wrapperPool[nextWrapper++]; - if (nextWrapper > maxPages) - nextWrapper = 0; - - DEBUG_PAGE_ACCESS_START(page); - - wrapper->SetTo(page); - - if (transferEmpty || transfer.AddPage(page)) { - if (transferEmpty) { - transfer.SetTo(NULL, page, maxPages); - transferEmpty = false; - } - - DEBUG_PAGE_ACCESS_END(page); - - wrappers[usedWrappers++] = wrapper; - continue; - } - - DEBUG_PAGE_ACCESS_END(page); - } - - if (transferEmpty) - continue; - - cache->Unlock(); - status_t status = transfer.Schedule(0); - cache->Lock(); - - for (int32 i = 0; i < usedWrappers; i++) - wrappers[i]->Done(status); - - usedWrappers = 0; - - if (page != NULL) { - transfer.SetTo(NULL, page, maxPages); - wrappers[usedWrappers++] = wrapper; - } else - transferEmpty = true; - } - - if (wrapperPool != stackWrappersPool) { - delete[] wrapperPool; - delete[] wrappers; - } - - return B_OK; -} - - -/*! You need to hold the VMCache lock when calling this function. - Note that the cache lock is released in this function. -*/ -status_t -vm_page_write_modified_pages(VMCache *cache) -{ - return vm_page_write_modified_page_range(cache, 0, - (cache->virtual_end + B_PAGE_SIZE - 1) >> PAGE_SHIFT); -} - - -/*! Schedules the page writer to write back the specified \a page. - Note, however, that it might not do this immediately, and it can well - take several seconds until the page is actually written out. -*/ -void -vm_page_schedule_write_page(vm_page *page) -{ - PAGE_ASSERT(page, page->State() == PAGE_STATE_MODIFIED); - - vm_page_requeue(page, false); - - sPageWriterCondition.WakeUp(); -} - - -/*! Cache must be locked. -*/ -void -vm_page_schedule_write_page_range(struct VMCache *cache, uint32 firstPage, - uint32 endPage) -{ - uint32 modified = 0; - for (VMCachePagesTree::Iterator it - = cache->pages.GetIterator(firstPage, true, true); - vm_page *page = it.Next();) { - if (page->cache_offset >= endPage) - break; - - if (!page->busy && page->State() == PAGE_STATE_MODIFIED) { - DEBUG_PAGE_ACCESS_START(page); - vm_page_requeue(page, false); - modified++; - DEBUG_PAGE_ACCESS_END(page); - } - } - - if (modified > 0) - sPageWriterCondition.WakeUp(); -} - - void vm_page_init_num_pages(kernel_args *args) { @@ -3488,12 +2593,7 @@ vm_page_init_post_thread(kernel_args *args) resume_thread(thread); // start page writer - - sPageWriterCondition.Init("page writer"); - - thread = spawn_kernel_thread(&page_writer, "page writer", - B_NORMAL_PRIORITY + 1, NULL); - resume_thread(thread); + sModifiedPageQueue.StartWriter(); // start page daemon @@ -3647,7 +2747,7 @@ vm_page_allocate_page(vm_page_reservation* reservation, uint32 flags) locker.Unlock(); if (pageState < PAGE_STATE_FIRST_UNQUEUED) - sPageQueues[pageState].AppendUnlocked(page); + page_queue_for(page, pageState)->AppendUnlocked(page); // clear the page, if we had to take it from the free queue and a clear // page was requested @@ -3838,7 +2938,7 @@ allocate_page_run(page_num_t start, page_num_t length, uint32 flags, // add pages to target queue if (pageState < PAGE_STATE_FIRST_UNQUEUED) { freePages.TakeFrom(&clearPages); - sPageQueues[pageState].AppendUnlocked(freePages, length); + page_queue_for(NULL, pageState)->AppendUnlocked(freePages, length); } #if VM_PAGE_ALLOCATION_TRACKING_AVAILABLE @@ -4063,40 +3163,23 @@ vm_page_set_state(vm_page *page, int pageState) The page must have a cache and the cache must be locked! */ void -vm_page_requeue(struct vm_page *page, bool tail) +vm_page_requeue(struct vm_page *page, bool tail, VMPageQueue** _queue) { PAGE_ASSERT(page, page->Cache() != NULL); page->Cache()->AssertLocked(); DEBUG_PAGE_ACCESS_CHECK(page); - VMPageQueue *queue = NULL; - - switch (page->State()) { - case PAGE_STATE_ACTIVE: - queue = &sActivePageQueue; - break; - case PAGE_STATE_INACTIVE: - queue = &sInactivePageQueue; - break; - case PAGE_STATE_MODIFIED: - queue = &sModifiedPageQueue; - break; - case PAGE_STATE_CACHED: - queue = &sCachedPageQueue; - break; - case PAGE_STATE_FREE: - case PAGE_STATE_CLEAR: - panic("vm_page_requeue() called for free/clear page %p", page); - return; - case PAGE_STATE_WIRED: - case PAGE_STATE_UNUSED: - return; - default: - panic("vm_page_touch: vm_page %p in invalid state %d\n", - page, page->State()); - break; + if (page->State() == PAGE_STATE_FREE || page->State() == PAGE_STATE_CLEAR) { + panic("vm_page_requeue() called for free/clear page %p", page); + return; } + VMPageQueue *queue = page_queue_for(page, page->State()); + if (_queue != NULL) + *_queue = queue; + if (queue == NULL) + return; + queue->RequeueUnlocked(page, tail); } diff --git a/src/system/kernel/vm/vm_page_writer.cpp b/src/system/kernel/vm/vm_page_writer.cpp new file mode 100644 index 0000000000..e9a187c7f5 --- /dev/null +++ b/src/system/kernel/vm/vm_page_writer.cpp @@ -0,0 +1,921 @@ +/* + * Copyright 2010-2011, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. + * Distributed under the terms of the MIT License. + * + * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. + * Distributed under the terms of the NewOS License. + */ + + +#include +#include + +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "IORequest.h" +#include "PageCacheLocker.h" +#include "ModifiedPageQueue.h" + + +//#define TRACE_VM_PAGE_WRITER +#ifdef TRACE_VM_PAGE_WRITER +# define TRACE(x) dprintf x +#else +# define TRACE(x) ; +#endif + +#define MAX_PAGE_WRITER_IO_PRIORITY B_URGENT_DISPLAY_PRIORITY + // maximum I/O priority of the page writer +#define MAX_PAGE_WRITER_IO_PRIORITY_THRESHOLD 10000 + // the maximum I/O priority shall be reached when this many pages need to + // be written + + +#if PAGE_WRITER_TRACING + +namespace PageWriterTracing { + +class WritePage : public AbstractTraceEntry { + public: + WritePage(vm_page* page) + : + fCache(page->Cache()), + fPage(page) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("page write: %p, cache: %p", fPage, fCache); + } + + private: + VMCache* fCache; + vm_page* fPage; +}; + +} // namespace PageWriterTracing + +# define TPW(x) new(std::nothrow) PageWriterTracing::x + +#else +# define TPW(x) +#endif // PAGE_WRITER_TRACING + + + +class PageWriteTransfer; +class PageWriteWrapper; + + +class PageWriterRun { +public: + status_t Init(uint32 maxPages); + + void PrepareNextRun(); + void AddPage(vm_page* page); + uint32 Go(); + + void PageWritten(PageWriteTransfer* transfer, status_t status, + bool partialTransfer, size_t bytesTransferred); + +private: + uint32 fMaxPages; + uint32 fWrapperCount; + uint32 fTransferCount; + int32 fPendingTransfers; + PageWriteWrapper* fWrappers; + PageWriteTransfer* fTransfers; + ConditionVariable fAllFinishedCondition; +}; + + +class PageWriteTransfer : public AsyncIOCallback { +public: + void SetTo(PageWriterRun* run, vm_page* page, int32 maxPages); + bool AddPage(vm_page* page); + + status_t Schedule(uint32 flags); + + void SetStatus(status_t status, size_t transferred); + + status_t Status() const { return fStatus; } + struct VMCache* Cache() const { return fCache; } + uint32 PageCount() const { return fPageCount; } + + virtual void IOFinished(status_t status, bool partialTransfer, + generic_size_t bytesTransferred); + +private: + PageWriterRun* fRun; + struct VMCache* fCache; + off_t fOffset; + uint32 fPageCount; + int32 fMaxPages; + status_t fStatus; + uint32 fVecCount; + generic_io_vec fVecs[32]; // TODO: make dynamic/configurable +}; + + +class PageWriteWrapper { +public: + PageWriteWrapper(); + ~PageWriteWrapper(); + void SetTo(vm_page* page); + bool Done(status_t result); + +private: + vm_page* fPage; + struct VMCache* fCache; + bool fIsActive; +}; + + +PageWriteWrapper::PageWriteWrapper() + : + fIsActive(false) +{ +} + + +PageWriteWrapper::~PageWriteWrapper() +{ + if (fIsActive) + panic("page write wrapper going out of scope but isn't completed"); +} + + +/*! The page's cache must be locked. +*/ +void +PageWriteWrapper::SetTo(vm_page* page) +{ + DEBUG_PAGE_ACCESS_CHECK(page); + + if (page->busy) + panic("setting page write wrapper to busy page"); + + if (fIsActive) + panic("re-setting page write wrapper that isn't completed"); + + fPage = page; + fCache = page->Cache(); + fIsActive = true; + + fPage->busy = true; + fPage->busy_io = true; + + // We have a modified page -- however, while we're writing it back, + // the page might still be mapped. In order not to lose any changes to the + // page, we mark it clean before actually writing it back; if + // writing the page fails for some reason, we'll just keep it in the + // modified page list, but that should happen only rarely. + + // If the page is changed after we cleared the dirty flag, but before we + // had the chance to write it back, then we'll write it again later -- that + // will probably not happen that often, though. + + vm_clear_map_flags(fPage, PAGE_MODIFIED); +} + + +/*! Moves a previously modified page into a now appropriate queue. + The page queues must not be locked. +*/ +static void +move_page_to_appropriate_queue(vm_page *page) +{ + DEBUG_PAGE_ACCESS_CHECK(page); + + // Note, this logic must be in sync with what the page daemon does. + int32 state; + if (page->IsMapped()) + state = PAGE_STATE_ACTIVE; + else if (page->modified) + state = PAGE_STATE_MODIFIED; + else + state = PAGE_STATE_CACHED; + +// TODO: If free + cached pages are low, we might directly want to free the +// page. + vm_page_set_state(page, state); +} + + +/*! The page's cache must be locked. + The page queues must not be locked. + \return \c true if the page was written successfully respectively could be + handled somehow, \c false otherwise. +*/ +bool +PageWriteWrapper::Done(status_t result) +{ + if (!fIsActive) + panic("completing page write wrapper that is not active"); + + DEBUG_PAGE_ACCESS_START(fPage); + + fPage->busy = false; + // Set unbusy and notify later by hand. + + bool success = true; + + if (!fPage->busy_io) { + // The busy_io flag was cleared. That means the cache tried to remove + // the page while we were trying to write it. Let the cache handle the rest. + fCache->FreeRemovedPage(fPage); + } else if (result == B_OK) { + // put it into the active/inactive queue + move_page_to_appropriate_queue(fPage); + fPage->busy_io = false; + DEBUG_PAGE_ACCESS_END(fPage); + + fCache->NotifyPageEvents(fPage, PAGE_EVENT_NOT_BUSY); + } else { + // Writing the page failed -- mark the page modified and move it to + // an appropriate queue other than the modified queue, so we don't + // keep trying to write it over and over again. We keep + // non-temporary pages in the modified queue, though, so they don't + // get lost in the inactive queue. + dprintf("PageWriteWrapper: Failed to write page %p: %s\n", fPage, + strerror(result)); + + fPage->modified = true; + if (!fCache->temporary) + vm_page_set_state(fPage, PAGE_STATE_MODIFIED); + else if (fPage->IsMapped()) + vm_page_set_state(fPage, PAGE_STATE_ACTIVE); + else + vm_page_set_state(fPage, PAGE_STATE_INACTIVE); + + fPage->busy_io = false; + DEBUG_PAGE_ACCESS_END(fPage); + + fCache->NotifyPageEvents(fPage, PAGE_EVENT_NOT_BUSY); + success = false; + } + + fIsActive = false; + + return success; +} + + +/*! The page's cache must be locked. +*/ +void +PageWriteTransfer::SetTo(PageWriterRun* run, vm_page* page, int32 maxPages) +{ + fRun = run; + fCache = page->Cache(); + fOffset = page->cache_offset; + fPageCount = 1; + fMaxPages = maxPages; + fStatus = B_OK; + + fVecs[0].base = (phys_addr_t)page->physical_page_number << PAGE_SHIFT; + fVecs[0].length = B_PAGE_SIZE; + fVecCount = 1; +} + + +/*! The page's cache must be locked. +*/ +bool +PageWriteTransfer::AddPage(vm_page* page) +{ + if (page->Cache() != fCache + || (fMaxPages >= 0 && fPageCount >= (uint32)fMaxPages)) + return false; + + phys_addr_t nextBase = fVecs[fVecCount - 1].base + + fVecs[fVecCount - 1].length; + + if ((phys_addr_t)page->physical_page_number << PAGE_SHIFT == nextBase + && (off_t)page->cache_offset == fOffset + fPageCount) { + // append to last iovec + fVecs[fVecCount - 1].length += B_PAGE_SIZE; + fPageCount++; + return true; + } + + nextBase = fVecs[0].base - B_PAGE_SIZE; + if ((phys_addr_t)page->physical_page_number << PAGE_SHIFT == nextBase + && (off_t)page->cache_offset == fOffset - 1) { + // prepend to first iovec and adjust offset + fVecs[0].base = nextBase; + fVecs[0].length += B_PAGE_SIZE; + fOffset = page->cache_offset; + fPageCount++; + return true; + } + + if (((off_t)page->cache_offset == fOffset + fPageCount + || (off_t)page->cache_offset == fOffset - 1) + && fVecCount < sizeof(fVecs) / sizeof(fVecs[0])) { + // not physically contiguous or not in the right order + uint32 vectorIndex; + if ((off_t)page->cache_offset < fOffset) { + // we are pre-pending another vector, move the other vecs + for (uint32 i = fVecCount; i > 0; i--) + fVecs[i] = fVecs[i - 1]; + + fOffset = page->cache_offset; + vectorIndex = 0; + } else + vectorIndex = fVecCount; + + fVecs[vectorIndex].base + = (phys_addr_t)page->physical_page_number << PAGE_SHIFT; + fVecs[vectorIndex].length = B_PAGE_SIZE; + + fVecCount++; + fPageCount++; + return true; + } + + return false; +} + + +status_t +PageWriteTransfer::Schedule(uint32 flags) +{ + off_t writeOffset = (off_t)fOffset << PAGE_SHIFT; + generic_size_t writeLength = (phys_size_t)fPageCount << PAGE_SHIFT; + + if (fRun != NULL) { + return fCache->WriteAsync(writeOffset, fVecs, fVecCount, writeLength, + flags | B_PHYSICAL_IO_REQUEST, this); + } + + status_t status = fCache->Write(writeOffset, fVecs, fVecCount, + flags | B_PHYSICAL_IO_REQUEST, &writeLength); + + SetStatus(status, writeLength); + return fStatus; +} + + +void +PageWriteTransfer::SetStatus(status_t status, size_t transferred) +{ + // only succeed if all pages up to the last one have been written fully + // and the last page has at least been written partially + if (status == B_OK && transferred <= (fPageCount - 1) * B_PAGE_SIZE) + status = B_ERROR; + + fStatus = status; +} + + +void +PageWriteTransfer::IOFinished(status_t status, bool partialTransfer, + generic_size_t bytesTransferred) +{ + SetStatus(status, bytesTransferred); + fRun->PageWritten(this, fStatus, partialTransfer, bytesTransferred); +} + + +status_t +PageWriterRun::Init(uint32 maxPages) +{ + fMaxPages = maxPages; + fWrapperCount = 0; + fTransferCount = 0; + fPendingTransfers = 0; + + fWrappers = new(std::nothrow) PageWriteWrapper[maxPages]; + fTransfers = new(std::nothrow) PageWriteTransfer[maxPages]; + if (fWrappers == NULL || fTransfers == NULL) + return B_NO_MEMORY; + + return B_OK; +} + + +void +PageWriterRun::PrepareNextRun() +{ + fWrapperCount = 0; + fTransferCount = 0; + fPendingTransfers = 0; +} + + +/*! The page's cache must be locked. +*/ +void +PageWriterRun::AddPage(vm_page* page) +{ + fWrappers[fWrapperCount++].SetTo(page); + + if (fTransferCount == 0 || !fTransfers[fTransferCount - 1].AddPage(page)) { + fTransfers[fTransferCount++].SetTo(this, page, + page->Cache()->MaxPagesPerAsyncWrite()); + } +} + + +/*! Writes all pages previously added. + \return The number of pages that could not be written or otherwise handled. +*/ +uint32 +PageWriterRun::Go() +{ + atomic_set(&fPendingTransfers, fTransferCount); + + fAllFinishedCondition.Init(this, "page writer wait for I/O"); + ConditionVariableEntry waitEntry; + fAllFinishedCondition.Add(&waitEntry); + + // schedule writes + for (uint32 i = 0; i < fTransferCount; i++) + fTransfers[i].Schedule(B_VIP_IO_REQUEST); + + // wait until all pages have been written + waitEntry.Wait(); + + // mark pages depending on whether they could be written or not + + uint32 failedPages = 0; + uint32 wrapperIndex = 0; + for (uint32 i = 0; i < fTransferCount; i++) { + PageWriteTransfer& transfer = fTransfers[i]; + transfer.Cache()->Lock(); + + for (uint32 j = 0; j < transfer.PageCount(); j++) { + if (!fWrappers[wrapperIndex++].Done(transfer.Status())) + failedPages++; + } + + transfer.Cache()->Unlock(); + } + + ASSERT(wrapperIndex == fWrapperCount); + + for (uint32 i = 0; i < fTransferCount; i++) { + PageWriteTransfer& transfer = fTransfers[i]; + struct VMCache* cache = transfer.Cache(); + + // We've acquired a references for each page + for (uint32 j = 0; j < transfer.PageCount(); j++) { + // We release the cache references after all pages were made + // unbusy again - otherwise releasing a vnode could deadlock. + cache->ReleaseStoreRef(); + cache->ReleaseRef(); + } + } + + return failedPages; +} + + +void +PageWriterRun::PageWritten(PageWriteTransfer* transfer, status_t status, + bool partialTransfer, size_t bytesTransferred) +{ + if (atomic_add(&fPendingTransfers, -1) == 1) + fAllFinishedCondition.NotifyAll(); +} + + +static vm_page* +next_modified_page(VMPageQueue& queue, page_num_t& maxPagesToSee) +{ + InterruptsSpinLocker locker(queue.GetLock()); + + while (maxPagesToSee > 0) { + vm_page* page = queue.Head(); + if (page == NULL) + return NULL; + + queue.Requeue(page, true); + + maxPagesToSee--; + + if (!page->busy) + return page; + } + + return NULL; +} + + +/*! The page writer continuously takes some pages from the modified + queue, writes them back, and moves them back to the active queue. + It runs in its own thread, and is only there to keep the number + of modified pages low, so that more pages can be reused with + fewer costs. +*/ +status_t +ModifiedPageQueue::_PageWriter() +{ + ModifiedPageQueue& queue = *this; + + const uint32 kNumPages = 256; +#ifdef TRACE_VM_PAGE_WRITER + uint32 writtenPages = 0; + bigtime_t lastWrittenTime = 0; + bigtime_t pageCollectionTime = 0; + bigtime_t pageWritingTime = 0; +#endif + + PageWriterRun run; + if (run.Init(kNumPages) != B_OK) { + panic("page writer: Failed to init PageWriterRun!"); + return B_ERROR; + } + + page_num_t pagesSinceLastSuccessfulWrite = 0; + + while (true) { +// TODO: Maybe wait shorter when memory is low! + if (queue.Count() < kNumPages) { + fPageWriterCondition.Wait(3000000, true); + // all 3 seconds when no one triggers us + } + + page_num_t modifiedPages = queue.Count(); + if (modifiedPages == 0) + continue; + + if (modifiedPages <= pagesSinceLastSuccessfulWrite) { + // We ran through the whole queue without being able to write a + // single page. Take a break. + snooze(500000); + pagesSinceLastSuccessfulWrite = 0; + } + +#if ENABLE_SWAP_SUPPORT + const bool activePaging = vm_page_should_do_active_paging(); +#endif + + // depending on how urgent it becomes to get pages to disk, we adjust + // our I/O priority + uint32 lowPagesState = low_resource_state(B_KERNEL_RESOURCE_PAGES); + int32 ioPriority = B_IDLE_PRIORITY; + if (lowPagesState >= B_LOW_RESOURCE_CRITICAL + || modifiedPages > MAX_PAGE_WRITER_IO_PRIORITY_THRESHOLD) { + ioPriority = MAX_PAGE_WRITER_IO_PRIORITY; + } else { + ioPriority = (uint64)MAX_PAGE_WRITER_IO_PRIORITY * modifiedPages + / MAX_PAGE_WRITER_IO_PRIORITY_THRESHOLD; + } + + thread_set_io_priority(ioPriority); + + uint32 numPages = 0; + run.PrepareNextRun(); + + // TODO: make this laptop friendly, too (ie. only start doing + // something if someone else did something or there is really + // enough to do). + + // collect pages to be written +#ifdef TRACE_VM_PAGE_WRITER + pageCollectionTime -= system_time(); +#endif + + page_num_t maxPagesToSee = modifiedPages; + + while (numPages < kNumPages && maxPagesToSee > 0) { + vm_page *page = next_modified_page(queue, maxPagesToSee); + if (page == NULL) + break; + + PageCacheLocker cacheLocker(page, false); + if (!cacheLocker.IsLocked()) + continue; + + VMCache *cache = page->Cache(); + + // If the page is busy or its state has changed while we were + // locking the cache, just ignore it. + if (page->busy || page->State() != PAGE_STATE_MODIFIED) + continue; + + DEBUG_PAGE_ACCESS_START(page); + + // Don't write back wired (locked) pages. + if (page->WiredCount() > 0) { + vm_page_set_state(page, PAGE_STATE_ACTIVE); + DEBUG_PAGE_ACCESS_END(page); + continue; + } + + // Write back temporary pages only when we're actively paging. + if (cache->temporary +#if ENABLE_SWAP_SUPPORT + && (!activePaging + || !cache->CanWritePage( + (off_t)page->cache_offset << PAGE_SHIFT)) +#endif + ) { + // We can't/don't want to do anything with this page, so move it + // to one of the other queues. + if (page->mappings.IsEmpty()) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else + vm_page_set_state(page, PAGE_STATE_ACTIVE); + + DEBUG_PAGE_ACCESS_END(page); + continue; + } + + // We need our own reference to the store, as it might currently be + // destroyed. + if (cache->AcquireUnreferencedStoreRef() != B_OK) { + DEBUG_PAGE_ACCESS_END(page); + cacheLocker.Unlock(); + thread_yield(); + continue; + } + + run.AddPage(page); + // TODO: We're possibly adding pages of different caches and + // thus maybe of different underlying file systems here. This + // is a potential problem for loop file systems/devices, since + // we could mark a page busy that would need to be accessed + // when writing back another page, thus causing a deadlock. + + DEBUG_PAGE_ACCESS_END(page); + + //dprintf("write page %p, cache %p (%ld)\n", page, page->cache, page->cache->ref_count); + TPW(WritePage(page)); + + cache->AcquireRefLocked(); + numPages++; + + // Write adjacent pages at the same time, if they're also modified. + if (cache->temporary) + continue; + while (page->cache_next != NULL && numPages < kNumPages) { + page = page->cache_next; + if (page->busy || page->State() != PAGE_STATE_MODIFIED) + break; + if (page->WiredCount() > 0) + break; + + DEBUG_PAGE_ACCESS_START(page); + queue.RequeueUnlocked(page, true); + run.AddPage(page); + DEBUG_PAGE_ACCESS_END(page); + + cache->AcquireStoreRef(); + cache->AcquireRefLocked(); + numPages++; + if (maxPagesToSee > 0) + maxPagesToSee--; + } + } + +#ifdef TRACE_VM_PAGE_WRITER + pageCollectionTime += system_time(); +#endif + if (numPages == 0) + continue; + + // write pages to disk and do all the cleanup +#ifdef TRACE_VM_PAGE_WRITER + pageWritingTime -= system_time(); +#endif + uint32 failedPages = run.Go(); +#ifdef TRACE_VM_PAGE_WRITER + pageWritingTime += system_time(); + + // debug output only... + writtenPages += numPages; + if (writtenPages >= 1024) { + bigtime_t now = system_time(); + TRACE(("page writer: wrote 1024 pages (total: %" B_PRIu64 " ms, " + "collect: %" B_PRIu64 " ms, write: %" B_PRIu64 " ms)\n", + (now - lastWrittenTime) / 1000, + pageCollectionTime / 1000, pageWritingTime / 1000)); + lastWrittenTime = now; + + writtenPages -= 1024; + pageCollectionTime = 0; + pageWritingTime = 0; + } +#endif + + if (failedPages == numPages) + pagesSinceLastSuccessfulWrite += modifiedPages - maxPagesToSee; + else + pagesSinceLastSuccessfulWrite = 0; + } + + return B_OK; +} + + +/*static*/ status_t +ModifiedPageQueue::_WriterThreadEntry(void* _this) +{ + return ((ModifiedPageQueue*)_this)->_PageWriter(); +} + + + +// #pragma mark - private kernel API + + +/*! Writes a range of modified pages of a cache to disk. + You need to hold the VMCache lock when calling this function. + Note that the cache lock is released in this function. + \param cache The cache. + \param firstPage Offset (in page size units) of the first page in the range. + \param endPage End offset (in page size units) of the page range. The page + at this offset is not included. +*/ +status_t +vm_page_write_modified_page_range(struct VMCache* cache, uint32 firstPage, + uint32 endPage) +{ + static const int32 kMaxPages = 256; + int32 maxPages = cache->MaxPagesPerWrite(); + if (maxPages < 0 || maxPages > kMaxPages) + maxPages = kMaxPages; + + const uint32 allocationFlags = HEAP_DONT_WAIT_FOR_MEMORY + | HEAP_DONT_LOCK_KERNEL_SPACE; + + PageWriteWrapper stackWrappersPool[2]; + PageWriteWrapper* stackWrappers[1]; + PageWriteWrapper* wrapperPool + = new(malloc_flags(allocationFlags)) PageWriteWrapper[maxPages + 1]; + PageWriteWrapper** wrappers + = new(malloc_flags(allocationFlags)) PageWriteWrapper*[maxPages]; + if (wrapperPool == NULL || wrappers == NULL) { + // don't fail, just limit our capabilities + delete[] wrapperPool; + delete[] wrappers; + wrapperPool = stackWrappersPool; + wrappers = stackWrappers; + maxPages = 1; + } + + int32 nextWrapper = 0; + int32 usedWrappers = 0; + + PageWriteTransfer transfer; + bool transferEmpty = true; + + VMCachePagesTree::Iterator it + = cache->pages.GetIterator(firstPage, true, true); + + while (true) { + vm_page* page = it.Next(); + if (page == NULL || page->cache_offset >= endPage) { + if (transferEmpty) + break; + + page = NULL; + } + + if (page != NULL) { + if (page->busy + || (page->State() != PAGE_STATE_MODIFIED + && !vm_test_map_modification(page))) { + page = NULL; + } + } + + PageWriteWrapper* wrapper = NULL; + if (page != NULL) { + wrapper = &wrapperPool[nextWrapper++]; + if (nextWrapper > maxPages) + nextWrapper = 0; + + DEBUG_PAGE_ACCESS_START(page); + + wrapper->SetTo(page); + + if (transferEmpty || transfer.AddPage(page)) { + if (transferEmpty) { + transfer.SetTo(NULL, page, maxPages); + transferEmpty = false; + } + + DEBUG_PAGE_ACCESS_END(page); + + wrappers[usedWrappers++] = wrapper; + continue; + } + + DEBUG_PAGE_ACCESS_END(page); + } + + if (transferEmpty) + continue; + + cache->Unlock(); + status_t status = transfer.Schedule(0); + cache->Lock(); + + for (int32 i = 0; i < usedWrappers; i++) + wrappers[i]->Done(status); + + usedWrappers = 0; + + if (page != NULL) { + transfer.SetTo(NULL, page, maxPages); + wrappers[usedWrappers++] = wrapper; + } else + transferEmpty = true; + } + + if (wrapperPool != stackWrappersPool) { + delete[] wrapperPool; + delete[] wrappers; + } + + return B_OK; +} + + +/*! You need to hold the VMCache lock when calling this function. + Note that the cache lock is released in this function. +*/ +status_t +vm_page_write_modified_pages(VMCache *cache) +{ + return vm_page_write_modified_page_range(cache, 0, + (cache->virtual_end + B_PAGE_SIZE - 1) >> PAGE_SHIFT); +} + + +/*! Schedules the page writer to write back the specified \a page. + Note, however, that it might not do this immediately, and it can well + take several seconds until the page is actually written out. +*/ +void +vm_page_schedule_write_page(vm_page *page) +{ + ASSERT(page->State() == PAGE_STATE_MODIFIED); + + VMPageQueue* queue = NULL; + vm_page_requeue(page, false, &queue); + + ((ModifiedPageQueue*)queue)->NotifyWriter(); +} + + +/*! Cache must be locked. +*/ +void +vm_page_schedule_write_page_range(struct VMCache *cache, uint32 firstPage, + uint32 endPage) +{ + VMPageQueue* queue = NULL; + + uint32 modified = 0; + for (VMCachePagesTree::Iterator it + = cache->pages.GetIterator(firstPage, true, true); + vm_page *page = it.Next();) { + if (page->cache_offset >= endPage) + break; + + if (!page->busy && page->State() == PAGE_STATE_MODIFIED) { + DEBUG_PAGE_ACCESS_START(page); + vm_page_requeue(page, false, &queue); + modified++; + DEBUG_PAGE_ACCESS_END(page); + } + } + + if (modified > 0) + ((ModifiedPageQueue*)queue)->NotifyWriter(); +} + + +status_t +ModifiedPageQueue::StartWriter() +{ + fPageWriterCondition.Init("page writer"); + + fWriterThread = spawn_kernel_thread(&_WriterThreadEntry, "page writer", + B_NORMAL_PRIORITY + 1, this); + if (fWriterThread < 0) + return fWriterThread; + + return resume_thread(fWriterThread); +}