From 40bb94819e6c39d72ab29edc1a0dcd80b15b8b42 Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Wed, 3 Feb 2010 18:53:52 +0000 Subject: [PATCH] * Removed useless return parameter from vm_remove_all_page_mappings(). * Added vm_clear_page_mapping_accessed_flags() and vm_remove_all_page_mappings_if_unaccessed(), which combine the functionality of vm_test_map_activation(), vm_clear_map_flags(), and vm_remove_all_page_mappings(), thus saving lots of calls to translation map methods. The backend is the new method VMTranslationMap::ClearAccessedAndModified(). * Started to make use of the cached page queue and changed the meaning of the other non-free queues slightly: - Active queue: Contains mapped pages that have been used recently. - Inactive queue: Contains mapped pages that have not been used recently. Also contains unmapped temporary pages. - Modified queue: Contains unmapped modified pages. - Cached queue: Contains unmapped unmodified pages (LRU sorted). Unless we're actually low on memory and actively do paging, modified and cached queues only contain non-temporary pages. Cached pages are considered quasi free. They still belong to a cache, but since they are unmodified and unmapped, they can be freed immediately. And this is what vm_page_[try_]reserve_pages() do now when there are no more actually free pages at hand. Essentially this means that pages storing cached file data, unless mmap()ped, no longer are considered used and don't contribute to page pressure. Paging will not happen as long there are enough free + cached pages available. * Reimplemented the page daemon. It no longer scans all pages, but instead works the page queues. As long as the free pages situation is harmless, it only iterates through the active queue and deactivates pages that have not been used recently. When paging occurs it additionally scans the inactive queue and frees pages that have not been used recently. * Changed the page reservation/allocation interface: vm_page_[try_]reserve_pages(), vm_page_unreserve_pages(), and vm_page_allocate_page() now take a vm_page_reservation structure pointer. The reservation functions initialize the structure -- currently consisting only of a count member for the number of still reserved pages. vm_page_allocate_page() decrements the count and vm_page_unreserve_pages() unreserves the remaining pages (if any). Advantages are that reservation/ unreservation mismatches cannot occur anymore, that vm_page_allocate_page() can verify that the caller has indeed a reserved page left, and that there's no unnecessary pressure on the free page pool anymore. The only disadvantage is that the vm_page_reservation object needs to be passed around a bit. * Reworked the page reservation implementation: - Got rid of sSystemReservedPages and sPageDeficit. Instead sUnreservedFreePages now actually contains the number of free pages that have not yet been reserved (it cannot become negative anymore) and the new sUnsatisfiedPageReservations contains the number of pages that are still needed for reservation. - Threads waiting for reservations do now add themselves to a waiter queue, which is ordered by descending priority (VM priority and thread priority). High priority waiters are served first when pages become available. Fixes #5328. * cache_prefetch_vnode(): Would reserve one less page than allocated later, if the size wasn't page aligned. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@35393 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/private/kernel/vm/VMTranslationMap.h | 14 +- headers/private/kernel/vm/vm.h | 5 +- headers/private/kernel/vm/vm_page.h | 17 +- headers/private/kernel/vm/vm_priv.h | 1 - headers/private/kernel/vm/vm_types.h | 2 +- .../kernel/bus_managers/agp_gart/agp_gart.cpp | 7 +- .../arch/ppc/arch_vm_translation_map.cpp | 82 +- .../kernel/arch/x86/X86VMTranslationMap.h | 13 +- .../arch/x86/arch_vm_translation_map.cpp | 181 ++- src/system/kernel/cache/file_cache.cpp | 112 +- src/system/kernel/low_resource_manager.cpp | 3 - src/system/kernel/slab/MemoryManager.cpp | 11 +- src/system/kernel/vm/Jamfile | 2 +- src/system/kernel/vm/PageCacheLocker.h | 30 +- src/system/kernel/vm/VMCache.cpp | 3 +- src/system/kernel/vm/VMTranslationMap.cpp | 7 +- src/system/kernel/vm/vm.cpp | 253 +-- src/system/kernel/vm/vm_daemons.cpp | 473 ------ src/system/kernel/vm/vm_page.cpp | 1439 ++++++++++++----- 19 files changed, 1588 insertions(+), 1067 deletions(-) delete mode 100644 src/system/kernel/vm/vm_daemons.cpp diff --git a/headers/private/kernel/vm/VMTranslationMap.h b/headers/private/kernel/vm/VMTranslationMap.h index 7cdf8bff79..01fc14926b 100644 --- a/headers/private/kernel/vm/VMTranslationMap.h +++ b/headers/private/kernel/vm/VMTranslationMap.h @@ -14,6 +14,7 @@ struct kernel_args; +struct vm_page_reservation; struct VMArea; @@ -32,13 +33,15 @@ struct VMTranslationMap { virtual status_t Map(addr_t virtualAddress, addr_t physicalAddress, - uint32 attributes) = 0; + uint32 attributes, + vm_page_reservation* reservation) = 0; virtual status_t Unmap(addr_t start, addr_t end) = 0; // map not locked - virtual status_t UnmapPage(VMArea* area, addr_t address) = 0; + virtual status_t UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue) = 0; virtual void UnmapPages(VMArea* area, addr_t base, - size_t size); + size_t size, bool updatePageQueue); virtual void UnmapArea(VMArea* area, bool deletingAddressSpace, bool ignoreTopCachePageFlags); @@ -60,6 +63,11 @@ struct VMTranslationMap { virtual status_t ClearFlags(addr_t virtualAddress, uint32 flags) = 0; + virtual bool ClearAccessedAndModified( + VMArea* area, addr_t address, + bool unmapIfUnaccessed, + bool& _modified) = 0; + virtual void Flush() = 0; protected: diff --git a/headers/private/kernel/vm/vm.h b/headers/private/kernel/vm/vm.h index 096ed7a286..7149edd3da 100644 --- a/headers/private/kernel/vm/vm.h +++ b/headers/private/kernel/vm/vm.h @@ -109,9 +109,10 @@ status_t vm_create_vnode_cache(struct vnode *vnode, struct VMCache **_cache); status_t vm_set_area_memory_type(area_id id, addr_t physicalBase, uint32 type); status_t vm_get_page_mapping(team_id team, addr_t vaddr, addr_t *paddr); bool vm_test_map_modification(struct vm_page *page); -int32 vm_test_map_activation(struct vm_page *page, bool *_modified); void vm_clear_map_flags(struct vm_page *page, uint32 flags); -void vm_remove_all_page_mappings(struct vm_page *page, uint32 *_flags); +void vm_remove_all_page_mappings(struct vm_page *page); +int32 vm_clear_page_mapping_accessed_flags(struct vm_page *page); +int32 vm_remove_all_page_mappings_if_unaccessed(struct vm_page *page); status_t vm_get_physical_page(addr_t paddr, addr_t* vaddr, void** _handle); status_t vm_put_physical_page(addr_t vaddr, void* handle); diff --git a/headers/private/kernel/vm/vm_page.h b/headers/private/kernel/vm/vm_page.h index df436c0a91..80279517a3 100644 --- a/headers/private/kernel/vm/vm_page.h +++ b/headers/private/kernel/vm/vm_page.h @@ -16,6 +16,12 @@ struct kernel_args; extern int32 gMappedPagesCount; + +struct vm_page_reservation { + uint32 count; +}; + + #ifdef __cplusplus extern "C" { #endif @@ -45,11 +51,14 @@ void vm_page_schedule_write_page(struct vm_page *page); void vm_page_schedule_write_page_range(struct VMCache *cache, uint32 firstPage, uint32 endPage); -void vm_page_unreserve_pages(uint32 count); -void vm_page_reserve_pages(uint32 count, int priority); -bool vm_page_try_reserve_pages(uint32 count, int priority); +void vm_page_unreserve_pages(vm_page_reservation* reservation); +void vm_page_reserve_pages(vm_page_reservation* reservation, uint32 count, + int priority); +bool vm_page_try_reserve_pages(vm_page_reservation* reservation, uint32 count, + int priority); -struct vm_page *vm_page_allocate_page(uint32 flags); +struct vm_page *vm_page_allocate_page(vm_page_reservation* reservation, + uint32 flags); struct vm_page *vm_page_allocate_page_run(uint32 flags, addr_t base, addr_t length, int priority); struct vm_page *vm_page_allocate_page_run_no_base(uint32 flags, addr_t count, diff --git a/headers/private/kernel/vm/vm_priv.h b/headers/private/kernel/vm/vm_priv.h index b176a173d0..afb15a714a 100644 --- a/headers/private/kernel/vm/vm_priv.h +++ b/headers/private/kernel/vm/vm_priv.h @@ -31,7 +31,6 @@ status_t vm_page_fault(addr_t address, addr_t faultAddress, bool isWrite, bool isUser, addr_t *newip); void vm_unreserve_memory(size_t bytes); status_t vm_try_reserve_memory(size_t bytes, int priority, bigtime_t timeout); -void vm_schedule_page_scanner(uint32 target); status_t vm_daemon_init(void); const char *page_state_to_string(int state); diff --git a/headers/private/kernel/vm/vm_types.h b/headers/private/kernel/vm/vm_types.h index c685695028..178f196e7a 100644 --- a/headers/private/kernel/vm/vm_types.h +++ b/headers/private/kernel/vm/vm_types.h @@ -113,7 +113,7 @@ public: bool modified : 1; uint8 unused : 1; - int8 usage_count; + uint8 usage_count; uint16 wired_count; VMCacheRef* CacheRef() const { return cache_ref; } diff --git a/src/add-ons/kernel/bus_managers/agp_gart/agp_gart.cpp b/src/add-ons/kernel/bus_managers/agp_gart/agp_gart.cpp index 69b5220a08..0833823b88 100644 --- a/src/add-ons/kernel/bus_managers/agp_gart/agp_gart.cpp +++ b/src/add-ons/kernel/bus_managers/agp_gart/agp_gart.cpp @@ -550,12 +550,13 @@ Aperture::AllocateMemory(aperture_memory *memory, uint32 flags) if (memory->pages == NULL) return B_NO_MEMORY; - vm_page_reserve_pages(count, VM_PRIORITY_SYSTEM); + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, count, VM_PRIORITY_SYSTEM); for (uint32 i = 0; i < count; i++) { - memory->pages[i] = vm_page_allocate_page( + memory->pages[i] = vm_page_allocate_page(&reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); } - vm_page_unreserve_pages(count); + vm_page_unreserve_pages(&reservation); } #ifdef DEBUG_PAGE_ACCESS diff --git a/src/system/kernel/arch/ppc/arch_vm_translation_map.cpp b/src/system/kernel/arch/ppc/arch_vm_translation_map.cpp index a0c79aacc4..5f38346c03 100644 --- a/src/system/kernel/arch/ppc/arch_vm_translation_map.cpp +++ b/src/system/kernel/arch/ppc/arch_vm_translation_map.cpp @@ -147,10 +147,12 @@ struct PPCVMTranslationMap : VMTranslationMap { virtual status_t Map(addr_t virtualAddress, addr_t physicalAddress, - uint32 attributes); + uint32 attributes, + vm_page_reservation* reservation); virtual status_t Unmap(addr_t start, addr_t end); - virtual status_t UnmapPage(VMArea* area, addr_t address); + virtual status_t UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue); virtual status_t Query(addr_t virtualAddress, addr_t* _physicalAddress, @@ -164,6 +166,11 @@ struct PPCVMTranslationMap : VMTranslationMap { virtual status_t ClearFlags(addr_t virtualAddress, uint32 flags); + virtual bool ClearAccessedAndModified( + VMArea* area, addr_t address, + bool unmapIfUnaccessed, + bool& _modified); + virtual void Flush(); protected: @@ -389,7 +396,7 @@ PPCVMTranslationMap::MaxPagesNeededToMap(addr_t start, addr_t end) const status_t PPCVMTranslationMap::Map(addr_t virtualAddress, addr_t physicalAddress, - uint32 attributes) + uint32 attributes, vm_page_reservation* reservation) { // lookup the vsid based off the va uint32 virtualSegmentID = VADDR_TO_VSID(fVSIDBase, virtualAddress); @@ -463,7 +470,8 @@ PPCVMTranslationMap::Unmap(addr_t start, addr_t end) status_t -PPCVMTranslationMap::UnmapPage(VMArea* area, addr_t address) +PPCVMTranslationMap::UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue) { ASSERT(address % B_PAGE_SIZE == 0); @@ -513,10 +521,20 @@ PPCVMTranslationMap::UnmapPage(VMArea* area, addr_t address) } else page->wired_count--; - if (page->wired_count == 0 && page->mappings.IsEmpty()) + locker.Unlock(); + + if (page->wired_count == 0 && page->mappings.IsEmpty()) { atomic_add(&gMappedPagesCount, -1); - locker.Unlock(); + if (updatePageQueue) { + if (page->Cache()->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + } + } if (mapping != NULL) { bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); @@ -604,6 +622,52 @@ PPCVMTranslationMap::ClearFlags(addr_t virtualAddress, uint32 flags) } +bool +PPCVMTranslationMap::ClearAccessedAndModified(VMArea* area, addr_t address, + bool unmapIfUnaccessed, bool& _modified) +{ + // TODO: Implement for real! ATM this is just an approximation using + // Query(), ClearFlags(), and UnmapPage(). See below! + + RecursiveLocker locker(fLock); + + uint32 flags; + addr_t physicalAddress; + if (Query(address, &physicalAddress, &flags) != B_OK + || (flags & PAGE_PRESENT) == 0) { + return false; + } + + _modified = (flags & PAGE_MODIFIED) != 0; + + if ((flags & (PAGE_ACCESSED | PAGE_MODIFIED)) != 0) + ClearFlags(address, flags & (PAGE_ACCESSED | PAGE_MODIFIED)); + + if ((flags & PAGE_ACCESSED) != 0) + return true; + + if (!unmapIfUnaccessed) + return false; + + locker.Unlock(); + + UnmapPage(area, address, false); + // TODO: Obvious race condition: Between querying and unmapping the + // page could have been accessed. We try to compensate by considering + // vm_page::{accessed,modified} (which would have been updated by + // UnmapPage()) below, but that doesn't quite match the required + // semantics of the method. + + vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE); + if (page == NULL) + return false; + + _modified |= page->modified; + + return page->accessed; +} + + void PPCVMTranslationMap::Flush() { @@ -786,11 +850,15 @@ ppc_map_address_range(addr_t virtualAddress, addr_t physicalAddress, PPCVMTranslationMap* map = static_cast( addressSpace->TranslationMap()); + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, 0, VM_PRIORITY_USER); + // We don't need any pages for mapping. + // map the pages for (; virtualAddress < virtualEnd; virtualAddress += B_PAGE_SIZE, physicalAddress += B_PAGE_SIZE) { status_t error = map->Map(virtualAddress, physicalAddress, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, &reservation); if (error != B_OK) return error; } diff --git a/src/system/kernel/arch/x86/X86VMTranslationMap.h b/src/system/kernel/arch/x86/X86VMTranslationMap.h index 8a63c5918b..2310d8aa38 100644 --- a/src/system/kernel/arch/x86/X86VMTranslationMap.h +++ b/src/system/kernel/arch/x86/X86VMTranslationMap.h @@ -31,12 +31,14 @@ struct X86VMTranslationMap : VMTranslationMap { virtual status_t Map(addr_t virtualAddress, addr_t physicalAddress, - uint32 attributes); + uint32 attributes, + vm_page_reservation* reservation); virtual status_t Unmap(addr_t start, addr_t end); - virtual status_t UnmapPage(VMArea* area, addr_t address); + virtual status_t UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue); virtual void UnmapPages(VMArea* area, addr_t base, - size_t size); + size_t size, bool updatePageQueue); virtual void UnmapArea(VMArea* area, bool deletingAddressSpace, bool ignoreTopCachePageFlags); @@ -54,6 +56,11 @@ struct X86VMTranslationMap : VMTranslationMap { virtual status_t ClearFlags(addr_t virtualAddress, uint32 flags); + virtual bool ClearAccessedAndModified( + VMArea* area, addr_t address, + bool unmapIfUnaccessed, + bool& _modified); + virtual void Flush(); protected: diff --git a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp index a551d0b096..18a080c88e 100644 --- a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp +++ b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp @@ -401,7 +401,8 @@ X86VMTranslationMap::MaxPagesNeededToMap(addr_t start, addr_t end) const status_t -X86VMTranslationMap::Map(addr_t va, addr_t pa, uint32 attributes) +X86VMTranslationMap::Map(addr_t va, addr_t pa, uint32 attributes, + vm_page_reservation* reservation) { TRACE(("map_tmap: entry pa 0x%lx va 0x%lx\n", pa, va)); @@ -422,7 +423,8 @@ X86VMTranslationMap::Map(addr_t va, addr_t pa, uint32 attributes) vm_page *page; // we need to allocate a pgtable - page = vm_page_allocate_page(PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); + page = vm_page_allocate_page(reservation, + PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); DEBUG_PAGE_ACCESS_END(page); @@ -533,7 +535,8 @@ restart: This object shouldn't be locked. */ status_t -X86VMTranslationMap::UnmapPage(VMArea* area, addr_t address) +X86VMTranslationMap::UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue) { ASSERT(address % B_PAGE_SIZE == 0); @@ -620,10 +623,20 @@ X86VMTranslationMap::UnmapPage(VMArea* area, addr_t address) } else page->wired_count--; - if (page->wired_count == 0 && page->mappings.IsEmpty()) + locker.Unlock(); + + if (page->wired_count == 0 && page->mappings.IsEmpty()) { atomic_add(&gMappedPagesCount, -1); - locker.Unlock(); + if (updatePageQueue) { + if (page->Cache()->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + } + } if (mapping != NULL) { bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); @@ -637,7 +650,8 @@ X86VMTranslationMap::UnmapPage(VMArea* area, addr_t address) void -X86VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size) +X86VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size, + bool updatePageQueue) { page_directory_entry* pd = fArchData->pgdir_virt; @@ -720,13 +734,23 @@ X86VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size) } else page->wired_count--; - if (page->wired_count == 0 && page->mappings.IsEmpty()) + if (page->wired_count == 0 && page->mappings.IsEmpty()) { atomic_add(&gMappedPagesCount, -1); + + if (updatePageQueue) { + if (page->Cache()->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + } + } } } Flush(); - // flush explicitely, since we directly use the lock + // flush explicitly, since we directly use the lock pinner.Unlock(); } @@ -752,7 +776,7 @@ X86VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace, bool ignoreTopCachePageFlags) { if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - X86VMTranslationMap::UnmapPages(area, area->Base(), area->Size()); + X86VMTranslationMap::UnmapPages(area, area->Base(), area->Size(), true); return; } @@ -770,10 +794,22 @@ X86VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace, vm_page* page = mapping->page; page->mappings.Remove(mapping); - if (page->wired_count == 0 && page->mappings.IsEmpty()) + VMCache* cache = page->Cache(); + + if (page->wired_count == 0 && page->mappings.IsEmpty()) { atomic_add(&gMappedPagesCount, -1); - if (unmapPages || page->Cache() != area->cache) { + if (!ignoreTopCachePageFlags || cache != area->cache) { + if (cache->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + } + } + + if (unmapPages || cache != area->cache) { addr_t address = area->Base() + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); @@ -1040,6 +1076,129 @@ X86VMTranslationMap::ClearFlags(addr_t va, uint32 flags) } +bool +X86VMTranslationMap::ClearAccessedAndModified(VMArea* area, addr_t address, + bool unmapIfUnaccessed, bool& _modified) +{ + ASSERT(address % B_PAGE_SIZE == 0); + + page_directory_entry* pd = fArchData->pgdir_virt; + + TRACE(("X86VMTranslationMap::ClearAccessedAndModified(%#" B_PRIxADDR ")\n", + address)); + + RecursiveLocker locker(fLock); + + int index = VADDR_TO_PDENT(address); + if ((pd[index] & X86_PDE_PRESENT) == 0) + return false; + + ThreadCPUPinner pinner(thread_get_current_thread()); + + page_table_entry* pt = fArchData->page_mapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + + index = VADDR_TO_PTENT(address); + + // perform the deed + page_table_entry oldEntry; + + if (unmapIfUnaccessed) { + while (true) { + oldEntry = pt[index]; + if ((oldEntry & X86_PTE_PRESENT) == 0) { + // page mapping not valid + return false; + } + + if (oldEntry & X86_PTE_ACCESSED) { + // page was accessed -- just clear the flags + oldEntry = clear_page_table_entry_flags(&pt[index], + X86_PTE_ACCESSED | X86_PTE_DIRTY); + break; + } + + // page hasn't been accessed -- unmap it + if (test_and_set_page_table_entry(&pt[index], 0, oldEntry) + == oldEntry) { + break; + } + + // something changed -- check again + } + } else { + oldEntry = clear_page_table_entry_flags(&pt[index], + X86_PTE_ACCESSED | X86_PTE_DIRTY); + } + + pinner.Unlock(); + + _modified = (oldEntry & X86_PTE_DIRTY) != 0; + + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + // Note, that we only need to invalidate the address, if the + // accessed flags was set, since only then the entry could have been + // in any TLB. + if (fArchData->num_invalidate_pages + < PAGE_INVALIDATE_CACHE_SIZE) { + fArchData->pages_to_invalidate[fArchData->num_invalidate_pages] + = address; + } + + fArchData->num_invalidate_pages++; + + Flush(); + + return true; + } + + if (!unmapIfUnaccessed) + return false; + + // We have unmapped the address. Do the "high level" stuff. + + fMapCount--; + + if (area->cache_type == CACHE_TYPE_DEVICE) + return false; + + // get the page + vm_page* page = vm_lookup_page( + (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); + ASSERT(page != NULL); + + // remove the mapping object/decrement the wired_count of the page + vm_page_mapping* mapping = NULL; + if (area->wiring == B_NO_LOCK) { + vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); + while ((mapping = iterator.Next()) != NULL) { + if (mapping->area == area) { + area->mappings.Remove(mapping); + page->mappings.Remove(mapping); + break; + } + } + + ASSERT(mapping != NULL); + } else + page->wired_count--; + + locker.Unlock(); + + if (page->wired_count == 0 && page->mappings.IsEmpty()) + atomic_add(&gMappedPagesCount, -1); + + if (mapping != NULL) { + object_cache_free(gPageMappingsObjectCache, mapping, + CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); + // Since this is called by the page daemon, we never want to lock + // the kernel address space. + } + + return false; +} + + void X86VMTranslationMap::Flush() { diff --git a/src/system/kernel/cache/file_cache.cpp b/src/system/kernel/cache/file_cache.cpp index 9aad7b7115..71340d81ba 100644 --- a/src/system/kernel/cache/file_cache.cpp +++ b/src/system/kernel/cache/file_cache.cpp @@ -75,7 +75,7 @@ public: size_t size); ~PrecacheIO(); - status_t Prepare(); + status_t Prepare(vm_page_reservation* reservation); void ReadAsync(); virtual void IOFinished(status_t status, @@ -99,7 +99,7 @@ private: typedef status_t (*cache_func)(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer, - size_t lastReservedPages, size_t reservePages); + vm_page_reservation* reservation, size_t reservePages); static void add_to_iovec(iovec* vecs, uint32 &index, uint32 max, addr_t address, size_t size); @@ -141,7 +141,7 @@ PrecacheIO::~PrecacheIO() status_t -PrecacheIO::Prepare() +PrecacheIO::Prepare(vm_page_reservation* reservation) { if (fPageCount == 0) return B_BAD_VALUE; @@ -157,8 +157,8 @@ PrecacheIO::Prepare() // allocate pages for the cache and mark them busy uint32 i = 0; for (size_t pos = 0; pos < fSize; pos += B_PAGE_SIZE) { - vm_page* page = vm_page_allocate_page( - PAGE_STATE_ACTIVE | VM_PAGE_ALLOC_BUSY); + vm_page* page = vm_page_allocate_page(reservation, + PAGE_STATE_CACHED | VM_PAGE_ALLOC_BUSY); fCache->InsertPage(page, fOffset + pos); @@ -281,7 +281,8 @@ push_access(file_cache_ref* ref, off_t offset, size_t bytes, bool isWrite) static void -reserve_pages(file_cache_ref* ref, size_t reservePages, bool isWrite) +reserve_pages(file_cache_ref* ref, vm_page_reservation* reservation, + size_t reservePages, bool isWrite) { if (low_resource_state(B_KERNEL_RESOURCE_PAGES) != B_NO_LOW_RESOURCE) { VMCache* cache = ref->cache; @@ -320,7 +321,7 @@ reserve_pages(file_cache_ref* ref, size_t reservePages, bool isWrite) cache->Unlock(); } - vm_page_reserve_pages(reservePages, VM_PRIORITY_USER); + vm_page_reserve_pages(reservation, reservePages, VM_PRIORITY_USER); } @@ -367,7 +368,7 @@ read_pages_and_clear_partial(file_cache_ref* ref, void* cookie, off_t offset, static status_t read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer, - size_t lastReservedPages, size_t reservePages) + vm_page_reservation* reservation, size_t reservePages) { TRACE(("read_into_cache(offset = %Ld, pageOffset = %ld, buffer = %#lx, " "bufferSize = %lu\n", offset, pageOffset, buffer, bufferSize)); @@ -386,7 +387,7 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, // allocate pages for the cache and mark them busy for (size_t pos = 0; pos < numBytes; pos += B_PAGE_SIZE) { vm_page* page = pages[pageIndex++] = vm_page_allocate_page( - PAGE_STATE_ACTIVE | VM_PAGE_ALLOC_BUSY); + reservation, PAGE_STATE_CACHED | VM_PAGE_ALLOC_BUSY); cache->InsertPage(page, offset + pos); @@ -397,7 +398,7 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, push_access(ref, offset, bufferSize, false); cache->Unlock(); - vm_page_unreserve_pages(lastReservedPages); + vm_page_unreserve_pages(reservation); // read file into reserved pages status_t status = read_pages_and_clear_partial(ref, cookie, offset, vecs, @@ -434,7 +435,7 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, } } - reserve_pages(ref, reservePages, false); + reserve_pages(ref, reservation, reservePages, false); cache->Lock(); // make the pages accessible in the cache @@ -451,7 +452,7 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, static status_t read_from_file(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer, - size_t lastReservedPages, size_t reservePages) + vm_page_reservation* reservation, size_t reservePages) { TRACE(("read_from_file(offset = %Ld, pageOffset = %ld, buffer = %#lx, " "bufferSize = %lu\n", offset, pageOffset, buffer, bufferSize)); @@ -465,13 +466,13 @@ read_from_file(file_cache_ref* ref, void* cookie, off_t offset, push_access(ref, offset, bufferSize, false); ref->cache->Unlock(); - vm_page_unreserve_pages(lastReservedPages); + vm_page_unreserve_pages(reservation); status_t status = vfs_read_pages(ref->vnode, cookie, offset + pageOffset, &vec, 1, 0, &bufferSize); if (status == B_OK) - reserve_pages(ref, reservePages, false); + reserve_pages(ref, reservation, reservePages, false); ref->cache->Lock(); @@ -487,7 +488,7 @@ read_from_file(file_cache_ref* ref, void* cookie, off_t offset, static status_t write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer, - size_t lastReservedPages, size_t reservePages) + vm_page_reservation* reservation, size_t reservePages) { // TODO: We're using way too much stack! Rather allocate a sufficiently // large chunk on the heap. @@ -509,7 +510,9 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, // TODO: the pages we allocate here should have been reserved upfront // in cache_io() vm_page* page = pages[pageIndex++] = vm_page_allocate_page( - PAGE_STATE_ACTIVE | VM_PAGE_ALLOC_BUSY); + reservation, + (writeThrough ? PAGE_STATE_CACHED : PAGE_STATE_MODIFIED) + | VM_PAGE_ALLOC_BUSY); ref->cache->InsertPage(page, offset + pos); @@ -519,7 +522,7 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, push_access(ref, offset, bufferSize, true); ref->cache->Unlock(); - vm_page_unreserve_pages(lastReservedPages); + vm_page_unreserve_pages(reservation); // copy contents (and read in partially written pages first) @@ -601,7 +604,7 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, } if (status == B_OK) - reserve_pages(ref, reservePages, true); + reserve_pages(ref, reservation, reservePages, true); ref->cache->Lock(); @@ -609,9 +612,6 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, for (int32 i = pageIndex; i-- > 0;) { ref->cache->MarkPageUnbusy(pages[i]); - if (!writeThrough) - vm_page_set_state(pages[i], PAGE_STATE_MODIFIED); - DEBUG_PAGE_ACCESS_END(pages[i]); } @@ -621,12 +621,12 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, static status_t write_to_file(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, - addr_t buffer, size_t bufferSize, bool useBuffer, size_t lastReservedPages, - size_t reservePages) + addr_t buffer, size_t bufferSize, bool useBuffer, + vm_page_reservation* reservation, size_t reservePages) { push_access(ref, offset, bufferSize, true); ref->cache->Unlock(); - vm_page_unreserve_pages(lastReservedPages); + vm_page_unreserve_pages(reservation); status_t status = B_OK; @@ -652,7 +652,7 @@ write_to_file(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, } if (status == B_OK) - reserve_pages(ref, reservePages, true); + reserve_pages(ref, reservation, reservePages, true); ref->cache->Lock(); @@ -665,7 +665,7 @@ satisfy_cache_io(file_cache_ref* ref, void* cookie, cache_func function, off_t offset, addr_t buffer, bool useBuffer, int32 &pageOffset, size_t bytesLeft, size_t &reservePages, off_t &lastOffset, addr_t &lastBuffer, int32 &lastPageOffset, size_t &lastLeft, - size_t &lastReservedPages) + size_t &lastReservedPages, vm_page_reservation* reservation) { if (lastBuffer == buffer) return B_OK; @@ -675,7 +675,7 @@ satisfy_cache_io(file_cache_ref* ref, void* cookie, cache_func function, + lastPageOffset + B_PAGE_SIZE - 1) >> PAGE_SHIFT); status_t status = function(ref, cookie, lastOffset, lastPageOffset, - lastBuffer, requestSize, useBuffer, lastReservedPages, reservePages); + lastBuffer, requestSize, useBuffer, reservation, reservePages); if (status == B_OK) { lastReservedPages = reservePages; lastBuffer = buffer; @@ -736,7 +736,9 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, size_t pagesProcessed = 0; cache_func function = NULL; - reserve_pages(ref, lastReservedPages, doWrite); + vm_page_reservation reservation; + reserve_pages(ref, &reservation, lastReservedPages, doWrite); + AutoLocker locker(cache); while (bytesLeft > 0) { @@ -763,7 +765,7 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, status_t status = satisfy_cache_io(ref, cookie, function, offset, buffer, useBuffer, pageOffset, bytesLeft, reservePages, lastOffset, lastBuffer, lastPageOffset, lastLeft, - lastReservedPages); + lastReservedPages, &reservation); if (status != B_OK) return status; @@ -779,13 +781,6 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, "= %lu\n", offset, page, bytesLeft, pageOffset)); if (page != NULL) { - // Since we don't actually map pages as part of an area, we have - // to manually maintain their usage_count - page->usage_count = 2; - // TODO: Just because this request comes from the FS API, it - // doesn't mean the page is not mapped. We might actually - // decrease the usage count of a hot page here. - if (doWrite || useBuffer) { // Since the following user_mem{cpy,set}() might cause a page // fault, which in turn might cause pages to be reserved, we @@ -824,10 +819,19 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, if (bytesLeft <= bytesInPage) { // we've read the last page, so we're done! locker.Unlock(); - vm_page_unreserve_pages(lastReservedPages); + vm_page_unreserve_pages(&reservation); return B_OK; } + // If it is cached only, requeue the page, so the respective queue + // roughly remains LRU first sorted. + if (page->state == PAGE_STATE_CACHED + || page->state == PAGE_STATE_MODIFIED) { + DEBUG_PAGE_ACCESS_START(page); + vm_page_requeue(page, true); + DEBUG_PAGE_ACCESS_END(page); + } + // prepare a potential gap request lastBuffer = buffer + bytesInPage; lastLeft = bytesLeft - bytesInPage; @@ -848,7 +852,7 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, status_t status = satisfy_cache_io(ref, cookie, function, offset, buffer, useBuffer, pageOffset, bytesLeft, reservePages, lastOffset, lastBuffer, lastPageOffset, lastLeft, - lastReservedPages); + lastReservedPages, &reservation); if (status != B_OK) return status; } @@ -857,7 +861,7 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, // fill the last remaining bytes of the request (either write or read) return function(ref, cookie, lastOffset, lastPageOffset, lastBuffer, - lastLeft, useBuffer, lastReservedPages, 0); + lastLeft, useBuffer, &reservation, 0); } @@ -929,6 +933,11 @@ cache_prefetch_vnode(struct vnode* vnode, off_t offset, size_t size) if (offset + size > fileSize) size = fileSize - offset; + + // "offset" and "size" are always aligned to B_PAGE_SIZE, + offset = ROUNDDOWN(offset, B_PAGE_SIZE); + size = ROUNDUP(size, B_PAGE_SIZE); + size_t reservePages = size / B_PAGE_SIZE; // Don't do anything if we don't have the resources left, or the cache @@ -939,14 +948,11 @@ cache_prefetch_vnode(struct vnode* vnode, off_t offset, size_t size) return; } - // "offset" and "size" are always aligned to B_PAGE_SIZE, - offset &= ~(B_PAGE_SIZE - 1); - size = ROUNDUP(size, B_PAGE_SIZE); - size_t bytesToRead = 0; off_t lastOffset = offset; - vm_page_reserve_pages(reservePages, VM_PRIORITY_USER); + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, reservePages, VM_PRIORITY_USER); cache->Lock(); @@ -965,9 +971,9 @@ cache_prefetch_vnode(struct vnode* vnode, off_t offset, size_t size) } if (bytesToRead != 0) { // read the part before the current page (or the end of the request) - PrecacheIO* io - = new(std::nothrow) PrecacheIO(ref, lastOffset, bytesToRead); - if (io == NULL || io->Prepare() != B_OK) { + PrecacheIO* io = new(std::nothrow) PrecacheIO(ref, lastOffset, + bytesToRead); + if (io == NULL || io->Prepare(&reservation) != B_OK) { delete io; break; } @@ -989,9 +995,7 @@ cache_prefetch_vnode(struct vnode* vnode, off_t offset, size_t size) } cache->ReleaseRefAndUnlock(); - vm_page_unreserve_pages(reservePages); - // TODO: We should periodically unreserve as we go, so we don't - // unnecessarily put pressure on the free page pool. + vm_page_unreserve_pages(&reservation); } @@ -1074,10 +1078,11 @@ extern "C" status_t file_cache_init(void) { // allocate a clean page we can use for writing zeroes - vm_page_reserve_pages(1, VM_PRIORITY_SYSTEM); - vm_page* page = vm_page_allocate_page( + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, 1, VM_PRIORITY_SYSTEM); + vm_page* page = vm_page_allocate_page(&reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); - vm_page_unreserve_pages(1); + vm_page_unreserve_pages(&reservation); sZeroPage = (addr_t)page->physical_page_number * B_PAGE_SIZE; @@ -1319,4 +1324,3 @@ file_cache_write(void* _cacheRef, void* cookie, off_t offset, return status; } - diff --git a/src/system/kernel/low_resource_manager.cpp b/src/system/kernel/low_resource_manager.cpp index 241c540bf6..3647ae77d9 100644 --- a/src/system/kernel/low_resource_manager.cpp +++ b/src/system/kernel/low_resource_manager.cpp @@ -301,9 +301,6 @@ low_resource(uint32 resource, uint64 requirements, uint32 flags, uint32 timeout) switch (resource) { case B_KERNEL_RESOURCE_PAGES: - vm_schedule_page_scanner(requirements); - break; - case B_KERNEL_RESOURCE_MEMORY: case B_KERNEL_RESOURCE_SEMAPHORES: case B_KERNEL_RESOURCE_ADDRESS_SPACE: diff --git a/src/system/kernel/slab/MemoryManager.cpp b/src/system/kernel/slab/MemoryManager.cpp index 0bb76db760..ff6d1f789e 100644 --- a/src/system/kernel/slab/MemoryManager.cpp +++ b/src/system/kernel/slab/MemoryManager.cpp @@ -1333,13 +1333,14 @@ MemoryManager::_MapChunk(VMArea* vmArea, addr_t address, size_t size, // reserve the pages we need now size_t reservedPages = size / B_PAGE_SIZE + translationMap->MaxPagesNeededToMap(address, address + size - 1); + vm_page_reservation reservation; if ((flags & CACHE_DONT_WAIT_FOR_MEMORY) != 0) { - if (!vm_page_try_reserve_pages(reservedPages, priority)) { + if (!vm_page_try_reserve_pages(&reservation, reservedPages, priority)) { vm_unreserve_memory(reservedMemory); return B_WOULD_BLOCK; } } else - vm_page_reserve_pages(reservedPages, priority); + vm_page_reserve_pages(&reservation, reservedPages, priority); VMCache* cache = vm_area_get_locked_cache(vmArea); @@ -1350,7 +1351,7 @@ MemoryManager::_MapChunk(VMArea* vmArea, addr_t address, size_t size, addr_t endAreaOffset = areaOffset + size; for (size_t offset = areaOffset; offset < endAreaOffset; offset += B_PAGE_SIZE) { - vm_page* page = vm_page_allocate_page(PAGE_STATE_WIRED); + vm_page* page = vm_page_allocate_page(&reservation, PAGE_STATE_WIRED); cache->InsertPage(page, offset); page->wired_count++; @@ -1359,14 +1360,14 @@ MemoryManager::_MapChunk(VMArea* vmArea, addr_t address, size_t size, translationMap->Map(vmArea->Base() + offset, page->physical_page_number * B_PAGE_SIZE, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, &reservation); } translationMap->Unlock(); cache->ReleaseRefAndUnlock(); - vm_page_unreserve_pages(reservedPages); + vm_page_unreserve_pages(&reservation); return B_OK; } diff --git a/src/system/kernel/vm/Jamfile b/src/system/kernel/vm/Jamfile index 595901e806..fa0282d658 100644 --- a/src/system/kernel/vm/Jamfile +++ b/src/system/kernel/vm/Jamfile @@ -5,8 +5,8 @@ UsePrivateHeaders shared ; UseHeaders [ FDirName $(SUBDIR) $(DOTDOT) device_manager ] ; KernelMergeObject kernel_vm.o : + PageCacheLocker.cpp vm.cpp - vm_daemons.cpp vm_page.cpp VMAddressSpace.cpp VMAddressSpaceLocking.cpp diff --git a/src/system/kernel/vm/PageCacheLocker.h b/src/system/kernel/vm/PageCacheLocker.h index c3008050bd..36a21b2c7b 100644 --- a/src/system/kernel/vm/PageCacheLocker.h +++ b/src/system/kernel/vm/PageCacheLocker.h @@ -13,18 +13,34 @@ struct vm_page; class PageCacheLocker { public: - PageCacheLocker(vm_page* page, bool dontWait = true); - ~PageCacheLocker(); + inline PageCacheLocker(vm_page* page, + bool dontWait = true); + inline ~PageCacheLocker(); - bool IsLocked() { return fPage != NULL; } + bool IsLocked() { return fPage != NULL; } - bool Lock(vm_page* page, bool dontWait = true); - void Unlock(); + bool Lock(vm_page* page, bool dontWait = true); + void Unlock(); private: - bool _IgnorePage(vm_page* page); + bool _IgnorePage(vm_page* page); - vm_page* fPage; + vm_page* fPage; }; + +PageCacheLocker::PageCacheLocker(vm_page* page, bool dontWait) + : + fPage(NULL) +{ + Lock(page, dontWait); +} + + +PageCacheLocker::~PageCacheLocker() +{ + Unlock(); +} + + #endif // PAGE_CACHE_LOCKER_H diff --git a/src/system/kernel/vm/VMCache.cpp b/src/system/kernel/vm/VMCache.cpp index 9347b8259d..07c92b6c74 100644 --- a/src/system/kernel/vm/VMCache.cpp +++ b/src/system/kernel/vm/VMCache.cpp @@ -743,7 +743,6 @@ VMCache::InsertPage(vm_page* page, off_t offset) page->cache_offset = (page_num_t)(offset >> PAGE_SHIFT); page_count++; - page->usage_count = 2; page->SetCacheRef(fCacheRef); #if KDEBUG @@ -1076,7 +1075,7 @@ VMCache::Resize(off_t newSize, int priority) // remove the page and put it into the free queue DEBUG_PAGE_ACCESS_START(page); - vm_remove_all_page_mappings(page, NULL); + vm_remove_all_page_mappings(page); ASSERT(page->wired_count == 0); // TODO: Find a real solution! Unmapping is probably fine, but // we have no way of unmapping wired pages here. diff --git a/src/system/kernel/vm/VMTranslationMap.cpp b/src/system/kernel/vm/VMTranslationMap.cpp index fce7199832..eb77f37eb0 100644 --- a/src/system/kernel/vm/VMTranslationMap.cpp +++ b/src/system/kernel/vm/VMTranslationMap.cpp @@ -32,7 +32,8 @@ VMTranslationMap::~VMTranslationMap() range and calls UnmapPage(). This is obviously not particularly efficient. */ void -VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size) +VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size, + bool updatePageQueue) { ASSERT(base % B_PAGE_SIZE == 0); ASSERT(size % B_PAGE_SIZE == 0); @@ -40,7 +41,7 @@ VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size) addr_t address = base; addr_t end = address + size; for (; address != end; address += B_PAGE_SIZE) - UnmapPage(area, address); + UnmapPage(area, address, updatePageQueue); } @@ -64,7 +65,7 @@ VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace, addr_t address = area->Base(); addr_t end = address + area->Size(); for (; address != end; address += B_PAGE_SIZE) - UnmapPage(area, address); + UnmapPage(area, address, true); } diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index dec63795f9..67b2bb9d7b 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -472,7 +472,8 @@ get_area_page_protection(VMArea* area, addr_t pageAddress) The page's cache must be locked. */ static status_t -map_page(VMArea* area, vm_page* page, addr_t address, uint32 protection) +map_page(VMArea* area, vm_page* page, addr_t address, uint32 protection, + vm_page_reservation* reservation, bool activatePage) { VMTranslationMap* map = area->address_space->TranslationMap(); @@ -492,7 +493,8 @@ map_page(VMArea* area, vm_page* page, addr_t address, uint32 protection) map->Lock(); - map->Map(address, page->physical_page_number * B_PAGE_SIZE, protection); + map->Map(address, page->physical_page_number * B_PAGE_SIZE, protection, + reservation); // insert mapping into lists if (page->mappings.IsEmpty() && page->wired_count == 0) @@ -506,14 +508,20 @@ map_page(VMArea* area, vm_page* page, addr_t address, uint32 protection) DEBUG_PAGE_ACCESS_CHECK(page); map->Lock(); - map->Map(address, page->physical_page_number * B_PAGE_SIZE, protection); + map->Map(address, page->physical_page_number * B_PAGE_SIZE, protection, + reservation); map->Unlock(); increment_page_wired_count(page); } - if (page->usage_count < 0) - page->usage_count = 1; + if (page->state == PAGE_STATE_CACHED) { + if (page->usage_count == 0 && !activatePage) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else + vm_page_set_state(page, PAGE_STATE_ACTIVE); + } else if (page->state == PAGE_STATE_INACTIVE && activatePage) + vm_page_set_state(page, PAGE_STATE_ACTIVE); return B_OK; } @@ -526,7 +534,7 @@ static inline bool unmap_page(VMArea* area, addr_t virtualAddress) { return area->address_space->TranslationMap()->UnmapPage(area, - virtualAddress); + virtualAddress, true); } @@ -536,7 +544,7 @@ unmap_page(VMArea* area, addr_t virtualAddress) static inline void unmap_pages(VMArea* area, addr_t base, size_t size) { - area->address_space->TranslationMap()->UnmapPages(area, base, size); + area->address_space->TranslationMap()->UnmapPages(area, base, size, true); } @@ -1029,15 +1037,18 @@ vm_create_anonymous_area(team_id team, const char* name, void** address, page_num_t reservedPages = reservedMapPages; if (wiring == B_FULL_LOCK) reservedPages += size / B_PAGE_SIZE; + + vm_page_reservation reservation; if (reservedPages > 0) { if ((flags & CREATE_AREA_DONT_WAIT) != 0) { - if (!vm_page_try_reserve_pages(reservedPages, priority)) { + if (!vm_page_try_reserve_pages(&reservation, reservedPages, + priority)) { reservedPages = 0; status = B_WOULD_BLOCK; goto err0; } } else - vm_page_reserve_pages(reservedPages, priority); + vm_page_reserve_pages(&reservation, reservedPages, priority); } status = locker.SetTo(team); @@ -1122,29 +1133,12 @@ vm_create_anonymous_area(team_id team, const char* name, void** address, # endif continue; #endif - vm_page* page = vm_page_allocate_page( - PAGE_STATE_ACTIVE | pageAllocFlags); -// PAGE_STATE_WIRED | pageAllocFlags); - // TODO: The pages should be PAGE_STATE_WIRED, since there's - // no need for the page daemon to play with them (the same - // should be considered in vm_soft_fault()). ATM doing that - // will result in bad thrashing in systems with little - // memory due to the current tuning of the page daemon. It - // will age pages way too fast (since it just skips - // PAGE_STATE_WIRED pages, while it processes - // PAGE_STATE_ACTIVE with wired_count > 0). + vm_page* page = vm_page_allocate_page(&reservation, + PAGE_STATE_WIRED | pageAllocFlags); cache->InsertPage(page, offset); - map_page(area, page, address, protection); + map_page(area, page, address, protection, &reservation, false); DEBUG_PAGE_ACCESS_END(page); - - // Periodically unreserve pages we've already allocated, so that - // we don't unnecessarily increase the pressure on the VM. - if (offset > 0 && offset % (128 * B_PAGE_SIZE) == 0) { - page_num_t toUnreserve = 128; - vm_page_unreserve_pages(toUnreserve); - reservedPages -= toUnreserve; - } } break; @@ -1211,7 +1205,8 @@ vm_create_anonymous_area(team_id team, const char* name, void** address, if (page == NULL) panic("couldn't lookup physical page just allocated\n"); - status = map->Map(virtualAddress, physicalAddress, protection); + status = map->Map(virtualAddress, physicalAddress, protection, + &reservation); if (status < B_OK) panic("couldn't map physical page in page run\n"); @@ -1232,7 +1227,7 @@ vm_create_anonymous_area(team_id team, const char* name, void** address, cache->Unlock(); if (reservedPages > 0) - vm_page_unreserve_pages(reservedPages); + vm_page_unreserve_pages(&reservation); TRACE(("vm_create_anonymous_area: done\n")); @@ -1255,7 +1250,7 @@ err1: err0: if (reservedPages > 0) - vm_page_unreserve_pages(reservedPages); + vm_page_unreserve_pages(&reservation); if (reservedMemory > 0) vm_unreserve_memory(reservedMemory); @@ -1326,18 +1321,19 @@ vm_map_physical_memory(team_id team, const char* name, void** _address, size_t reservePages = map->MaxPagesNeededToMap(area->Base(), area->Base() + (size - 1)); - vm_page_reserve_pages(reservePages, + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, reservePages, team == VMAddressSpace::KernelID() ? VM_PRIORITY_SYSTEM : VM_PRIORITY_USER); map->Lock(); for (addr_t offset = 0; offset < size; offset += B_PAGE_SIZE) { map->Map(area->Base() + offset, physicalAddress + offset, - protection); + protection, &reservation); } map->Unlock(); - vm_page_unreserve_pages(reservePages); + vm_page_unreserve_pages(&reservation); } if (status < B_OK) @@ -1414,7 +1410,8 @@ vm_map_physical_memory_vecs(team_id team, const char* name, void** _address, size_t reservePages = map->MaxPagesNeededToMap(area->Base(), area->Base() + (size - 1)); - vm_page_reserve_pages(reservePages, + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, reservePages, team == VMAddressSpace::KernelID() ? VM_PRIORITY_SYSTEM : VM_PRIORITY_USER); map->Lock(); @@ -1431,13 +1428,14 @@ vm_map_physical_memory_vecs(team_id team, const char* name, void** _address, break; map->Map(area->Base() + offset, - (addr_t)vecs[vecIndex].iov_base + vecOffset, protection); + (addr_t)vecs[vecIndex].iov_base + vecOffset, protection, + &reservation); vecOffset += B_PAGE_SIZE; } map->Unlock(); - vm_page_unreserve_pages(reservePages); + vm_page_unreserve_pages(&reservation); if (_size != NULL) *_size = size; @@ -1502,7 +1500,8 @@ vm_create_vnode_cache(struct vnode* vnode, struct VMCache** cache) /*! \a cache must be locked. The area's address space must be read-locked. */ static void -pre_map_area_pages(VMArea* area, VMCache* cache) +pre_map_area_pages(VMArea* area, VMCache* cache, + vm_page_reservation* reservation) { addr_t baseAddress = area->Base(); addr_t cacheOffset = area->cache_offset; @@ -1515,14 +1514,14 @@ pre_map_area_pages(VMArea* area, VMCache* cache) if (page->cache_offset >= endPage) break; - // skip inactive pages - if (page->busy || page->usage_count <= 0) + // skip busy and inactive pages + if (page->busy || page->usage_count == 0) continue; DEBUG_PAGE_ACCESS_START(page); map_page(area, page, baseAddress + (page->cache_offset * B_PAGE_SIZE - cacheOffset), - B_READ_AREA | B_KERNEL_READ_AREA); + B_READ_AREA | B_KERNEL_READ_AREA, reservation, false); DEBUG_PAGE_ACCESS_END(page); } } @@ -1584,6 +1583,7 @@ _vm_map_file(team_id team, const char* name, void** _address, // If we're going to pre-map pages, we need to reserve the pages needed by // the mapping backend upfront. page_num_t reservedPreMapPages = 0; + vm_page_reservation reservation; if ((protection & B_READ_AREA) != 0) { AddressSpaceWriteLocker locker; status = locker.SetTo(team); @@ -1595,25 +1595,26 @@ _vm_map_file(team_id team, const char* name, void** _address, locker.Unlock(); - vm_page_reserve_pages(reservedPreMapPages, + vm_page_reserve_pages(&reservation, reservedPreMapPages, team == VMAddressSpace::KernelID() ? VM_PRIORITY_SYSTEM : VM_PRIORITY_USER); } struct PageUnreserver { - PageUnreserver(page_num_t count) - : fCount(count) + PageUnreserver(vm_page_reservation* reservation) + : + fReservation(reservation) { } ~PageUnreserver() { - if (fCount > 0) - vm_page_unreserve_pages(fCount); + if (fReservation != NULL) + vm_page_unreserve_pages(fReservation); } - page_num_t fCount; - } pageUnreserver(reservedPreMapPages); + vm_page_reservation* fReservation; + } pageUnreserver(reservedPreMapPages > 0 ? &reservation : NULL); AddressSpaceWriteLocker locker(team); if (!locker.IsLocked()) @@ -1638,7 +1639,7 @@ _vm_map_file(team_id team, const char* name, void** _address, } if (status == B_OK && (protection & B_READ_AREA) != 0) - pre_map_area_pages(area, cache); + pre_map_area_pages(area, cache, &reservation); cache->Unlock(); @@ -1798,7 +1799,8 @@ vm_clone_area(team_id team, const char* name, void** address, size_t reservePages = map->MaxPagesNeededToMap(newArea->Base(), newArea->Base() + (newArea->Size() - 1)); - vm_page_reserve_pages(reservePages, + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, reservePages, targetAddressSpace == VMAddressSpace::Kernel() ? VM_PRIORITY_SYSTEM : VM_PRIORITY_USER); map->Lock(); @@ -1806,16 +1808,17 @@ vm_clone_area(team_id team, const char* name, void** address, for (addr_t offset = 0; offset < newArea->Size(); offset += B_PAGE_SIZE) { map->Map(newArea->Base() + offset, physicalAddress + offset, - protection); + protection, &reservation); } map->Unlock(); - vm_page_unreserve_pages(reservePages); + vm_page_unreserve_pages(&reservation); } else { VMTranslationMap* map = targetAddressSpace->TranslationMap(); size_t reservePages = map->MaxPagesNeededToMap( newArea->Base(), newArea->Base() + (newArea->Size() - 1)); - vm_page_reserve_pages(reservePages, + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, reservePages, targetAddressSpace == VMAddressSpace::Kernel() ? VM_PRIORITY_SYSTEM : VM_PRIORITY_USER); @@ -1827,14 +1830,14 @@ vm_clone_area(team_id team, const char* name, void** address, map_page(newArea, page, newArea->Base() + ((page->cache_offset << PAGE_SHIFT) - newArea->cache_offset), - protection); + protection, &reservation, false); DEBUG_PAGE_ACCESS_END(page); } } // TODO: B_FULL_LOCK means that all pages are locked. We are not // ensuring that! - vm_page_unreserve_pages(reservePages); + vm_page_unreserve_pages(&reservation); } } if (status == B_OK) @@ -2206,42 +2209,6 @@ vm_test_map_modification(vm_page* page) } -/*! The page's cache must be locked. -*/ -int32 -vm_test_map_activation(vm_page* page, bool* _modified) -{ - int32 activation = 0; - bool modified = false; - - vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); - vm_page_mapping* mapping; - while ((mapping = iterator.Next()) != NULL) { - VMArea* area = mapping->area; - VMTranslationMap* map = area->address_space->TranslationMap(); - - addr_t physicalAddress; - uint32 flags; - map->Lock(); - map->Query(virtual_page_address(area, page), &physicalAddress, &flags); - map->Unlock(); - - if ((flags & PAGE_ACCESSED) != 0) - activation++; - if ((flags & PAGE_MODIFIED) != 0) - modified = true; - } - - if (_modified != NULL) - *_modified = modified || page->modified; - - if (page->accessed) - activation++; - - return activation; -} - - /*! The page's cache must be locked. */ void @@ -2266,26 +2233,84 @@ vm_clear_map_flags(vm_page* page, uint32 flags) /*! Removes all mappings from a page. - After you've called this function, the page is unmapped from memory. - The accumulated page flags of all mappings can be found in \a _flags. + After you've called this function, the page is unmapped from memory and + the page's \c accessed and \c modified flags have been updated according + to the state of the mappings. The page's cache must be locked. */ void -vm_remove_all_page_mappings(vm_page* page, uint32* _flags) +vm_remove_all_page_mappings(vm_page* page) { while (vm_page_mapping* mapping = page->mappings.Head()) { VMArea* area = mapping->area; VMTranslationMap* map = area->address_space->TranslationMap(); addr_t address = virtual_page_address(area, page); - map->UnmapPage(area, address); + map->UnmapPage(area, address, false); + } +} + + +int32 +vm_clear_page_mapping_accessed_flags(struct vm_page *page) +{ + int32 count = 0; + + vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); + vm_page_mapping* mapping; + while ((mapping = iterator.Next()) != NULL) { + VMArea* area = mapping->area; + VMTranslationMap* map = area->address_space->TranslationMap(); + + bool modified; + if (map->ClearAccessedAndModified(area, + virtual_page_address(area, page), false, modified)) { + count++; + } + + page->modified |= modified; } - if (_flags != NULL) { - *_flags = (page->modified ? PAGE_MODIFIED : 0) - | (page->accessed ? PAGE_ACCESSED : 0); - // TODO: This return value is obviously not particularly useful, as - // the caller could simply check the page's flags. + + if (page->accessed) { + count++; + page->accessed = false; } + + return count; +} + + +/*! Removes all mappings of a page and/or clears the accessed bits of the + mappings. + The function iterates through the page mappings and removes them until + encountering one that has been accessed. From then on it will continue to + iterate, but only clear the accessed flag of the mapping. The page's + \c modified bit will be updated accordingly, the \c accessed bit will be + cleared. + \return The number of mapping accessed bits encountered, including the + \c accessed bit of the page itself. If \c 0 is returned, all mappings + of the page have been removed. +*/ +int32 +vm_remove_all_page_mappings_if_unaccessed(struct vm_page *page) +{ + if (page->accessed) + return vm_clear_page_mapping_accessed_flags(page); + + while (vm_page_mapping* mapping = page->mappings.Head()) { + VMArea* area = mapping->area; + VMTranslationMap* map = area->address_space->TranslationMap(); + addr_t address = virtual_page_address(area, page); + bool modified = false; + if (map->ClearAccessedAndModified(area, address, true, modified)) { + page->accessed = true; + page->modified |= modified; + return vm_clear_page_mapping_accessed_flags(page); + } + page->modified |= modified; + } + + return 0; } @@ -3429,7 +3454,6 @@ status_t vm_init_post_thread(kernel_args* args) { vm_page_init_post_thread(args); - vm_daemon_init(); slab_init_post_thread(); return heap_init_post_thread(); } @@ -3630,6 +3654,7 @@ struct PageFaultContext { VMTranslationMap* map; VMCache* topCache; off_t cacheOffset; + vm_page_reservation reservation; bool isWrite; // return values @@ -3648,6 +3673,7 @@ struct PageFaultContext { ~PageFaultContext() { UnlockAll(); + vm_page_unreserve_pages(&reservation); } void Prepare(VMCache* topCache, off_t cacheOffset) @@ -3718,7 +3744,7 @@ fault_get_page(PageFaultContext& context) // see if the backing store has it if (cache->HasPage(context.cacheOffset)) { // insert a fresh page and mark it busy -- we're going to read it in - page = vm_page_allocate_page( + page = vm_page_allocate_page(&context.reservation, PAGE_STATE_ACTIVE | VM_PAGE_ALLOC_BUSY); cache->InsertPage(page, context.cacheOffset); @@ -3773,7 +3799,8 @@ fault_get_page(PageFaultContext& context) cache = context.isWrite ? context.topCache : lastCache; // allocate a clean page - page = vm_page_allocate_page(PAGE_STATE_ACTIVE | VM_PAGE_ALLOC_CLEAR); + page = vm_page_allocate_page(&context.reservation, + PAGE_STATE_ACTIVE | VM_PAGE_ALLOC_CLEAR); FTRACE(("vm_soft_fault: just allocated page 0x%lx\n", page->physical_page_number)); @@ -3787,7 +3814,7 @@ fault_get_page(PageFaultContext& context) // TODO: If memory is low, it might be a good idea to steal the page // from our source cache -- if possible, that is. FTRACE(("get new page, copy it, and put it into the topmost cache\n")); - page = vm_page_allocate_page(PAGE_STATE_ACTIVE); + page = vm_page_allocate_page(&context.reservation, PAGE_STATE_ACTIVE); // To not needlessly kill concurrency we unlock all caches but the top // one while copying the page. Lacking another mechanism to ensure that @@ -3832,7 +3859,7 @@ vm_soft_fault(VMAddressSpace* addressSpace, addr_t originalAddress, size_t reservePages = 2 + context.map->MaxPagesNeededToMap(originalAddress, originalAddress); context.addressSpaceLocker.Unlock(); - vm_page_reserve_pages(reservePages, + vm_page_reserve_pages(&context.reservation, reservePages, addressSpace == VMAddressSpace::Kernel() ? VM_PRIORITY_SYSTEM : VM_PRIORITY_USER); @@ -3960,7 +3987,8 @@ vm_soft_fault(VMAddressSpace* addressSpace, addr_t originalAddress, } if (mapPage) { - if (map_page(area, context.page, address, newProtection) != B_OK) { + if (map_page(area, context.page, address, newProtection, + &context.reservation, true) != B_OK) { // Mapping can only fail, when the page mapping object couldn't // be allocated. Save for the missing mapping everything is // fine, though. We'll simply leave and probably fault again. @@ -3979,15 +4007,14 @@ vm_soft_fault(VMAddressSpace* addressSpace, addr_t originalAddress, break; } - } + } else if (context.page->state == PAGE_STATE_INACTIVE) + vm_page_set_state(context.page, PAGE_STATE_ACTIVE); DEBUG_PAGE_ACCESS_END(context.page); break; } - vm_page_unreserve_pages(reservePages); - return status; } @@ -5335,10 +5362,8 @@ _user_set_memory_protection(void* _address, size_t size, int protection) bool unmapPage = page->Cache() != topCache && (protection & B_WRITE_AREA) != 0; - if (!unmapPage) { - map->Unmap(pageAddress, pageAddress + B_PAGE_SIZE - 1); - map->Map(pageAddress, physicalAddress, actualProtection); - } + if (!unmapPage) + map->ProtectPage(area, pageAddress, actualProtection); map->Unlock(); diff --git a/src/system/kernel/vm/vm_daemons.cpp b/src/system/kernel/vm/vm_daemons.cpp deleted file mode 100644 index 3a0a9b9828..0000000000 --- a/src/system/kernel/vm/vm_daemons.cpp +++ /dev/null @@ -1,473 +0,0 @@ -/* - * Copyright 2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved. - * Distributed under the terms of the MIT License. - */ - - -#include "PageCacheLocker.h" - -#include - -#include - -#include - -#include -#include -#include -#include -#include - -#include "VMAnonymousCache.h" - - -//#define TRACK_PAGE_USAGE_STATS 1 - -//#define TRACE_VM_DAEMONS -#ifdef TRACE_VM_DAEMONS -#define TRACE(x) dprintf x -#else -#define TRACE(x) ; -#endif - -const static uint32 kMinScanPagesCount = 512; -const static uint32 kMaxScanPagesCount = 8192; -const static bigtime_t kMinScanWaitInterval = 50000LL; // 50 ms -const static bigtime_t kMaxScanWaitInterval = 1000000LL; // 1 sec - -static uint32 sLowPagesCount; -static sem_id sPageDaemonSem; -static uint32 sNumPages; - -#ifdef TRACK_PAGE_USAGE_STATS -static page_num_t sPageUsageArrays[512]; -static page_num_t* sPageUsage = sPageUsageArrays; -static page_num_t sPageUsagePageCount; -static page_num_t* sNextPageUsage = sPageUsageArrays + 256; -static page_num_t sNextPageUsagePageCount; -#endif - - -PageCacheLocker::PageCacheLocker(vm_page* page, bool dontWait) - : - fPage(NULL) -{ - Lock(page, dontWait); -} - - -PageCacheLocker::~PageCacheLocker() -{ - Unlock(); -} - - -bool -PageCacheLocker::_IgnorePage(vm_page* page) -{ - if (page->busy || page->state == PAGE_STATE_WIRED - || page->state == PAGE_STATE_FREE || page->state == PAGE_STATE_CLEAR - || page->state == PAGE_STATE_UNUSED || page->wired_count > 0) - return true; - - return false; -} - - -bool -PageCacheLocker::Lock(vm_page* page, bool dontWait) -{ - if (_IgnorePage(page)) - return false; - - // Grab a reference to this cache. - VMCache* cache = vm_cache_acquire_locked_page_cache(page, dontWait); - if (cache == NULL) - return false; - - if (_IgnorePage(page)) { - cache->ReleaseRefAndUnlock(); - return false; - } - - fPage = page; - return true; -} - - -void -PageCacheLocker::Unlock() -{ - if (fPage == NULL) - return; - - fPage->Cache()->ReleaseRefAndUnlock(); - - fPage = NULL; -} - - -// #pragma mark - - - -#if PAGE_DAEMON_TRACING - -namespace PageDaemonTracing { - -class ActivatePage : public AbstractTraceEntry { - public: - ActivatePage(vm_page* page) - : - fCache(page->cache), - fPage(page) - { - Initialized(); - } - - virtual void AddDump(TraceOutput& out) - { - out.Print("page activated: %p, cache: %p", fPage, fCache); - } - - private: - VMCache* fCache; - vm_page* fPage; -}; - - -class DeactivatePage : public AbstractTraceEntry { - public: - DeactivatePage(vm_page* page) - : - fCache(page->cache), - fPage(page) - { - Initialized(); - } - - virtual void AddDump(TraceOutput& out) - { - out.Print("page deactivated: %p, cache: %p", fPage, fCache); - } - - private: - VMCache* fCache; - vm_page* fPage; -}; - - -class FreedPageSwap : public AbstractTraceEntry { - public: - FreedPageSwap(vm_page* page) - : - fCache(page->cache), - fPage(page) - { - Initialized(); - } - - virtual void AddDump(TraceOutput& out) - { - out.Print("page swap freed: %p, cache: %p", fPage, fCache); - } - - private: - VMCache* fCache; - vm_page* fPage; -}; - -} // namespace PageDaemonTracing - -# define T(x) new(std::nothrow) PageDaemonTracing::x - -#else -# define T(x) -#endif // PAGE_DAEMON_TRACING - - -// #pragma mark - - - -#ifdef TRACK_PAGE_USAGE_STATS - -static void -track_page_usage(vm_page* page) -{ - if (page->wired_count == 0) { - sNextPageUsage[(int32)page->usage_count + 128]++; - sNextPageUsagePageCount++; - } -} - - -static void -update_page_usage_stats() -{ - std::swap(sPageUsage, sNextPageUsage); - sPageUsagePageCount = sNextPageUsagePageCount; - - memset(sNextPageUsage, 0, sizeof(page_num_t) * 256); - sNextPageUsagePageCount = 0; - - // compute average - if (sPageUsagePageCount > 0) { - int64 sum = 0; - for (int32 i = 0; i < 256; i++) - sum += (int64)sPageUsage[i] * (i - 128); - - TRACE(("average page usage: %f (%lu pages)\n", - (float)sum / sPageUsagePageCount, sPageUsagePageCount)); - } -} - - -static int -dump_page_usage_stats(int argc, char** argv) -{ - kprintf("distribution of page usage counts (%lu pages):", - sPageUsagePageCount); - - int64 sum = 0; - for (int32 i = 0; i < 256; i++) { - if (i % 8 == 0) - kprintf("\n%4ld:", i - 128); - - int64 count = sPageUsage[i]; - sum += count * (i - 128); - - kprintf(" %9llu", count); - } - - kprintf("\n\n"); - - kprintf("average usage count: %f\n", - sPageUsagePageCount > 0 ? (float)sum / sPageUsagePageCount : 0); - - return 0; -} - -#endif - - -static void -clear_page_activation(int32 index) -{ - vm_page *page = vm_page_at_index(index); - PageCacheLocker locker(page); - if (!locker.IsLocked()) - return; - - if (!page->busy && page->state == PAGE_STATE_ACTIVE) - vm_clear_map_flags(page, PAGE_ACCESSED); -} - - -static bool -check_page_activation(int32 index) -{ - vm_page *page = vm_page_at_index(index); - PageCacheLocker locker(page); - if (!locker.IsLocked()) - return false; - - DEBUG_PAGE_ACCESS_START(page); - - bool modified; - int32 activation = vm_test_map_activation(page, &modified); - if (modified && page->state != PAGE_STATE_MODIFIED) { - TRACE(("page %p -> move to modified\n", page); - vm_page_set_state(page, PAGE_STATE_MODIFIED)); - } - - if (activation > 0) { - // page is still in active use - if (page->usage_count < 0) { - if (page->state != PAGE_STATE_MODIFIED) - vm_page_set_state(page, PAGE_STATE_ACTIVE); - page->usage_count = 1; - TRACE(("page %p -> move to active\n", page)); - T(ActivatePage(page)); - } else if (page->usage_count < 127) - page->usage_count++; - -#ifdef TRACK_PAGE_USAGE_STATS - track_page_usage(page); -#endif - - DEBUG_PAGE_ACCESS_END(page); - return false; - } - - if (page->usage_count > -128) - page->usage_count--; - -#ifdef TRACK_PAGE_USAGE_STATS - track_page_usage(page); -#endif - - if (page->usage_count < 0) { - uint32 flags; - vm_remove_all_page_mappings(page, &flags); - - // recheck eventual last minute changes - if ((flags & PAGE_MODIFIED) != 0 && page->state != PAGE_STATE_MODIFIED) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - if ((flags & PAGE_ACCESSED) != 0 && ++page->usage_count >= 0) { - DEBUG_PAGE_ACCESS_END(page); - return false; - } - - if (page->state == PAGE_STATE_MODIFIED) - vm_page_schedule_write_page(page); - else - vm_page_set_state(page, PAGE_STATE_INACTIVE); - TRACE(("page %p -> move to inactive\n", page)); - T(DeactivatePage(page)); - } - - DEBUG_PAGE_ACCESS_END(page); - return true; -} - - -#if ENABLE_SWAP_SUPPORT -static bool -free_page_swap_space(int32 index) -{ - vm_page *page = vm_page_at_index(index); - PageCacheLocker locker(page); - if (!locker.IsLocked()) - return false; - - DEBUG_PAGE_ACCESS_START(page); - - VMCache* cache = page->Cache(); - if (cache->temporary && page->wired_count == 0 - && cache->HasPage(page->cache_offset << PAGE_SHIFT) - && page->usage_count > 0) { - // TODO: how to judge a page is highly active? - if (swap_free_page_swap_space(page)) { - // We need to mark the page modified, since otherwise it could be - // stolen and we'd lose its data. - vm_page_set_state(page, PAGE_STATE_MODIFIED); - T(FreedPageSwap(page)); - DEBUG_PAGE_ACCESS_END(page); - return true; - } - } - DEBUG_PAGE_ACCESS_END(page); - return false; -} -#endif - - -static status_t -page_daemon(void* /*unused*/) -{ - bigtime_t scanWaitInterval = kMaxScanWaitInterval; - uint32 scanPagesCount = kMinScanPagesCount; - uint32 clearPage = 0; - uint32 checkPage = sNumPages / 2; - -#if ENABLE_SWAP_SUPPORT - uint32 swapPage = 0; -#endif - - while (true) { - acquire_sem_etc(sPageDaemonSem, 1, B_RELATIVE_TIMEOUT, - scanWaitInterval); - - // Compute next run time - uint32 pagesLeft = vm_page_num_free_pages(); - if (pagesLeft > sLowPagesCount) { - // don't do anything if we have enough free memory left - continue; - } - - scanWaitInterval = kMinScanWaitInterval - + (kMaxScanWaitInterval - kMinScanWaitInterval) - * pagesLeft / sLowPagesCount; - scanPagesCount = kMaxScanPagesCount - - (kMaxScanPagesCount - kMinScanPagesCount) - * pagesLeft / sLowPagesCount; - uint32 leftToFree = sLowPagesCount - pagesLeft; - TRACE(("wait interval %Ld, scan pages %lu, free %lu, " - "target %lu\n", scanWaitInterval, scanPagesCount, - pagesLeft, leftToFree)); - - for (uint32 i = 0; i < scanPagesCount && leftToFree > 0; i++) { - if (clearPage == 0) - TRACE(("clear through\n")); - if (checkPage == 0) { - TRACE(("check through\n")); -#ifdef TRACK_PAGE_USAGE_STATS - update_page_usage_stats(); -#endif - } - clear_page_activation(clearPage); - - if (check_page_activation(checkPage)) - leftToFree--; - - if (++clearPage == sNumPages) - clearPage = 0; - if (++checkPage == sNumPages) - checkPage = 0; - } - -#if ENABLE_SWAP_SUPPORT - uint32 availableSwapPages = swap_available_pages(); - if (swap_total_swap_pages() > 0 && availableSwapPages < sNumPages / 8) { - uint32 swapPagesToFree = sNumPages / 8 - availableSwapPages; - uint32 swapScanCount = kMaxScanPagesCount - - (kMaxScanPagesCount - kMinScanPagesCount) - * availableSwapPages / (sNumPages / 8); - - for (uint32 i = 0; i < swapScanCount && swapPagesToFree > 0; i++) { - if (free_page_swap_space(swapPage)) - swapPagesToFree--; - if (++swapPage == sNumPages) - swapPage = 0; - } - } -#endif - } - - return B_OK; -} - - -void -vm_schedule_page_scanner(uint32 target) -{ - release_sem(sPageDaemonSem); -} - - -status_t -vm_daemon_init() -{ -#ifdef TRACK_PAGE_USAGE_STATS - add_debugger_command_etc("page_usage", &dump_page_usage_stats, - "Dumps statistics about page usage counts", - "\n" - "Dumps statistics about page usage counts.\n", - B_KDEBUG_DONT_PARSE_ARGUMENTS); -#endif - - sPageDaemonSem = create_sem(0, "page daemon"); - - sNumPages = vm_page_num_pages(); - - sLowPagesCount = sNumPages / 16; - if (sLowPagesCount < 1024) - sLowPagesCount = 1024; - - // create a kernel thread to select pages for pageout - thread_id thread = spawn_kernel_thread(&page_daemon, "page daemon", - B_NORMAL_PRIORITY, NULL); - send_signal_etc(thread, SIGCONT, B_DO_NOT_RESCHEDULE); - - return B_OK; -} - diff --git a/src/system/kernel/vm/vm_page.cpp b/src/system/kernel/vm/vm_page.cpp index 8b0092f062..0b86655b63 100644 --- a/src/system/kernel/vm/vm_page.cpp +++ b/src/system/kernel/vm/vm_page.cpp @@ -51,6 +51,15 @@ # define TRACE(x) ; #endif +//#define TRACE_VM_DAEMONS +#ifdef TRACE_VM_DAEMONS +#define TRACE_DAEMON(x...) dprintf(x) +#else +#define TRACE_DAEMON(x...) do {} while (false) +#endif + +//#define TRACK_PAGE_USAGE_STATS 1 + #define SCRUB_SIZE 16 // this many pages will be cleared at once in the page scrubber thread @@ -68,8 +77,25 @@ static const size_t kPageReserveForPriority[] = { 0 // VIP }; -static const uint32 kMinimumSystemReserve = VM_PAGE_RESERVE_USER; +// Minimum number of free pages the page daemon will try to achieve. +static uint32 sFreePagesTarget; +static uint32 sFreeOrCachedPagesTarget; +static uint32 sInactivePagesTarget; +// Wait interval between page daemon runs. +static const bigtime_t kIdleScanWaitInterval = 1000000LL; // 1 sec +static const bigtime_t kBusyScanWaitInterval = 500000LL; // 0.5 sec + +// Number of idle runs after which we want to have processed the full active +// queue. +static const uint32 kIdleRunsForFullQueue = 20; + +// Maximum limit for the vm_page::usage_count. +static const int32 kPageUsageMax = 64; +// vm_page::usage_count buff an accessed page receives in a scan. +static const int32 kPageUsageAdvance = 3; +// vm_page::usage_count debuff an unaccessed page receives in a scan. +static const int32 kPageUsageDecline = 1; int32 gMappedPagesCount; @@ -86,8 +112,7 @@ static vm_page *sPages; static addr_t sPhysicalPageOffset; static size_t sNumPages; static vint32 sUnreservedFreePages; -static vint32 sSystemReservedPages; -static vint32 sPageDeficit; +static vint32 sUnsatisfiedPageReservations; static vint32 sModifiedTemporaryPages; static ConditionVariable sFreePageCondition; @@ -96,7 +121,95 @@ static mutex sPageDeficitLock = MUTEX_INITIALIZER("page deficit"); static rw_lock sFreePageQueuesLock = RW_LOCK_INITIALIZER("free/clear page queues"); -static sem_id sWriterWaitSem; +#ifdef TRACK_PAGE_USAGE_STATS +static page_num_t sPageUsageArrays[512]; +static page_num_t* sPageUsage = sPageUsageArrays; +static page_num_t sPageUsagePageCount; +static page_num_t* sNextPageUsage = sPageUsageArrays + 256; +static page_num_t sNextPageUsagePageCount; +#endif + + +struct PageReservationWaiter + : public DoublyLinkedListLinkImpl { + struct thread* thread; + uint32 dontTouch; // reserve not to touch + uint32 missing; // pages missing for the reservation + int32 threadPriority; + + bool operator<(const PageReservationWaiter& other) const + { + // Implies an order by descending VM priority (ascending dontTouch) + // and (secondarily) descending thread priority. + if (dontTouch != other.dontTouch) + return dontTouch < other.dontTouch; + return threadPriority > other.threadPriority; + } +}; + +typedef DoublyLinkedList PageReservationWaiterList; +static PageReservationWaiterList sPageReservationWaiters; + + +struct DaemonCondition { + void Init(const char* name) + { + mutex_init(&fLock, "daemon condition"); + fCondition.Init(this, name); + fActivated = false; + } + + bool Lock() + { + return mutex_lock(&fLock) == B_OK; + } + + void Unlock() + { + mutex_unlock(&fLock); + } + + bool Wait(bigtime_t timeout, bool clearActivated) + { + MutexLocker locker(fLock); + if (clearActivated) + fActivated = false; + else if (fActivated) + return true; + + ConditionVariableEntry entry; + fCondition.Add(&entry); + + locker.Unlock(); + + return entry.Wait(B_RELATIVE_TIMEOUT, timeout) == B_OK; + } + + void WakeUp() + { + if (fActivated) + return; + + MutexLocker locker(fLock); + fActivated = true; + fCondition.NotifyOne(); + } + + void ClearActivated() + { + MutexLocker locker(fLock); + fActivated = false; + } + +private: + mutex fLock; + ConditionVariable fCondition; + bool fActivated; +}; + + +static DaemonCondition sPageWriterCondition; +static DaemonCondition sPageDaemonCondition; #if PAGE_ALLOCATION_TRACING @@ -241,13 +354,88 @@ class StolenPage : public AbstractTraceEntry { } // namespace PageAllocationTracing -# define T(x) new(std::nothrow) PageAllocationTracing::x +# define TA(x) new(std::nothrow) PageAllocationTracing::x #else -# define T(x) +# define TA(x) #endif // PAGE_ALLOCATION_TRACING +#if PAGE_DAEMON_TRACING + +namespace PageDaemonTracing { + +class ActivatePage : public AbstractTraceEntry { + public: + ActivatePage(vm_page* page) + : + fCache(page->cache), + fPage(page) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("page activated: %p, cache: %p", fPage, fCache); + } + + private: + VMCache* fCache; + vm_page* fPage; +}; + + +class DeactivatePage : public AbstractTraceEntry { + public: + DeactivatePage(vm_page* page) + : + fCache(page->cache), + fPage(page) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("page deactivated: %p, cache: %p", fPage, fCache); + } + + private: + VMCache* fCache; + vm_page* fPage; +}; + + +class FreedPageSwap : public AbstractTraceEntry { + public: + FreedPageSwap(vm_page* page) + : + fCache(page->cache), + fPage(page) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("page swap freed: %p, cache: %p", fPage, fCache); + } + + private: + VMCache* fCache; + vm_page* fPage; +}; + +} // namespace PageDaemonTracing + +# define TD(x) new(std::nothrow) PageDaemonTracing::x + +#else +# define TD(x) +#endif // PAGE_DAEMON_TRACING + + #if PAGE_WRITER_TRACING namespace PageWriterTracing { @@ -569,7 +757,7 @@ dump_page_stats(int argc, char **argv) if (sPages[i].state == PAGE_STATE_MODIFIED && sPages[i].Cache() != NULL && sPages[i].Cache()->temporary && sPages[i].wired_count == 0) { swappableModified++; - if (sPages[i].usage_count < 0) + if (sPages[i].usage_count == 0) swappableModifiedInactive++; } } @@ -593,10 +781,19 @@ dump_page_stats(int argc, char **argv) kprintf("clear: %" B_PRIuSIZE "\n", counter[PAGE_STATE_CLEAR]); kprintf("unreserved free pages: %" B_PRId32 "\n", sUnreservedFreePages); - kprintf("system reserved pages: %" B_PRId32 "\n", sSystemReservedPages); - kprintf("page deficit: %lu\n", sPageDeficit); + kprintf("unsatisfied page reservations: %" B_PRId32 "\n", + sUnsatisfiedPageReservations); kprintf("mapped pages: %lu\n", gMappedPagesCount); + kprintf("waiting threads:\n"); + for (PageReservationWaiterList::Iterator it + = sPageReservationWaiters.GetIterator(); + PageReservationWaiter* waiter = it.Next();) { + kprintf(" %6" B_PRId32 ": missing: %6" B_PRIu32 + ", don't touch: %6" B_PRIu32 "\n", waiter->thread->id, + waiter->missing, waiter->dontTouch); + } + kprintf("\nfree queue: %p, count = %ld\n", &sFreePageQueue, sFreePageQueue.Count()); kprintf("clear queue: %p, count = %ld\n", &sClearPageQueue, @@ -614,32 +811,150 @@ dump_page_stats(int argc, char **argv) } -// #pragma mark - +#ifdef TRACK_PAGE_USAGE_STATS + +static void +track_page_usage(vm_page* page) +{ + if (page->wired_count == 0) { + sNextPageUsage[(int32)page->usage_count + 128]++; + sNextPageUsagePageCount++; + } +} static void -unreserve_page() +update_page_usage_stats() { - int32 systemReserve = sSystemReservedPages; - if (systemReserve >= (int32)kMinimumSystemReserve) { - atomic_add(&sUnreservedFreePages, 1); - } else { - // Note: Due to the race condition, we might increment - // sSystemReservedPages beyond its desired count. That doesn't matter - // all that much, though, since its only about a single page and - // vm_page_reserve_pages() will correct this when the general reserve - // is running low. - atomic_add(&sSystemReservedPages, 1); + std::swap(sPageUsage, sNextPageUsage); + sPageUsagePageCount = sNextPageUsagePageCount; + + memset(sNextPageUsage, 0, sizeof(page_num_t) * 256); + sNextPageUsagePageCount = 0; + + // compute average + if (sPageUsagePageCount > 0) { + int64 sum = 0; + for (int32 i = 0; i < 256; i++) + sum += (int64)sPageUsage[i] * (i - 128); + + TRACE_DAEMON("average page usage: %f (%lu pages)\n", + (float)sum / sPageUsagePageCount, sPageUsagePageCount); + } +} + + +static int +dump_page_usage_stats(int argc, char** argv) +{ + kprintf("distribution of page usage counts (%lu pages):", + sPageUsagePageCount); + + int64 sum = 0; + for (int32 i = 0; i < 256; i++) { + if (i % 8 == 0) + kprintf("\n%4ld:", i - 128); + + int64 count = sPageUsage[i]; + sum += count * (i - 128); + + kprintf(" %9llu", count); } - if (sPageDeficit > 0) { - MutexLocker pageDeficitLocker(sPageDeficitLock); - if (sPageDeficit > 0) - sFreePageCondition.NotifyAll(); + kprintf("\n\n"); + + kprintf("average usage count: %f\n", + sPageUsagePageCount > 0 ? (float)sum / sPageUsagePageCount : 0); + + return 0; +} + +#endif // TRACK_PAGE_USAGE_STATS + + +// #pragma mark - + + +struct page_stats { + int32 totalFreePages; + int32 unsatisfiedReservations; + int32 cachedPages; +}; + +static void +get_page_stats(page_stats& pageStats) +{ + pageStats.totalFreePages = sUnreservedFreePages; + pageStats.cachedPages = sCachedPageQueue.Count(); + pageStats.unsatisfiedReservations = sUnsatisfiedPageReservations; + // TODO: We don't get an actual snapshot here! +} + + +/*! Reserves as many pages as possible from \c sUnreservedFreePages up to + \a count. Doesn't touch the last \a dontTouch pages of + \c sUnreservedFreePages, though. + \return The number of actually reserved pages. +*/ +static uint32 +reserve_some_pages(uint32 count, uint32 dontTouch) +{ + while (true) { + int32 freePages = sUnreservedFreePages; + if (freePages <= (int32)dontTouch) + return 0; + + int32 toReserve = std::min(count, freePages - dontTouch); + if (atomic_test_and_set(&sUnreservedFreePages, + freePages - toReserve, freePages) + == freePages) { + return toReserve; + } + + // the count changed in the meantime -- retry } } +static void +wake_up_page_reservation_waiters() +{ + MutexLocker pageDeficitLocker(sPageDeficitLock); + + // TODO: If this is a low priority thread, we might want to disable + // interrupts or otherwise ensure that we aren't unscheduled. Otherwise + // high priority threads wait be kept waiting while a medium priority thread + // prevents us from running. + + while (PageReservationWaiter* waiter = sPageReservationWaiters.Head()) { + int32 reserved = reserve_some_pages(waiter->missing, + waiter->dontTouch); + if (reserved == 0) + return; + + atomic_add(&sUnsatisfiedPageReservations, -reserved); + waiter->missing -= reserved; + + if (waiter->missing > 0) + return; + + sPageReservationWaiters.Remove(waiter); + + InterruptsSpinLocker threadLocker(gThreadSpinlock); + thread_unblock_locked(waiter->thread, B_OK); + } +} + + +static inline void +unreserve_pages(uint32 count) +{ + atomic_add(&sUnreservedFreePages, count); + if (sUnsatisfiedPageReservations != 0) + wake_up_page_reservation_waiters(); +} + + static void free_page(vm_page* page, bool clear) { @@ -679,16 +994,10 @@ free_page(vm_page* page, bool clear) if (!page->mappings.IsEmpty() || page->wired_count > 0) panic("to be freed page %p has mappings", page); - if (sPageDeficit > 0) { - MutexLocker pageDeficitLocker(sPageDeficitLock); - if (sPageDeficit > 0) - sFreePageCondition.NotifyOne(); - } - if (fromQueue != NULL) fromQueue->RemoveUnlocked(page); - T(FreePage()); + TA(FreePage()); ReadLocker locker(sFreePageQueuesLock); @@ -704,7 +1013,7 @@ free_page(vm_page* page, bool clear) locker.Unlock(); - unreserve_page(); + unreserve_pages(1); } @@ -762,6 +1071,7 @@ set_page_state(vm_page *page, int pageState) toQueue = &sModifiedPageQueue; break; case PAGE_STATE_CACHED: + ASSERT(page->wired_count == 0 && page->mappings.IsEmpty()); toQueue = &sCachedPageQueue; break; case PAGE_STATE_FREE: @@ -777,12 +1087,6 @@ set_page_state(vm_page *page, int pageState) return; } - if (pageState == PAGE_STATE_INACTIVE && sPageDeficit > 0) { - MutexLocker pageDeficitLocker(sPageDeficitLock); - if (sPageDeficit > 0) - sFreePageCondition.NotifyOne(); - } - VMCache* cache = page->Cache(); if (cache != NULL && cache->temporary) { if (pageState == PAGE_STATE_MODIFIED) @@ -825,18 +1129,18 @@ move_page_to_active_or_inactive_queue(vm_page *page, bool dequeued) // Note, this logic must be in sync with what the page daemon does int32 state; - if (!page->mappings.IsEmpty() || page->usage_count >= 0 - || page->wired_count) + if (page->mappings.IsEmpty() && page->wired_count == 0) + state = PAGE_STATE_CACHED; + else if (page->usage_count > 0) state = PAGE_STATE_ACTIVE; else state = PAGE_STATE_INACTIVE; -// TODO: Cached queue! +// TODO: If free + cached pages are low, we might directly want to free the +// page. if (dequeued) { page->state = state; - VMPageQueue& queue = state == PAGE_STATE_ACTIVE - ? sActivePageQueue : sInactivePageQueue; - queue.AppendUnlocked(page); + sPageQueues[state].AppendUnlocked(page); if (page->Cache()->temporary) atomic_add(&sModifiedTemporaryPages, -1); } else @@ -868,15 +1172,19 @@ page_scrubber(void *unused) for (;;) { snooze(100000); // 100ms - if (sFreePageQueue.Count() == 0) + if (sFreePageQueue.Count() == 0 + || sUnreservedFreePages < (int32)sFreePagesTarget) { continue; + } // Since we temporarily remove pages from the free pages reserve, // we must make sure we don't cause a violation of the page // reservation warranty. The following is usually stricter than // necessary, because we don't have information on how many of the // reserved pages have already been allocated. - if (!vm_page_try_reserve_pages(SCRUB_SIZE, VM_PRIORITY_USER)) + int32 reserved = reserve_some_pages(SCRUB_SIZE, + kPageReserveForPriority[VM_PRIORITY_USER]); + if (reserved == 0) continue; // get some pages from the free queue @@ -884,7 +1192,7 @@ page_scrubber(void *unused) vm_page *page[SCRUB_SIZE]; int32 scrubCount = 0; - for (int32 i = 0; i < SCRUB_SIZE; i++) { + for (int32 i = 0; i < reserved; i++) { page[i] = sFreePageQueue.RemoveHeadUnlocked(); if (page[i] == NULL) break; @@ -899,11 +1207,11 @@ page_scrubber(void *unused) locker.Unlock(); if (scrubCount == 0) { - vm_page_unreserve_pages(SCRUB_SIZE); + unreserve_pages(reserved); continue; } - T(ScrubbingPages(scrubCount)); + TA(ScrubbingPages(scrubCount)); // clear them for (int32 i = 0; i < scrubCount; i++) @@ -921,44 +1229,37 @@ page_scrubber(void *unused) locker.Unlock(); - vm_page_unreserve_pages(SCRUB_SIZE); + unreserve_pages(reserved); - T(ScrubbedPages(scrubCount)); + TA(ScrubbedPages(scrubCount)); } return 0; } +static void +init_page_marker(vm_page &marker) +{ + marker.SetCacheRef(NULL); + marker.state = PAGE_STATE_UNUSED; + marker.busy = true; +#if DEBUG_PAGE_QUEUE + marker.queue = NULL; +#endif +#if DEBUG_PAGE_ACCESS + marker.accessing_thread = thread_get_current_thread_id(); +#endif +} + + static void remove_page_marker(struct vm_page &marker) { - if (marker.state == PAGE_STATE_UNUSED) - return; - DEBUG_PAGE_ACCESS_CHECK(&marker); - VMPageQueue *queue; - - switch (marker.state) { - case PAGE_STATE_ACTIVE: - queue = &sActivePageQueue; - break; - case PAGE_STATE_INACTIVE: - queue = &sInactivePageQueue; - break; - case PAGE_STATE_MODIFIED: - queue = &sModifiedPageQueue; - break; - case PAGE_STATE_CACHED: - queue = &sCachedPageQueue; - break; - - default: - return; - } - - queue->RemoveUnlocked(&marker); + if (marker.state < PAGE_STATE_FIRST_UNQUEUED) + sPageQueues[marker.state].RemoveUnlocked(&marker); marker.state = PAGE_STATE_UNUSED; } @@ -1129,7 +1430,7 @@ PageWriteWrapper::CheckRemoveFromShrunkenCache() if (fPage->busy_writing) return; - vm_remove_all_page_mappings(fPage, NULL); + vm_remove_all_page_mappings(fPage); fCache->RemovePage(fPage); } @@ -1426,31 +1727,17 @@ page_writer(void* /*unused*/) } vm_page marker; - marker.SetCacheRef(NULL); - marker.state = PAGE_STATE_UNUSED; - marker.busy = true; -#if DEBUG_PAGE_QUEUE - marker.queue = NULL; -#endif -#if DEBUG_PAGE_ACCESS - marker.accessing_thread = thread_get_current_thread_id(); -#endif + init_page_marker(marker); while (true) { - if ((int32)sModifiedPageQueue.Count() - sModifiedTemporaryPages - < 1024) { - int32 count = 0; - get_sem_count(sWriterWaitSem, &count); - if (count == 0) - count = 1; - - acquire_sem_etc(sWriterWaitSem, count, B_RELATIVE_TIMEOUT, 3000000); +// TODO: Maybe wait shorter when memory is low! + if (sModifiedPageQueue.Count() < kNumPages) { + sPageWriterCondition.Wait(3000000, true); // all 3 seconds when no one triggers us } - int32 modifiedPages = sModifiedPageQueue.Count() - - sModifiedTemporaryPages; - if (modifiedPages <= 0) + int32 modifiedPages = sModifiedPageQueue.Count(); + if (modifiedPages == 0) continue; // depending on how urgent it becomes to get pages to disk, we adjust @@ -1567,38 +1854,75 @@ page_writer(void* /*unused*/) } +// #pragma mark - + + +// TODO: This should be done in the page daemon! +#if 0 +#if ENABLE_SWAP_SUPPORT +static bool +free_page_swap_space(int32 index) +{ + vm_page *page = vm_page_at_index(index); + PageCacheLocker locker(page); + if (!locker.IsLocked()) + return false; + + DEBUG_PAGE_ACCESS_START(page); + + VMCache* cache = page->Cache(); + if (cache->temporary && page->wired_count == 0 + && cache->HasPage(page->cache_offset << PAGE_SHIFT) + && page->usage_count > 0) { + // TODO: how to judge a page is highly active? + if (swap_free_page_swap_space(page)) { + // We need to mark the page modified, since otherwise it could be + // stolen and we'd lose its data. + vm_page_set_state(page, PAGE_STATE_MODIFIED); + TD(FreedPageSwap(page)); + DEBUG_PAGE_ACCESS_END(page); + return true; + } + } + DEBUG_PAGE_ACCESS_END(page); + return false; +} +#endif +#endif // 0 + + static vm_page * -find_page_candidate(struct vm_page &marker) +find_cached_page_candidate(struct vm_page &marker) { DEBUG_PAGE_ACCESS_CHECK(&marker); - InterruptsSpinLocker locker(sInactivePageQueue.GetLock()); + InterruptsSpinLocker locker(sCachedPageQueue.GetLock()); vm_page *page; if (marker.state == PAGE_STATE_UNUSED) { // Get the first free pages of the (in)active queue - page = sInactivePageQueue.Head(); + page = sCachedPageQueue.Head(); } else { // Get the next page of the current queue - if (marker.state != PAGE_STATE_INACTIVE) { + if (marker.state != PAGE_STATE_CACHED) { panic("invalid marker %p state", &marker); return NULL; } - page = sInactivePageQueue.Next(&marker); - sInactivePageQueue.Remove(&marker); + page = sCachedPageQueue.Next(&marker); + sCachedPageQueue.Remove(&marker); marker.state = PAGE_STATE_UNUSED; } while (page != NULL) { if (!page->busy) { // we found a candidate, insert marker - marker.state = PAGE_STATE_INACTIVE; - sInactivePageQueue.InsertAfter(page, &marker); + marker.state = PAGE_STATE_CACHED; + sCachedPageQueue.InsertAfter(page, &marker); return page; } - page = sInactivePageQueue.Next(page); + page = sCachedPageQueue.Next(page); } return NULL; @@ -1606,10 +1930,10 @@ find_page_candidate(struct vm_page &marker) static bool -steal_page(vm_page *page) +free_cached_page(vm_page *page, bool dontWait) { // try to lock the page's cache - if (vm_cache_acquire_locked_page_cache(page, false) == NULL) + if (vm_cache_acquire_locked_page_cache(page, dontWait) == NULL) return false; VMCache* cache = page->Cache(); @@ -1617,122 +1941,596 @@ steal_page(vm_page *page) MethodDeleter _2(cache, &VMCache::ReleaseRefLocked); // check again if that page is still a candidate - if (page->busy || page->state != PAGE_STATE_INACTIVE) + if (page->busy || page->state != PAGE_STATE_CACHED) return false; DEBUG_PAGE_ACCESS_START(page); - // recheck eventual last minute changes - uint32 flags; - vm_remove_all_page_mappings(page, &flags); - if ((flags & PAGE_MODIFIED) != 0) { - // page was modified, don't steal it - set_page_state(page, PAGE_STATE_MODIFIED); - DEBUG_PAGE_ACCESS_END(page); - return false; - } else if ((flags & PAGE_ACCESSED) != 0) { - // page is in active use, don't steal it - set_page_state(page, PAGE_STATE_ACTIVE); - DEBUG_PAGE_ACCESS_END(page); - return false; - } + ASSERT(page->wired_count == 0 && page->mappings.IsEmpty()); + ASSERT(!page->modified); // we can now steal this page - //dprintf(" steal page %p from cache %p%s\n", page, page->cache, - // page->state == PAGE_STATE_INACTIVE ? "" : " (ACTIVE)"); - cache->RemovePage(page); - sInactivePageQueue.RemoveUnlocked(page); + sCachedPageQueue.RemoveUnlocked(page); return true; } -static size_t -steal_pages(vm_page **pages, size_t count) +static uint32 +free_cached_pages(uint32 pagesToFree, bool dontWait) { - while (true) { - vm_page marker; - marker.SetCacheRef(NULL); - marker.state = PAGE_STATE_UNUSED; - marker.busy = true; -#if DEBUG_PAGE_QUEUE - marker.queue = NULL; -#endif -#if DEBUG_PAGE_ACCESS - marker.accessing_thread = thread_get_current_thread_id(); -#endif + vm_page marker; + init_page_marker(marker); - bool tried = false; - size_t stolen = 0; + uint32 pagesFreed = 0; - while (count > 0) { - vm_page *page = find_page_candidate(marker); - if (page == NULL) - break; + while (pagesFreed < pagesToFree) { + vm_page *page = find_cached_page_candidate(marker); + if (page == NULL) + break; - if (steal_page(page)) { - ReadLocker locker(sFreePageQueuesLock); - page->state = PAGE_STATE_FREE; - DEBUG_PAGE_ACCESS_END(page); - sFreePageQueue.PrependUnlocked(page); - locker.Unlock(); + if (free_cached_page(page, dontWait)) { + ReadLocker locker(sFreePageQueuesLock); + page->state = PAGE_STATE_FREE; + DEBUG_PAGE_ACCESS_END(page); + sFreePageQueue.PrependUnlocked(page); + locker.Unlock(); - unreserve_page(); + TA(StolenPage()); - T(StolenPage()); - - stolen++; - count--; - } else - tried = true; + pagesFreed++; } + } - remove_page_marker(marker); + remove_page_marker(marker); - if (count == 0 || sUnreservedFreePages >= 0) - return stolen; + return pagesFreed; +} - if (stolen && !tried && sInactivePageQueue.Count() > 0) { - count++; + +// TODO: Not needed when we remove the mappings of pages in the active scan when +// moved to the inactive queue. +#if 0 +static void +idle_scan_inactive_pages(page_stats& pageStats) +{ + VMPageQueue& queue = sInactivePageQueue; + + // We want to scan the whole queue in roughly kIdleRunsForFullQueue runs. + uint32 maxToScan = queue.Count() / kIdleRunsForFullQueue + 1; + + while (maxToScan > 0) { + maxToScan--; + + // Get the next page. Note that we don't bother to lock here. We go with + // the assumption that on all architectures reading/writing pointers is + // atomic. Beyond that it doesn't really matter. We have to unlock the + // queue anyway to lock the page's cache, and we'll recheck afterwards. + vm_page* page = queue.Head(); + if (page == NULL) + break; + + // lock the page's cache + VMCache* cache = vm_cache_acquire_locked_page_cache(page, true); + if (cache == NULL) + continue; + + if (page->state != PAGE_STATE_INACTIVE) { + // page is no longer in this queue + cache->ReleaseRefAndUnlock(); continue; } - // we need to wait for pages to become inactive + if (page->busy) { + // page is busy -- requeue at the end + vm_page_requeue(page, true); + cache->ReleaseRefAndUnlock(); + continue; + } + + DEBUG_PAGE_ACCESS_START(page); + + // Get the accessed count, clear the accessed/modified flags. + int32 usageCount = vm_clear_page_mapping_accessed_flags(page); + + // update usage count + if (usageCount > 0) { + usageCount += page->usage_count + kPageUsageAdvance; + if (usageCount > kPageUsageMax) + usageCount = kPageUsageMax; + } else { + usageCount += page->usage_count - (int32)kPageUsageDecline; + if (usageCount < 0) + usageCount = 0; + } + + page->usage_count = usageCount; + + // Move to fitting queue or requeue: + // * Active mapped pages go to the active queue. + // * Inactive mapped (i.e. wired) pages are requeued. + // * Inactive unmapped pages of a temporary cache are requeued. We don't + // want to page out memory until really necessary. + // * The remaining pages are cachable. Thus, if unmodified they go to + // the cached queue, otherwise to the modified queue. + bool isMapped = page->wired_count > 0 || !page->mappings.IsEmpty(); + if (usageCount > 0) { + if (isMapped) + set_page_state(page, PAGE_STATE_ACTIVE); + else + vm_page_requeue(page, true); + } else if (isMapped || cache->temporary) { + vm_page_requeue(page, true); + } else if (page->modified) { + set_page_state(page, PAGE_STATE_MODIFIED); + } else + set_page_state(page, PAGE_STATE_CACHED); + + DEBUG_PAGE_ACCESS_END(page); + + cache->ReleaseRefAndUnlock(); + } +} +#endif + + +static void +idle_scan_active_pages(page_stats& pageStats) +{ + VMPageQueue& queue = sActivePageQueue; + + // We want to scan the whole queue in roughly kIdleRunsForFullQueue runs. + uint32 maxToScan = queue.Count() / kIdleRunsForFullQueue + 1; + + while (maxToScan > 0) { + maxToScan--; + + // Get the next page. Note that we don't bother to lock here. We go with + // the assumption that on all architectures reading/writing pointers is + // atomic. Beyond that it doesn't really matter. We have to unlock the + // queue anyway to lock the page's cache, and we'll recheck afterwards. + vm_page* page = queue.Head(); + if (page == NULL) + break; + + // lock the page's cache + VMCache* cache = vm_cache_acquire_locked_page_cache(page, true); + if (cache == NULL) + continue; + + if (cache == NULL || page->state != PAGE_STATE_ACTIVE) { + // page is no longer in the cache or in this queue + cache->ReleaseRefAndUnlock(); + continue; + } + + if (page->busy) { + // page is busy -- requeue at the end + vm_page_requeue(page, true); + cache->ReleaseRefAndUnlock(); + continue; + } + + DEBUG_PAGE_ACCESS_START(page); + + // Get the page active/modified flags and update the page's usage count. + // We completely unmap inactive temporary pages. This saves us to + // iterate through the inactive list as well, since we'll be notified + // via page fault whenever such an inactive page is used again. + // We don't remove the mappings of non-temporary pages, since we + // wouldn't notice when those would become unused and could thus be + // moved to the cached list. + int32 usageCount; + if (page->wired_count > 0 || page->usage_count > 0 || !cache->temporary) + usageCount = vm_clear_page_mapping_accessed_flags(page); + else + usageCount = vm_remove_all_page_mappings_if_unaccessed(page); + + if (usageCount > 0) { + usageCount += page->usage_count + kPageUsageAdvance; + if (usageCount > kPageUsageMax) + usageCount = kPageUsageMax; +// TODO: This would probably also be the place to reclaim swap space. + } else { + usageCount += page->usage_count - (int32)kPageUsageDecline; + if (usageCount < 0) { + usageCount = 0; + set_page_state(page, PAGE_STATE_INACTIVE); + } + } + + page->usage_count = usageCount; + + DEBUG_PAGE_ACCESS_END(page); + + cache->ReleaseRefAndUnlock(); + } +} + + +static void +full_scan_inactive_pages(page_stats& pageStats, int32 despairLevel) +{ + int32 pagesToFree = pageStats.unsatisfiedReservations + + sFreeOrCachedPagesTarget + - (pageStats.totalFreePages + pageStats.cachedPages); + if (pagesToFree <= 0) + return; + + bigtime_t time = system_time(); + uint32 pagesScanned = 0; + uint32 pagesToCached = 0; + uint32 pagesToModified = 0; + uint32 pagesToActive = 0; + + // Determine how many pages at maximum to send to the modified queue. Since + // it is relatively expensive to page out pages, we do that on a grander + // scale only when things get desperate. + uint32 maxToFlush = despairLevel <= 1 ? 32 : 10000; + + vm_page marker; + init_page_marker(marker); + + VMPageQueue& queue = sInactivePageQueue; + InterruptsSpinLocker queueLocker(queue.GetLock()); + uint32 maxToScan = queue.Count(); + + vm_page* nextPage = queue.Head(); + + while (pagesToFree > 0 && maxToScan > 0) { + maxToScan--; + + // get the next page + vm_page* page = nextPage; + if (page == NULL) + break; + nextPage = queue.Next(page); + + if (page->busy) + continue; + + // mark the position + queue.InsertAfter(page, &marker); + queueLocker.Unlock(); + + // lock the page's cache + VMCache* cache = vm_cache_acquire_locked_page_cache(page, true); + if (cache == NULL || page->busy || page->state != PAGE_STATE_INACTIVE) { + if (cache != NULL) + cache->ReleaseRefAndUnlock(); + queueLocker.Lock(); + nextPage = queue.Next(&marker); + queue.Remove(&marker); + continue; + } + + pagesScanned++; + + DEBUG_PAGE_ACCESS_START(page); + + // Get the accessed count, clear the accessed/modified flags and + // unmap the page, if it hasn't been accessed. + int32 usageCount; + if (page->wired_count > 0) + usageCount = vm_clear_page_mapping_accessed_flags(page); + else + usageCount = vm_remove_all_page_mappings_if_unaccessed(page); + + // update usage count + if (usageCount > 0) { + usageCount += page->usage_count + kPageUsageAdvance; + if (usageCount > kPageUsageMax) + usageCount = kPageUsageMax; + } else { + usageCount += page->usage_count - (int32)kPageUsageDecline; + if (usageCount < 0) + usageCount = 0; + } + + page->usage_count = usageCount; + + // Move to fitting queue or requeue: + // * Active mapped pages go to the active queue. + // * Inactive mapped (i.e. wired) pages are requeued. + // * The remaining pages are cachable. Thus, if unmodified they go to + // the cached queue, otherwise to the modified queue (up to a limit). + // Note that until in the idle scanning we don't exempt pages of + // temporary caches. Apparently we really need memory, so we better + // page out memory as well. + bool isMapped = page->wired_count > 0 || !page->mappings.IsEmpty(); + if (usageCount > 0) { + if (isMapped) { + set_page_state(page, PAGE_STATE_ACTIVE); + pagesToActive++; + } else + vm_page_requeue(page, true); + } if (isMapped) { + vm_page_requeue(page, true); + } else if (!page->modified) { + set_page_state(page, PAGE_STATE_CACHED); + pagesToFree--; + pagesToCached++; + } else if (maxToFlush > 0) { + set_page_state(page, PAGE_STATE_MODIFIED); + maxToFlush--; + pagesToModified++; + } else + vm_page_requeue(page, true); + + DEBUG_PAGE_ACCESS_END(page); + + cache->ReleaseRefAndUnlock(); + + // remove the marker + queueLocker.Lock(); + nextPage = queue.Next(&marker); + queue.Remove(&marker); + } + + time = system_time() - time; + TRACE_DAEMON(" -> inactive scan (%7lld us): scanned: %7lu, " + "moved: %lu -> cached, %lu -> modified, %lu -> active\n", time, + pagesScanned, pagesToCached, pagesToModified, pagesToActive); + + // wake up the page writer, if we tossed it some pages + if (pagesToModified > 0) + sPageWriterCondition.WakeUp(); +} + + +static void +full_scan_active_pages(page_stats& pageStats, int32 despairLevel) +{ + vm_page marker; + init_page_marker(marker); + + VMPageQueue& queue = sActivePageQueue; + InterruptsSpinLocker queueLocker(queue.GetLock()); + uint32 maxToScan = queue.Count(); + + int32 pagesToDeactivate = pageStats.unsatisfiedReservations + + sFreeOrCachedPagesTarget + - (pageStats.totalFreePages + pageStats.cachedPages) + + std::max((int32)sInactivePagesTarget - (int32)maxToScan, (int32)0); + if (pagesToDeactivate <= 0) + return; + + bigtime_t time = system_time(); + uint32 pagesAccessed = 0; + uint32 pagesToInactive = 0; + uint32 pagesScanned = 0; + + vm_page* nextPage = queue.Head(); + + while (pagesToDeactivate > 0 && maxToScan > 0) { + maxToScan--; + + // get the next page + vm_page* page = nextPage; + if (page == NULL) + break; + nextPage = queue.Next(page); + + if (page->busy) + continue; + + // mark the position + queue.InsertAfter(page, &marker); + queueLocker.Unlock(); + + // lock the page's cache + VMCache* cache = vm_cache_acquire_locked_page_cache(page, true); + if (cache == NULL || page->busy || page->state != PAGE_STATE_ACTIVE) { + if (cache != NULL) + cache->ReleaseRefAndUnlock(); + queueLocker.Lock(); + nextPage = queue.Next(&marker); + queue.Remove(&marker); + continue; + } + + pagesScanned++; + + DEBUG_PAGE_ACCESS_START(page); + + // Get the page active/modified flags and update the page's usage count. + int32 usageCount = vm_clear_page_mapping_accessed_flags(page); + + if (usageCount > 0) { + usageCount += page->usage_count + kPageUsageAdvance; + if (usageCount > kPageUsageMax) + usageCount = kPageUsageMax; + pagesAccessed++; +// TODO: This would probably also be the place to reclaim swap space. + } else { + usageCount += page->usage_count - (int32)kPageUsageDecline; + if (usageCount <= 0) { + usageCount = 0; + set_page_state(page, PAGE_STATE_INACTIVE); + pagesToInactive++; + } + } + + page->usage_count = usageCount; + + DEBUG_PAGE_ACCESS_END(page); + + cache->ReleaseRefAndUnlock(); + + // remove the marker + queueLocker.Lock(); + nextPage = queue.Next(&marker); + queue.Remove(&marker); + } + + time = system_time() - time; + TRACE_DAEMON(" -> active scan (%7lld us): scanned: %7lu, " + "moved: %lu -> inactive, encountered %lu accessed ones\n", time, + pagesScanned, pagesToInactive, pagesAccessed); +} + + +static void +page_daemon_idle_scan(page_stats& pageStats) +{ + TRACE_DAEMON("page daemon: idle run\n"); + + if (pageStats.totalFreePages < (int32)sFreePagesTarget) { + // We want more actually free pages, so free some from the cached + // ones. + uint32 freed = free_cached_pages( + sFreePagesTarget - pageStats.totalFreePages, false); + if (freed > 0) + unreserve_pages(freed); + get_page_stats(pageStats); + } + + // Walk the inactive list and transfer pages to the cached and modified and + // active queues. +// get_page_stats(pageStats); +// idle_scan_inactive_pages(pageStats); + + // Walk the active list and move pages to the inactive queue. + get_page_stats(pageStats); + idle_scan_active_pages(pageStats); +} + + +static void +page_daemon_full_scan(page_stats& pageStats, int32 despairLevel) +{ + TRACE_DAEMON("page daemon: full run: free: %lu, cached: %lu, " + "to free: %lu\n", pageStats.totalFreePages, pageStats.cachedPages, + pageStats.unsatisfiedReservations + sFreeOrCachedPagesTarget + - (pageStats.totalFreePages + pageStats.cachedPages)); + + // Walk the inactive list and transfer pages to the cached and modified + // queues. + full_scan_inactive_pages(pageStats, despairLevel); + + // Free cached pages. Also wake up reservation waiters. + get_page_stats(pageStats); + int32 pagesToFree = pageStats.unsatisfiedReservations + sFreePagesTarget + - (pageStats.totalFreePages); + if (pagesToFree > 0) { + uint32 freed = free_cached_pages(pagesToFree, true); + if (freed > 0) + unreserve_pages(freed); + } + + // Walk the active list and move pages to the inactive queue. + get_page_stats(pageStats); + full_scan_active_pages(pageStats, despairLevel); +} + + +static status_t +page_daemon(void* /*unused*/) +{ + int32 despairLevel = 0; + + while (true) { + sPageDaemonCondition.ClearActivated(); + + // evaluate the free pages situation + page_stats pageStats; + get_page_stats(pageStats); + + if (pageStats.totalFreePages + pageStats.cachedPages + >= pageStats.unsatisfiedReservations + + (int32)sFreeOrCachedPagesTarget) { + // Things look good -- just maintain statistics and keep the pool + // of actually free pages full enough. + despairLevel = 0; + page_daemon_idle_scan(pageStats); + sPageDaemonCondition.Wait(kIdleScanWaitInterval, false); + } else { + // Not enough free pages. We need to do some real work. + despairLevel = std::max(despairLevel + 1, (int32)3); + page_daemon_full_scan(pageStats, despairLevel); + + // Don't wait after the first full scan, but rather immediately + // check whether we were successful in freeing enough pages and + // re-run with increased despair level. The first scan is + // conservative with respect to moving inactive modified pages to + // the modified list to avoid thrashing. The second scan, however, + // will not hold back. + if (despairLevel > 1) + snooze(kBusyScanWaitInterval); + } + } + + return B_OK; +} + + +/*! Returns how many pages could *not* be reserved. +*/ +static uint32 +reserve_pages(uint32 count, int priority, bool dontWait) +{ + int32 dontTouch = kPageReserveForPriority[priority]; + + while (true) { + count -= reserve_some_pages(count, dontTouch); + if (count == 0) + return 0; + + if (sUnsatisfiedPageReservations == 0) { + count -= free_cached_pages(count, dontWait); + if (count == 0) + return count; + } + + if (dontWait) + return count; + + // we need to wait for pages to become available MutexLocker pageDeficitLocker(sPageDeficitLock); - sPageDeficit++; - if (sUnreservedFreePages >= 0) { - // There are enough pages available now. No need to wait after all. - sPageDeficit--; - return stolen; + bool notifyDaemon = sUnsatisfiedPageReservations == 0; + sUnsatisfiedPageReservations += count; + + if (sUnreservedFreePages > dontTouch) { + // the situation changed + sUnsatisfiedPageReservations -= count; + continue; } - ConditionVariableEntry freeConditionEntry; - freeConditionEntry.Add(&sFreePageQueue); + PageReservationWaiter waiter; + waiter.dontTouch = dontTouch; + waiter.missing = count; + waiter.thread = thread_get_current_thread(); + waiter.threadPriority = waiter.thread->priority; + + // insert ordered (i.e. after all waiters with higher or equal priority) + PageReservationWaiter* otherWaiter = NULL; + for (PageReservationWaiterList::Iterator it + = sPageReservationWaiters.GetIterator(); + (otherWaiter = it.Next()) != NULL;) { + if (waiter < *otherWaiter) + break; + } + + sPageReservationWaiters.InsertBefore(otherWaiter, &waiter); + + thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_OTHER, + "waiting for pages"); + + if (notifyDaemon) + sPageDaemonCondition.WakeUp(); + pageDeficitLocker.Unlock(); - if (tried) { - // We tried all potential pages, but one or more couldn't be stolen - // at that time (likely because their cache was locked). No one - // else will have any better luck, so we'll just retry a little - // later. - freeConditionEntry.Wait(B_RELATIVE_TIMEOUT, 10000); - } else { - low_resource(B_KERNEL_RESOURCE_PAGES, count, B_RELATIVE_TIMEOUT, 0); - //snooze(50000); - // sleep for 50ms - - freeConditionEntry.Wait(); - } + low_resource(B_KERNEL_RESOURCE_PAGES, count, B_RELATIVE_TIMEOUT, 0); + thread_block(); pageDeficitLocker.Lock(); - sPageDeficit--; - if (sUnreservedFreePages >= 0) - return stolen; + return 0; } } @@ -1879,7 +2677,7 @@ vm_page_schedule_write_page(vm_page *page) vm_page_requeue(page, false); - release_sem_etc(sWriterWaitSem, 1, B_DO_NOT_RESCHEDULE); + sPageWriterCondition.WakeUp(); } @@ -1905,20 +2703,18 @@ vm_page_schedule_write_page_range(struct VMCache *cache, uint32 firstPage, } if (modified > 0) - release_sem_etc(sWriterWaitSem, 1, B_DO_NOT_RESCHEDULE); + sPageWriterCondition.WakeUp(); } void vm_page_init_num_pages(kernel_args *args) { - uint32 i; - // calculate the size of memory by looking at the physical_memory_range array addr_t physicalPagesEnd = 0; sPhysicalPageOffset = args->physical_memory_range[0].start / B_PAGE_SIZE; - for (i = 0; i < args->num_physical_memory_ranges; i++) { + for (uint32 i = 0; i < args->num_physical_memory_ranges; i++) { physicalPagesEnd = (args->physical_memory_range[i].start + args->physical_memory_range[i].size) / B_PAGE_SIZE; } @@ -1929,8 +2725,8 @@ vm_page_init_num_pages(kernel_args *args) sNumPages = physicalPagesEnd - sPhysicalPageOffset; #ifdef LIMIT_AVAILABLE_MEMORY - if (sNumPages > LIMIT_AVAILABLE_MEMORY * 256) - sNumPages = LIMIT_AVAILABLE_MEMORY * 256; + if (sNumPages > LIMIT_AVAILABLE_MEMORY * (1024 * 1024 / B_PAGE_SIZE)) + sNumPages = LIMIT_AVAILABLE_MEMORY * (1024 * 1024 / B_PAGE_SIZE); #endif } @@ -1948,6 +2744,8 @@ vm_page_init(kernel_args *args) sFreePageQueue.Init("free pages queue"); sClearPageQueue.Init("clear pages queue"); + new (&sPageReservationWaiters) PageReservationWaiterList; + // map in the new free page table sPages = (vm_page *)vm_allocate_early(args, sNumPages * sizeof(vm_page), ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, false); @@ -1977,21 +2775,31 @@ vm_page_init(kernel_args *args) TRACE(("initialized table\n")); - // mark some of the page ranges inuse + // mark some of the page ranges in use for (uint32 i = 0; i < args->num_physical_allocated_ranges; i++) { vm_mark_page_range_inuse(args->physical_allocated_range[i].start / B_PAGE_SIZE, args->physical_allocated_range[i].size / B_PAGE_SIZE); } - TRACE(("vm_page_init: exit\n")); + // The target of actually free pages. This must be at least the system + // reserve, but should be a few more pages, so we don't have to extract + // a cached page with each allocation. + sFreePagesTarget = VM_PAGE_RESERVE_USER + + std::max((uint32)32, sNumPages / 1024); - // reserve pages for the system, that user allocations will not touch - if (sUnreservedFreePages < (int32)kMinimumSystemReserve) { - panic("Less pages than the system reserve!"); - sSystemReservedPages = sUnreservedFreePages; - } else - sSystemReservedPages = kMinimumSystemReserve; - sUnreservedFreePages -= sSystemReservedPages; + // The target of free + cached and inactive pages. On low-memory machines + // keep things tight. free + cached is the pool of immediately allocatable + // pages. We want a few inactive pages, so when we're actually paging, we + // have a reasonably large set of pages to work with. + if (sUnreservedFreePages < 16 * 1024) { + sFreeOrCachedPagesTarget = sFreePagesTarget + 128; + sInactivePagesTarget = sFreePagesTarget / 3; + } else { + sFreeOrCachedPagesTarget = 2 * sFreePagesTarget; + sInactivePagesTarget = sFreePagesTarget / 2; + } + + TRACE(("vm_page_init: exit\n")); return B_OK; } @@ -2025,6 +2833,14 @@ vm_page_init_post_area(kernel_args *args) add_debugger_command("find_page", &find_page, "Find out which queue a page is actually in"); +#ifdef TRACK_PAGE_USAGE_STATS + add_debugger_command_etc("page_usage", &dump_page_usage_stats, + "Dumps statistics about page usage counts", + "\n" + "Dumps statistics about page usage counts.\n", + B_KDEBUG_DONT_PARSE_ARGUMENTS); +#endif + return B_OK; } @@ -2043,12 +2859,20 @@ vm_page_init_post_thread(kernel_args *args) // start page writer - sWriterWaitSem = create_sem(0, "page writer"); + sPageWriterCondition.Init("page writer"); thread = spawn_kernel_thread(&page_writer, "page writer", B_NORMAL_PRIORITY + 1, NULL); send_signal_etc(thread, SIGCONT, B_DO_NOT_RESCHEDULE); + // start page daemon + + sPageDaemonCondition.Init("page daemon"); + + thread = spawn_kernel_thread(&page_daemon, "page daemon", + B_NORMAL_PRIORITY, NULL); + send_signal_etc(thread, SIGCONT, B_DO_NOT_RESCHEDULE); + return B_OK; } @@ -2118,65 +2942,19 @@ vm_mark_page_range_inuse(addr_t startPage, addr_t length) /*! Unreserve pages previously reserved with vm_page_reserve_pages(). - Note, you specify the same \a count here that you specified when - reserving the pages - you don't need to keep track how many pages - you actually needed of that upfront allocation. */ void -vm_page_unreserve_pages(uint32 count) +vm_page_unreserve_pages(vm_page_reservation* reservation) { + uint32 count = reservation->count; + reservation->count = 0; + if (count == 0) return; - T(UnreservePages(count)); + TA(UnreservePages(count)); - while (true) { - int32 freePages = sUnreservedFreePages; - if (freePages >= 0) - break; - - int32 toUnreserve = std::min((int32)count, -freePages); - if (atomic_test_and_set(&sUnreservedFreePages, - freePages + toUnreserve, freePages) == freePages) { - count -= toUnreserve; - if (count == 0) { - // TODO: Notify waiting system priority reservers. - return; - } - break; - } - - // the count changed in the meantime -- retry - } - - while (true) { - int32 systemReserve = sSystemReservedPages; - if (systemReserve >= (int32)kMinimumSystemReserve) - break; - - int32 toUnreserve = std::min((int32)count, - (int32)kMinimumSystemReserve - systemReserve); - if (atomic_test_and_set(&sSystemReservedPages, - systemReserve + toUnreserve, systemReserve) - == systemReserve) { - count -= toUnreserve; - if (count == 0) { - // TODO: Notify waiting system priority reservers. - return; - } - break; - } - - // the count changed in the meantime -- retry - } - - atomic_add(&sUnreservedFreePages, count); - - if (sPageDeficit > 0) { - MutexLocker pageDeficitLocker(sPageDeficitLock); - if (sPageDeficit > 0) - sFreePageCondition.NotifyAll(); - } + unreserve_pages(count); } @@ -2187,131 +2965,52 @@ vm_page_unreserve_pages(uint32 count) The caller must not hold any cache lock or the function might deadlock. */ void -vm_page_reserve_pages(uint32 count, int priority) +vm_page_reserve_pages(vm_page_reservation* reservation, uint32 count, + int priority) { + reservation->count = count; + if (count == 0) return; - T(ReservePages(count)); + TA(ReservePages(count)); - while (true) { - // Of the requested count reserve as many pages as possible from the - // general reserve. - int32 freePages = sUnreservedFreePages; - if (freePages <= 0) - break; - - uint32 toReserve = std::min((int32)count, freePages); - if (atomic_test_and_set(&sUnreservedFreePages, - freePages - toReserve, freePages) - != freePages) { - // the count changed in the meantime -- retry - continue; - } - - count -= toReserve; - if (count == 0) - return; - - break; - } - - // Try to get the remaining pages from the system reserve. - uint32 systemReserve = kPageReserveForPriority[priority]; - while (true) { - int32 systemFreePages = sSystemReservedPages; - if (systemFreePages > (int32)systemReserve) { - uint32 toReserve = std::min(count, systemFreePages - systemReserve); - if (atomic_test_and_set(&sSystemReservedPages, - systemFreePages - toReserve, systemFreePages) - != systemFreePages) { - // the count changed in the meantime -- retry - continue; - } - - count -= toReserve; - if (count == 0) - return; - } - - break; - } - - // subtract the remaining pages - int32 oldFreePages = atomic_add(&sUnreservedFreePages, -(int32)count); - if (oldFreePages > 0) { - if ((int32)count <= oldFreePages) - return; - count -= oldFreePages; -// TODO: Activate low-memory handling/page daemon! - } - - steal_pages(NULL, count + 1); - // we get one more, just in case we can do something someone - // else can't + reserve_pages(count, priority, false); } bool -vm_page_try_reserve_pages(uint32 count, int priority) +vm_page_try_reserve_pages(vm_page_reservation* reservation, uint32 count, + int priority) { - if (count == 0) + if (count == 0) { + reservation->count = count; return true; - - uint32 reserved; - while (true) { - // From the requested count reserve as many pages as possible from the - // general reserve. - int32 freePages = sUnreservedFreePages; - if (freePages > 0) { - reserved = std::min((int32)count, freePages); - if (atomic_test_and_set(&sUnreservedFreePages, - freePages - reserved, freePages) - != freePages) { - // the count changed in the meantime -- retry - continue; - } - - if (reserved == count) { - T(ReservePages(count)); - return true; - } - } else - reserved = 0; - - break; } - // Try to get the remaining pages from the system reserve. - uint32 systemReserve = kPageReserveForPriority[priority]; - uint32 leftToReserve = count - reserved; - while (true) { - int32 systemFreePages = sSystemReservedPages; - if ((uint32)systemFreePages < leftToReserve + systemReserve) { - // no dice - vm_page_unreserve_pages(reserved); - return false; - } - - if (atomic_test_and_set(&sSystemReservedPages, - systemFreePages - leftToReserve, systemFreePages) - == systemFreePages) { - T(ReservePages(count)); - return true; - } - - // the count changed in the meantime -- retry + uint32 remaining = reserve_pages(count, priority, true); + if (remaining == 0) { + TA(ReservePages(count)); + reservation->count = count; + return true; } + + unreserve_pages(count - remaining); + + return false; } vm_page * -vm_page_allocate_page(uint32 flags) +vm_page_allocate_page(vm_page_reservation* reservation, uint32 flags) { uint32 pageState = flags & VM_PAGE_ALLOC_STATE; ASSERT(pageState != PAGE_STATE_FREE); ASSERT(pageState != PAGE_STATE_CLEAR); + ASSERT(reservation->count > 0); + reservation->count--; + VMPageQueue* queue; VMPageQueue* otherQueue; @@ -2323,9 +3022,7 @@ vm_page_allocate_page(uint32 flags) otherQueue = &sClearPageQueue; } - atomic_add(&sUnreservedFreePages, -1); - - T(AllocatePage()); + TA(AllocatePage()); ReadLocker locker(sFreePageQueuesLock); @@ -2364,7 +3061,7 @@ vm_page_allocate_page(uint32 flags) int oldPageState = page->state; page->state = pageState; page->busy = (flags & VM_PAGE_ALLOC_BUSY) != 0; - page->usage_count = 2; + page->usage_count = 0; page->accessed = false; page->modified = false; @@ -2390,7 +3087,7 @@ allocate_page_run(page_num_t start, page_num_t length, uint32 flags, ASSERT(pageState != PAGE_STATE_FREE); ASSERT(pageState != PAGE_STATE_CLEAR); - T(AllocatePageRun(length)); + TA(AllocatePageRun(length)); // pull the pages out of the appropriate queues VMPageQueue::PageList freePages; @@ -2408,7 +3105,7 @@ allocate_page_run(page_num_t start, page_num_t length, uint32 flags, page.state = flags & VM_PAGE_ALLOC_STATE; page.busy = flags & VM_PAGE_ALLOC_BUSY; - page.usage_count = 1; + page.usage_count = 0; page.accessed = false; page.modified = false; } @@ -2442,7 +3139,8 @@ vm_page_allocate_page_run(uint32 flags, addr_t base, addr_t length, { uint32 start = base >> PAGE_SHIFT; - if (!vm_page_try_reserve_pages(length, priority)) + vm_page_reservation reservation; + if (!vm_page_try_reserve_pages(&reservation, length, priority)) return NULL; // TODO: add more tries, ie. free some inactive, ... // no free space @@ -2453,7 +3151,7 @@ vm_page_allocate_page_run(uint32 flags, addr_t base, addr_t length, bool foundRun = true; if (start + length > sNumPages) { freeClearQueueLocker.Unlock(); - vm_page_unreserve_pages(length); + vm_page_unreserve_pages(&reservation); return NULL; } @@ -2489,7 +3187,8 @@ vm_page_allocate_page_run_no_base(uint32 flags, addr_t count, int priority) otherQueue = &sClearPageQueue; } - if (!vm_page_try_reserve_pages(count, priority)) + vm_page_reservation reservation; + if (!vm_page_try_reserve_pages(&reservation, count, priority)) return NULL; // TODO: add more tries, ie. free some inactive, ... // no free space @@ -2520,7 +3219,7 @@ vm_page_allocate_page_run_no_base(uint32 flags, addr_t count, int priority) if (queue == otherQueue) { freeClearQueueLocker.Unlock(); - vm_page_unreserve_pages(count); + vm_page_unreserve_pages(&reservation); } queue = otherQueue; @@ -2650,7 +3349,7 @@ vm_page_num_available_pages(void) size_t vm_page_num_free_pages(void) { - int32 count = sUnreservedFreePages + sInactivePageQueue.Count(); + int32 count = sUnreservedFreePages + sCachedPageQueue.Count(); return count > 0 ? count : 0; }