diff --git a/headers/private/kernel/vm/VMTranslationMap.h b/headers/private/kernel/vm/VMTranslationMap.h index 7cdf8bff79..01fc14926b 100644 --- a/headers/private/kernel/vm/VMTranslationMap.h +++ b/headers/private/kernel/vm/VMTranslationMap.h @@ -14,6 +14,7 @@ struct kernel_args; +struct vm_page_reservation; struct VMArea; @@ -32,13 +33,15 @@ struct VMTranslationMap { virtual status_t Map(addr_t virtualAddress, addr_t physicalAddress, - uint32 attributes) = 0; + uint32 attributes, + vm_page_reservation* reservation) = 0; virtual status_t Unmap(addr_t start, addr_t end) = 0; // map not locked - virtual status_t UnmapPage(VMArea* area, addr_t address) = 0; + virtual status_t UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue) = 0; virtual void UnmapPages(VMArea* area, addr_t base, - size_t size); + size_t size, bool updatePageQueue); virtual void UnmapArea(VMArea* area, bool deletingAddressSpace, bool ignoreTopCachePageFlags); @@ -60,6 +63,11 @@ struct VMTranslationMap { virtual status_t ClearFlags(addr_t virtualAddress, uint32 flags) = 0; + virtual bool ClearAccessedAndModified( + VMArea* area, addr_t address, + bool unmapIfUnaccessed, + bool& _modified) = 0; + virtual void Flush() = 0; protected: diff --git a/headers/private/kernel/vm/vm.h b/headers/private/kernel/vm/vm.h index 096ed7a286..7149edd3da 100644 --- a/headers/private/kernel/vm/vm.h +++ b/headers/private/kernel/vm/vm.h @@ -109,9 +109,10 @@ status_t vm_create_vnode_cache(struct vnode *vnode, struct VMCache **_cache); status_t vm_set_area_memory_type(area_id id, addr_t physicalBase, uint32 type); status_t vm_get_page_mapping(team_id team, addr_t vaddr, addr_t *paddr); bool vm_test_map_modification(struct vm_page *page); -int32 vm_test_map_activation(struct vm_page *page, bool *_modified); void vm_clear_map_flags(struct vm_page *page, uint32 flags); -void vm_remove_all_page_mappings(struct vm_page *page, uint32 *_flags); +void vm_remove_all_page_mappings(struct vm_page *page); +int32 vm_clear_page_mapping_accessed_flags(struct vm_page *page); +int32 vm_remove_all_page_mappings_if_unaccessed(struct vm_page *page); status_t vm_get_physical_page(addr_t paddr, addr_t* vaddr, void** _handle); status_t vm_put_physical_page(addr_t vaddr, void* handle); diff --git a/headers/private/kernel/vm/vm_page.h b/headers/private/kernel/vm/vm_page.h index df436c0a91..80279517a3 100644 --- a/headers/private/kernel/vm/vm_page.h +++ b/headers/private/kernel/vm/vm_page.h @@ -16,6 +16,12 @@ struct kernel_args; extern int32 gMappedPagesCount; + +struct vm_page_reservation { + uint32 count; +}; + + #ifdef __cplusplus extern "C" { #endif @@ -45,11 +51,14 @@ void vm_page_schedule_write_page(struct vm_page *page); void vm_page_schedule_write_page_range(struct VMCache *cache, uint32 firstPage, uint32 endPage); -void vm_page_unreserve_pages(uint32 count); -void vm_page_reserve_pages(uint32 count, int priority); -bool vm_page_try_reserve_pages(uint32 count, int priority); +void vm_page_unreserve_pages(vm_page_reservation* reservation); +void vm_page_reserve_pages(vm_page_reservation* reservation, uint32 count, + int priority); +bool vm_page_try_reserve_pages(vm_page_reservation* reservation, uint32 count, + int priority); -struct vm_page *vm_page_allocate_page(uint32 flags); +struct vm_page *vm_page_allocate_page(vm_page_reservation* reservation, + uint32 flags); struct vm_page *vm_page_allocate_page_run(uint32 flags, addr_t base, addr_t length, int priority); struct vm_page *vm_page_allocate_page_run_no_base(uint32 flags, addr_t count, diff --git a/headers/private/kernel/vm/vm_priv.h b/headers/private/kernel/vm/vm_priv.h index b176a173d0..afb15a714a 100644 --- a/headers/private/kernel/vm/vm_priv.h +++ b/headers/private/kernel/vm/vm_priv.h @@ -31,7 +31,6 @@ status_t vm_page_fault(addr_t address, addr_t faultAddress, bool isWrite, bool isUser, addr_t *newip); void vm_unreserve_memory(size_t bytes); status_t vm_try_reserve_memory(size_t bytes, int priority, bigtime_t timeout); -void vm_schedule_page_scanner(uint32 target); status_t vm_daemon_init(void); const char *page_state_to_string(int state); diff --git a/headers/private/kernel/vm/vm_types.h b/headers/private/kernel/vm/vm_types.h index c685695028..178f196e7a 100644 --- a/headers/private/kernel/vm/vm_types.h +++ b/headers/private/kernel/vm/vm_types.h @@ -113,7 +113,7 @@ public: bool modified : 1; uint8 unused : 1; - int8 usage_count; + uint8 usage_count; uint16 wired_count; VMCacheRef* CacheRef() const { return cache_ref; } diff --git a/src/add-ons/kernel/bus_managers/agp_gart/agp_gart.cpp b/src/add-ons/kernel/bus_managers/agp_gart/agp_gart.cpp index 69b5220a08..0833823b88 100644 --- a/src/add-ons/kernel/bus_managers/agp_gart/agp_gart.cpp +++ b/src/add-ons/kernel/bus_managers/agp_gart/agp_gart.cpp @@ -550,12 +550,13 @@ Aperture::AllocateMemory(aperture_memory *memory, uint32 flags) if (memory->pages == NULL) return B_NO_MEMORY; - vm_page_reserve_pages(count, VM_PRIORITY_SYSTEM); + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, count, VM_PRIORITY_SYSTEM); for (uint32 i = 0; i < count; i++) { - memory->pages[i] = vm_page_allocate_page( + memory->pages[i] = vm_page_allocate_page(&reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); } - vm_page_unreserve_pages(count); + vm_page_unreserve_pages(&reservation); } #ifdef DEBUG_PAGE_ACCESS diff --git a/src/system/kernel/arch/ppc/arch_vm_translation_map.cpp b/src/system/kernel/arch/ppc/arch_vm_translation_map.cpp index a0c79aacc4..5f38346c03 100644 --- a/src/system/kernel/arch/ppc/arch_vm_translation_map.cpp +++ b/src/system/kernel/arch/ppc/arch_vm_translation_map.cpp @@ -147,10 +147,12 @@ struct PPCVMTranslationMap : VMTranslationMap { virtual status_t Map(addr_t virtualAddress, addr_t physicalAddress, - uint32 attributes); + uint32 attributes, + vm_page_reservation* reservation); virtual status_t Unmap(addr_t start, addr_t end); - virtual status_t UnmapPage(VMArea* area, addr_t address); + virtual status_t UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue); virtual status_t Query(addr_t virtualAddress, addr_t* _physicalAddress, @@ -164,6 +166,11 @@ struct PPCVMTranslationMap : VMTranslationMap { virtual status_t ClearFlags(addr_t virtualAddress, uint32 flags); + virtual bool ClearAccessedAndModified( + VMArea* area, addr_t address, + bool unmapIfUnaccessed, + bool& _modified); + virtual void Flush(); protected: @@ -389,7 +396,7 @@ PPCVMTranslationMap::MaxPagesNeededToMap(addr_t start, addr_t end) const status_t PPCVMTranslationMap::Map(addr_t virtualAddress, addr_t physicalAddress, - uint32 attributes) + uint32 attributes, vm_page_reservation* reservation) { // lookup the vsid based off the va uint32 virtualSegmentID = VADDR_TO_VSID(fVSIDBase, virtualAddress); @@ -463,7 +470,8 @@ PPCVMTranslationMap::Unmap(addr_t start, addr_t end) status_t -PPCVMTranslationMap::UnmapPage(VMArea* area, addr_t address) +PPCVMTranslationMap::UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue) { ASSERT(address % B_PAGE_SIZE == 0); @@ -513,10 +521,20 @@ PPCVMTranslationMap::UnmapPage(VMArea* area, addr_t address) } else page->wired_count--; - if (page->wired_count == 0 && page->mappings.IsEmpty()) + locker.Unlock(); + + if (page->wired_count == 0 && page->mappings.IsEmpty()) { atomic_add(&gMappedPagesCount, -1); - locker.Unlock(); + if (updatePageQueue) { + if (page->Cache()->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + } + } if (mapping != NULL) { bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); @@ -604,6 +622,52 @@ PPCVMTranslationMap::ClearFlags(addr_t virtualAddress, uint32 flags) } +bool +PPCVMTranslationMap::ClearAccessedAndModified(VMArea* area, addr_t address, + bool unmapIfUnaccessed, bool& _modified) +{ + // TODO: Implement for real! ATM this is just an approximation using + // Query(), ClearFlags(), and UnmapPage(). See below! + + RecursiveLocker locker(fLock); + + uint32 flags; + addr_t physicalAddress; + if (Query(address, &physicalAddress, &flags) != B_OK + || (flags & PAGE_PRESENT) == 0) { + return false; + } + + _modified = (flags & PAGE_MODIFIED) != 0; + + if ((flags & (PAGE_ACCESSED | PAGE_MODIFIED)) != 0) + ClearFlags(address, flags & (PAGE_ACCESSED | PAGE_MODIFIED)); + + if ((flags & PAGE_ACCESSED) != 0) + return true; + + if (!unmapIfUnaccessed) + return false; + + locker.Unlock(); + + UnmapPage(area, address, false); + // TODO: Obvious race condition: Between querying and unmapping the + // page could have been accessed. We try to compensate by considering + // vm_page::{accessed,modified} (which would have been updated by + // UnmapPage()) below, but that doesn't quite match the required + // semantics of the method. + + vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE); + if (page == NULL) + return false; + + _modified |= page->modified; + + return page->accessed; +} + + void PPCVMTranslationMap::Flush() { @@ -786,11 +850,15 @@ ppc_map_address_range(addr_t virtualAddress, addr_t physicalAddress, PPCVMTranslationMap* map = static_cast( addressSpace->TranslationMap()); + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, 0, VM_PRIORITY_USER); + // We don't need any pages for mapping. + // map the pages for (; virtualAddress < virtualEnd; virtualAddress += B_PAGE_SIZE, physicalAddress += B_PAGE_SIZE) { status_t error = map->Map(virtualAddress, physicalAddress, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, &reservation); if (error != B_OK) return error; } diff --git a/src/system/kernel/arch/x86/X86VMTranslationMap.h b/src/system/kernel/arch/x86/X86VMTranslationMap.h index 8a63c5918b..2310d8aa38 100644 --- a/src/system/kernel/arch/x86/X86VMTranslationMap.h +++ b/src/system/kernel/arch/x86/X86VMTranslationMap.h @@ -31,12 +31,14 @@ struct X86VMTranslationMap : VMTranslationMap { virtual status_t Map(addr_t virtualAddress, addr_t physicalAddress, - uint32 attributes); + uint32 attributes, + vm_page_reservation* reservation); virtual status_t Unmap(addr_t start, addr_t end); - virtual status_t UnmapPage(VMArea* area, addr_t address); + virtual status_t UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue); virtual void UnmapPages(VMArea* area, addr_t base, - size_t size); + size_t size, bool updatePageQueue); virtual void UnmapArea(VMArea* area, bool deletingAddressSpace, bool ignoreTopCachePageFlags); @@ -54,6 +56,11 @@ struct X86VMTranslationMap : VMTranslationMap { virtual status_t ClearFlags(addr_t virtualAddress, uint32 flags); + virtual bool ClearAccessedAndModified( + VMArea* area, addr_t address, + bool unmapIfUnaccessed, + bool& _modified); + virtual void Flush(); protected: diff --git a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp index a551d0b096..18a080c88e 100644 --- a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp +++ b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp @@ -401,7 +401,8 @@ X86VMTranslationMap::MaxPagesNeededToMap(addr_t start, addr_t end) const status_t -X86VMTranslationMap::Map(addr_t va, addr_t pa, uint32 attributes) +X86VMTranslationMap::Map(addr_t va, addr_t pa, uint32 attributes, + vm_page_reservation* reservation) { TRACE(("map_tmap: entry pa 0x%lx va 0x%lx\n", pa, va)); @@ -422,7 +423,8 @@ X86VMTranslationMap::Map(addr_t va, addr_t pa, uint32 attributes) vm_page *page; // we need to allocate a pgtable - page = vm_page_allocate_page(PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); + page = vm_page_allocate_page(reservation, + PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); DEBUG_PAGE_ACCESS_END(page); @@ -533,7 +535,8 @@ restart: This object shouldn't be locked. */ status_t -X86VMTranslationMap::UnmapPage(VMArea* area, addr_t address) +X86VMTranslationMap::UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue) { ASSERT(address % B_PAGE_SIZE == 0); @@ -620,10 +623,20 @@ X86VMTranslationMap::UnmapPage(VMArea* area, addr_t address) } else page->wired_count--; - if (page->wired_count == 0 && page->mappings.IsEmpty()) + locker.Unlock(); + + if (page->wired_count == 0 && page->mappings.IsEmpty()) { atomic_add(&gMappedPagesCount, -1); - locker.Unlock(); + if (updatePageQueue) { + if (page->Cache()->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + } + } if (mapping != NULL) { bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); @@ -637,7 +650,8 @@ X86VMTranslationMap::UnmapPage(VMArea* area, addr_t address) void -X86VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size) +X86VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size, + bool updatePageQueue) { page_directory_entry* pd = fArchData->pgdir_virt; @@ -720,13 +734,23 @@ X86VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size) } else page->wired_count--; - if (page->wired_count == 0 && page->mappings.IsEmpty()) + if (page->wired_count == 0 && page->mappings.IsEmpty()) { atomic_add(&gMappedPagesCount, -1); + + if (updatePageQueue) { + if (page->Cache()->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + } + } } } Flush(); - // flush explicitely, since we directly use the lock + // flush explicitly, since we directly use the lock pinner.Unlock(); } @@ -752,7 +776,7 @@ X86VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace, bool ignoreTopCachePageFlags) { if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - X86VMTranslationMap::UnmapPages(area, area->Base(), area->Size()); + X86VMTranslationMap::UnmapPages(area, area->Base(), area->Size(), true); return; } @@ -770,10 +794,22 @@ X86VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace, vm_page* page = mapping->page; page->mappings.Remove(mapping); - if (page->wired_count == 0 && page->mappings.IsEmpty()) + VMCache* cache = page->Cache(); + + if (page->wired_count == 0 && page->mappings.IsEmpty()) { atomic_add(&gMappedPagesCount, -1); - if (unmapPages || page->Cache() != area->cache) { + if (!ignoreTopCachePageFlags || cache != area->cache) { + if (cache->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + } + } + + if (unmapPages || cache != area->cache) { addr_t address = area->Base() + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); @@ -1040,6 +1076,129 @@ X86VMTranslationMap::ClearFlags(addr_t va, uint32 flags) } +bool +X86VMTranslationMap::ClearAccessedAndModified(VMArea* area, addr_t address, + bool unmapIfUnaccessed, bool& _modified) +{ + ASSERT(address % B_PAGE_SIZE == 0); + + page_directory_entry* pd = fArchData->pgdir_virt; + + TRACE(("X86VMTranslationMap::ClearAccessedAndModified(%#" B_PRIxADDR ")\n", + address)); + + RecursiveLocker locker(fLock); + + int index = VADDR_TO_PDENT(address); + if ((pd[index] & X86_PDE_PRESENT) == 0) + return false; + + ThreadCPUPinner pinner(thread_get_current_thread()); + + page_table_entry* pt = fArchData->page_mapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + + index = VADDR_TO_PTENT(address); + + // perform the deed + page_table_entry oldEntry; + + if (unmapIfUnaccessed) { + while (true) { + oldEntry = pt[index]; + if ((oldEntry & X86_PTE_PRESENT) == 0) { + // page mapping not valid + return false; + } + + if (oldEntry & X86_PTE_ACCESSED) { + // page was accessed -- just clear the flags + oldEntry = clear_page_table_entry_flags(&pt[index], + X86_PTE_ACCESSED | X86_PTE_DIRTY); + break; + } + + // page hasn't been accessed -- unmap it + if (test_and_set_page_table_entry(&pt[index], 0, oldEntry) + == oldEntry) { + break; + } + + // something changed -- check again + } + } else { + oldEntry = clear_page_table_entry_flags(&pt[index], + X86_PTE_ACCESSED | X86_PTE_DIRTY); + } + + pinner.Unlock(); + + _modified = (oldEntry & X86_PTE_DIRTY) != 0; + + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + // Note, that we only need to invalidate the address, if the + // accessed flags was set, since only then the entry could have been + // in any TLB. + if (fArchData->num_invalidate_pages + < PAGE_INVALIDATE_CACHE_SIZE) { + fArchData->pages_to_invalidate[fArchData->num_invalidate_pages] + = address; + } + + fArchData->num_invalidate_pages++; + + Flush(); + + return true; + } + + if (!unmapIfUnaccessed) + return false; + + // We have unmapped the address. Do the "high level" stuff. + + fMapCount--; + + if (area->cache_type == CACHE_TYPE_DEVICE) + return false; + + // get the page + vm_page* page = vm_lookup_page( + (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); + ASSERT(page != NULL); + + // remove the mapping object/decrement the wired_count of the page + vm_page_mapping* mapping = NULL; + if (area->wiring == B_NO_LOCK) { + vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); + while ((mapping = iterator.Next()) != NULL) { + if (mapping->area == area) { + area->mappings.Remove(mapping); + page->mappings.Remove(mapping); + break; + } + } + + ASSERT(mapping != NULL); + } else + page->wired_count--; + + locker.Unlock(); + + if (page->wired_count == 0 && page->mappings.IsEmpty()) + atomic_add(&gMappedPagesCount, -1); + + if (mapping != NULL) { + object_cache_free(gPageMappingsObjectCache, mapping, + CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); + // Since this is called by the page daemon, we never want to lock + // the kernel address space. + } + + return false; +} + + void X86VMTranslationMap::Flush() { diff --git a/src/system/kernel/cache/file_cache.cpp b/src/system/kernel/cache/file_cache.cpp index 9aad7b7115..71340d81ba 100644 --- a/src/system/kernel/cache/file_cache.cpp +++ b/src/system/kernel/cache/file_cache.cpp @@ -75,7 +75,7 @@ public: size_t size); ~PrecacheIO(); - status_t Prepare(); + status_t Prepare(vm_page_reservation* reservation); void ReadAsync(); virtual void IOFinished(status_t status, @@ -99,7 +99,7 @@ private: typedef status_t (*cache_func)(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer, - size_t lastReservedPages, size_t reservePages); + vm_page_reservation* reservation, size_t reservePages); static void add_to_iovec(iovec* vecs, uint32 &index, uint32 max, addr_t address, size_t size); @@ -141,7 +141,7 @@ PrecacheIO::~PrecacheIO() status_t -PrecacheIO::Prepare() +PrecacheIO::Prepare(vm_page_reservation* reservation) { if (fPageCount == 0) return B_BAD_VALUE; @@ -157,8 +157,8 @@ PrecacheIO::Prepare() // allocate pages for the cache and mark them busy uint32 i = 0; for (size_t pos = 0; pos < fSize; pos += B_PAGE_SIZE) { - vm_page* page = vm_page_allocate_page( - PAGE_STATE_ACTIVE | VM_PAGE_ALLOC_BUSY); + vm_page* page = vm_page_allocate_page(reservation, + PAGE_STATE_CACHED | VM_PAGE_ALLOC_BUSY); fCache->InsertPage(page, fOffset + pos); @@ -281,7 +281,8 @@ push_access(file_cache_ref* ref, off_t offset, size_t bytes, bool isWrite) static void -reserve_pages(file_cache_ref* ref, size_t reservePages, bool isWrite) +reserve_pages(file_cache_ref* ref, vm_page_reservation* reservation, + size_t reservePages, bool isWrite) { if (low_resource_state(B_KERNEL_RESOURCE_PAGES) != B_NO_LOW_RESOURCE) { VMCache* cache = ref->cache; @@ -320,7 +321,7 @@ reserve_pages(file_cache_ref* ref, size_t reservePages, bool isWrite) cache->Unlock(); } - vm_page_reserve_pages(reservePages, VM_PRIORITY_USER); + vm_page_reserve_pages(reservation, reservePages, VM_PRIORITY_USER); } @@ -367,7 +368,7 @@ read_pages_and_clear_partial(file_cache_ref* ref, void* cookie, off_t offset, static status_t read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer, - size_t lastReservedPages, size_t reservePages) + vm_page_reservation* reservation, size_t reservePages) { TRACE(("read_into_cache(offset = %Ld, pageOffset = %ld, buffer = %#lx, " "bufferSize = %lu\n", offset, pageOffset, buffer, bufferSize)); @@ -386,7 +387,7 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, // allocate pages for the cache and mark them busy for (size_t pos = 0; pos < numBytes; pos += B_PAGE_SIZE) { vm_page* page = pages[pageIndex++] = vm_page_allocate_page( - PAGE_STATE_ACTIVE | VM_PAGE_ALLOC_BUSY); + reservation, PAGE_STATE_CACHED | VM_PAGE_ALLOC_BUSY); cache->InsertPage(page, offset + pos); @@ -397,7 +398,7 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, push_access(ref, offset, bufferSize, false); cache->Unlock(); - vm_page_unreserve_pages(lastReservedPages); + vm_page_unreserve_pages(reservation); // read file into reserved pages status_t status = read_pages_and_clear_partial(ref, cookie, offset, vecs, @@ -434,7 +435,7 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, } } - reserve_pages(ref, reservePages, false); + reserve_pages(ref, reservation, reservePages, false); cache->Lock(); // make the pages accessible in the cache @@ -451,7 +452,7 @@ read_into_cache(file_cache_ref* ref, void* cookie, off_t offset, static status_t read_from_file(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer, - size_t lastReservedPages, size_t reservePages) + vm_page_reservation* reservation, size_t reservePages) { TRACE(("read_from_file(offset = %Ld, pageOffset = %ld, buffer = %#lx, " "bufferSize = %lu\n", offset, pageOffset, buffer, bufferSize)); @@ -465,13 +466,13 @@ read_from_file(file_cache_ref* ref, void* cookie, off_t offset, push_access(ref, offset, bufferSize, false); ref->cache->Unlock(); - vm_page_unreserve_pages(lastReservedPages); + vm_page_unreserve_pages(reservation); status_t status = vfs_read_pages(ref->vnode, cookie, offset + pageOffset, &vec, 1, 0, &bufferSize); if (status == B_OK) - reserve_pages(ref, reservePages, false); + reserve_pages(ref, reservation, reservePages, false); ref->cache->Lock(); @@ -487,7 +488,7 @@ read_from_file(file_cache_ref* ref, void* cookie, off_t offset, static status_t write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer, - size_t lastReservedPages, size_t reservePages) + vm_page_reservation* reservation, size_t reservePages) { // TODO: We're using way too much stack! Rather allocate a sufficiently // large chunk on the heap. @@ -509,7 +510,9 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, // TODO: the pages we allocate here should have been reserved upfront // in cache_io() vm_page* page = pages[pageIndex++] = vm_page_allocate_page( - PAGE_STATE_ACTIVE | VM_PAGE_ALLOC_BUSY); + reservation, + (writeThrough ? PAGE_STATE_CACHED : PAGE_STATE_MODIFIED) + | VM_PAGE_ALLOC_BUSY); ref->cache->InsertPage(page, offset + pos); @@ -519,7 +522,7 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, push_access(ref, offset, bufferSize, true); ref->cache->Unlock(); - vm_page_unreserve_pages(lastReservedPages); + vm_page_unreserve_pages(reservation); // copy contents (and read in partially written pages first) @@ -601,7 +604,7 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, } if (status == B_OK) - reserve_pages(ref, reservePages, true); + reserve_pages(ref, reservation, reservePages, true); ref->cache->Lock(); @@ -609,9 +612,6 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, for (int32 i = pageIndex; i-- > 0;) { ref->cache->MarkPageUnbusy(pages[i]); - if (!writeThrough) - vm_page_set_state(pages[i], PAGE_STATE_MODIFIED); - DEBUG_PAGE_ACCESS_END(pages[i]); } @@ -621,12 +621,12 @@ write_to_cache(file_cache_ref* ref, void* cookie, off_t offset, static status_t write_to_file(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, - addr_t buffer, size_t bufferSize, bool useBuffer, size_t lastReservedPages, - size_t reservePages) + addr_t buffer, size_t bufferSize, bool useBuffer, + vm_page_reservation* reservation, size_t reservePages) { push_access(ref, offset, bufferSize, true); ref->cache->Unlock(); - vm_page_unreserve_pages(lastReservedPages); + vm_page_unreserve_pages(reservation); status_t status = B_OK; @@ -652,7 +652,7 @@ write_to_file(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset, } if (status == B_OK) - reserve_pages(ref, reservePages, true); + reserve_pages(ref, reservation, reservePages, true); ref->cache->Lock(); @@ -665,7 +665,7 @@ satisfy_cache_io(file_cache_ref* ref, void* cookie, cache_func function, off_t offset, addr_t buffer, bool useBuffer, int32 &pageOffset, size_t bytesLeft, size_t &reservePages, off_t &lastOffset, addr_t &lastBuffer, int32 &lastPageOffset, size_t &lastLeft, - size_t &lastReservedPages) + size_t &lastReservedPages, vm_page_reservation* reservation) { if (lastBuffer == buffer) return B_OK; @@ -675,7 +675,7 @@ satisfy_cache_io(file_cache_ref* ref, void* cookie, cache_func function, + lastPageOffset + B_PAGE_SIZE - 1) >> PAGE_SHIFT); status_t status = function(ref, cookie, lastOffset, lastPageOffset, - lastBuffer, requestSize, useBuffer, lastReservedPages, reservePages); + lastBuffer, requestSize, useBuffer, reservation, reservePages); if (status == B_OK) { lastReservedPages = reservePages; lastBuffer = buffer; @@ -736,7 +736,9 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, size_t pagesProcessed = 0; cache_func function = NULL; - reserve_pages(ref, lastReservedPages, doWrite); + vm_page_reservation reservation; + reserve_pages(ref, &reservation, lastReservedPages, doWrite); + AutoLocker locker(cache); while (bytesLeft > 0) { @@ -763,7 +765,7 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, status_t status = satisfy_cache_io(ref, cookie, function, offset, buffer, useBuffer, pageOffset, bytesLeft, reservePages, lastOffset, lastBuffer, lastPageOffset, lastLeft, - lastReservedPages); + lastReservedPages, &reservation); if (status != B_OK) return status; @@ -779,13 +781,6 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, "= %lu\n", offset, page, bytesLeft, pageOffset)); if (page != NULL) { - // Since we don't actually map pages as part of an area, we have - // to manually maintain their usage_count - page->usage_count = 2; - // TODO: Just because this request comes from the FS API, it - // doesn't mean the page is not mapped. We might actually - // decrease the usage count of a hot page here. - if (doWrite || useBuffer) { // Since the following user_mem{cpy,set}() might cause a page // fault, which in turn might cause pages to be reserved, we @@ -824,10 +819,19 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, if (bytesLeft <= bytesInPage) { // we've read the last page, so we're done! locker.Unlock(); - vm_page_unreserve_pages(lastReservedPages); + vm_page_unreserve_pages(&reservation); return B_OK; } + // If it is cached only, requeue the page, so the respective queue + // roughly remains LRU first sorted. + if (page->state == PAGE_STATE_CACHED + || page->state == PAGE_STATE_MODIFIED) { + DEBUG_PAGE_ACCESS_START(page); + vm_page_requeue(page, true); + DEBUG_PAGE_ACCESS_END(page); + } + // prepare a potential gap request lastBuffer = buffer + bytesInPage; lastLeft = bytesLeft - bytesInPage; @@ -848,7 +852,7 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, status_t status = satisfy_cache_io(ref, cookie, function, offset, buffer, useBuffer, pageOffset, bytesLeft, reservePages, lastOffset, lastBuffer, lastPageOffset, lastLeft, - lastReservedPages); + lastReservedPages, &reservation); if (status != B_OK) return status; } @@ -857,7 +861,7 @@ cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer, // fill the last remaining bytes of the request (either write or read) return function(ref, cookie, lastOffset, lastPageOffset, lastBuffer, - lastLeft, useBuffer, lastReservedPages, 0); + lastLeft, useBuffer, &reservation, 0); } @@ -929,6 +933,11 @@ cache_prefetch_vnode(struct vnode* vnode, off_t offset, size_t size) if (offset + size > fileSize) size = fileSize - offset; + + // "offset" and "size" are always aligned to B_PAGE_SIZE, + offset = ROUNDDOWN(offset, B_PAGE_SIZE); + size = ROUNDUP(size, B_PAGE_SIZE); + size_t reservePages = size / B_PAGE_SIZE; // Don't do anything if we don't have the resources left, or the cache @@ -939,14 +948,11 @@ cache_prefetch_vnode(struct vnode* vnode, off_t offset, size_t size) return; } - // "offset" and "size" are always aligned to B_PAGE_SIZE, - offset &= ~(B_PAGE_SIZE - 1); - size = ROUNDUP(size, B_PAGE_SIZE); - size_t bytesToRead = 0; off_t lastOffset = offset; - vm_page_reserve_pages(reservePages, VM_PRIORITY_USER); + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, reservePages, VM_PRIORITY_USER); cache->Lock(); @@ -965,9 +971,9 @@ cache_prefetch_vnode(struct vnode* vnode, off_t offset, size_t size) } if (bytesToRead != 0) { // read the part before the current page (or the end of the request) - PrecacheIO* io - = new(std::nothrow) PrecacheIO(ref, lastOffset, bytesToRead); - if (io == NULL || io->Prepare() != B_OK) { + PrecacheIO* io = new(std::nothrow) PrecacheIO(ref, lastOffset, + bytesToRead); + if (io == NULL || io->Prepare(&reservation) != B_OK) { delete io; break; } @@ -989,9 +995,7 @@ cache_prefetch_vnode(struct vnode* vnode, off_t offset, size_t size) } cache->ReleaseRefAndUnlock(); - vm_page_unreserve_pages(reservePages); - // TODO: We should periodically unreserve as we go, so we don't - // unnecessarily put pressure on the free page pool. + vm_page_unreserve_pages(&reservation); } @@ -1074,10 +1078,11 @@ extern "C" status_t file_cache_init(void) { // allocate a clean page we can use for writing zeroes - vm_page_reserve_pages(1, VM_PRIORITY_SYSTEM); - vm_page* page = vm_page_allocate_page( + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, 1, VM_PRIORITY_SYSTEM); + vm_page* page = vm_page_allocate_page(&reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); - vm_page_unreserve_pages(1); + vm_page_unreserve_pages(&reservation); sZeroPage = (addr_t)page->physical_page_number * B_PAGE_SIZE; @@ -1319,4 +1324,3 @@ file_cache_write(void* _cacheRef, void* cookie, off_t offset, return status; } - diff --git a/src/system/kernel/low_resource_manager.cpp b/src/system/kernel/low_resource_manager.cpp index 241c540bf6..3647ae77d9 100644 --- a/src/system/kernel/low_resource_manager.cpp +++ b/src/system/kernel/low_resource_manager.cpp @@ -301,9 +301,6 @@ low_resource(uint32 resource, uint64 requirements, uint32 flags, uint32 timeout) switch (resource) { case B_KERNEL_RESOURCE_PAGES: - vm_schedule_page_scanner(requirements); - break; - case B_KERNEL_RESOURCE_MEMORY: case B_KERNEL_RESOURCE_SEMAPHORES: case B_KERNEL_RESOURCE_ADDRESS_SPACE: diff --git a/src/system/kernel/slab/MemoryManager.cpp b/src/system/kernel/slab/MemoryManager.cpp index 0bb76db760..ff6d1f789e 100644 --- a/src/system/kernel/slab/MemoryManager.cpp +++ b/src/system/kernel/slab/MemoryManager.cpp @@ -1333,13 +1333,14 @@ MemoryManager::_MapChunk(VMArea* vmArea, addr_t address, size_t size, // reserve the pages we need now size_t reservedPages = size / B_PAGE_SIZE + translationMap->MaxPagesNeededToMap(address, address + size - 1); + vm_page_reservation reservation; if ((flags & CACHE_DONT_WAIT_FOR_MEMORY) != 0) { - if (!vm_page_try_reserve_pages(reservedPages, priority)) { + if (!vm_page_try_reserve_pages(&reservation, reservedPages, priority)) { vm_unreserve_memory(reservedMemory); return B_WOULD_BLOCK; } } else - vm_page_reserve_pages(reservedPages, priority); + vm_page_reserve_pages(&reservation, reservedPages, priority); VMCache* cache = vm_area_get_locked_cache(vmArea); @@ -1350,7 +1351,7 @@ MemoryManager::_MapChunk(VMArea* vmArea, addr_t address, size_t size, addr_t endAreaOffset = areaOffset + size; for (size_t offset = areaOffset; offset < endAreaOffset; offset += B_PAGE_SIZE) { - vm_page* page = vm_page_allocate_page(PAGE_STATE_WIRED); + vm_page* page = vm_page_allocate_page(&reservation, PAGE_STATE_WIRED); cache->InsertPage(page, offset); page->wired_count++; @@ -1359,14 +1360,14 @@ MemoryManager::_MapChunk(VMArea* vmArea, addr_t address, size_t size, translationMap->Map(vmArea->Base() + offset, page->physical_page_number * B_PAGE_SIZE, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, &reservation); } translationMap->Unlock(); cache->ReleaseRefAndUnlock(); - vm_page_unreserve_pages(reservedPages); + vm_page_unreserve_pages(&reservation); return B_OK; } diff --git a/src/system/kernel/vm/Jamfile b/src/system/kernel/vm/Jamfile index 595901e806..fa0282d658 100644 --- a/src/system/kernel/vm/Jamfile +++ b/src/system/kernel/vm/Jamfile @@ -5,8 +5,8 @@ UsePrivateHeaders shared ; UseHeaders [ FDirName $(SUBDIR) $(DOTDOT) device_manager ] ; KernelMergeObject kernel_vm.o : + PageCacheLocker.cpp vm.cpp - vm_daemons.cpp vm_page.cpp VMAddressSpace.cpp VMAddressSpaceLocking.cpp diff --git a/src/system/kernel/vm/PageCacheLocker.h b/src/system/kernel/vm/PageCacheLocker.h index c3008050bd..36a21b2c7b 100644 --- a/src/system/kernel/vm/PageCacheLocker.h +++ b/src/system/kernel/vm/PageCacheLocker.h @@ -13,18 +13,34 @@ struct vm_page; class PageCacheLocker { public: - PageCacheLocker(vm_page* page, bool dontWait = true); - ~PageCacheLocker(); + inline PageCacheLocker(vm_page* page, + bool dontWait = true); + inline ~PageCacheLocker(); - bool IsLocked() { return fPage != NULL; } + bool IsLocked() { return fPage != NULL; } - bool Lock(vm_page* page, bool dontWait = true); - void Unlock(); + bool Lock(vm_page* page, bool dontWait = true); + void Unlock(); private: - bool _IgnorePage(vm_page* page); + bool _IgnorePage(vm_page* page); - vm_page* fPage; + vm_page* fPage; }; + +PageCacheLocker::PageCacheLocker(vm_page* page, bool dontWait) + : + fPage(NULL) +{ + Lock(page, dontWait); +} + + +PageCacheLocker::~PageCacheLocker() +{ + Unlock(); +} + + #endif // PAGE_CACHE_LOCKER_H diff --git a/src/system/kernel/vm/VMCache.cpp b/src/system/kernel/vm/VMCache.cpp index 9347b8259d..07c92b6c74 100644 --- a/src/system/kernel/vm/VMCache.cpp +++ b/src/system/kernel/vm/VMCache.cpp @@ -743,7 +743,6 @@ VMCache::InsertPage(vm_page* page, off_t offset) page->cache_offset = (page_num_t)(offset >> PAGE_SHIFT); page_count++; - page->usage_count = 2; page->SetCacheRef(fCacheRef); #if KDEBUG @@ -1076,7 +1075,7 @@ VMCache::Resize(off_t newSize, int priority) // remove the page and put it into the free queue DEBUG_PAGE_ACCESS_START(page); - vm_remove_all_page_mappings(page, NULL); + vm_remove_all_page_mappings(page); ASSERT(page->wired_count == 0); // TODO: Find a real solution! Unmapping is probably fine, but // we have no way of unmapping wired pages here. diff --git a/src/system/kernel/vm/VMTranslationMap.cpp b/src/system/kernel/vm/VMTranslationMap.cpp index fce7199832..eb77f37eb0 100644 --- a/src/system/kernel/vm/VMTranslationMap.cpp +++ b/src/system/kernel/vm/VMTranslationMap.cpp @@ -32,7 +32,8 @@ VMTranslationMap::~VMTranslationMap() range and calls UnmapPage(). This is obviously not particularly efficient. */ void -VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size) +VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size, + bool updatePageQueue) { ASSERT(base % B_PAGE_SIZE == 0); ASSERT(size % B_PAGE_SIZE == 0); @@ -40,7 +41,7 @@ VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size) addr_t address = base; addr_t end = address + size; for (; address != end; address += B_PAGE_SIZE) - UnmapPage(area, address); + UnmapPage(area, address, updatePageQueue); } @@ -64,7 +65,7 @@ VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace, addr_t address = area->Base(); addr_t end = address + area->Size(); for (; address != end; address += B_PAGE_SIZE) - UnmapPage(area, address); + UnmapPage(area, address, true); } diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index dec63795f9..67b2bb9d7b 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -472,7 +472,8 @@ get_area_page_protection(VMArea* area, addr_t pageAddress) The page's cache must be locked. */ static status_t -map_page(VMArea* area, vm_page* page, addr_t address, uint32 protection) +map_page(VMArea* area, vm_page* page, addr_t address, uint32 protection, + vm_page_reservation* reservation, bool activatePage) { VMTranslationMap* map = area->address_space->TranslationMap(); @@ -492,7 +493,8 @@ map_page(VMArea* area, vm_page* page, addr_t address, uint32 protection) map->Lock(); - map->Map(address, page->physical_page_number * B_PAGE_SIZE, protection); + map->Map(address, page->physical_page_number * B_PAGE_SIZE, protection, + reservation); // insert mapping into lists if (page->mappings.IsEmpty() && page->wired_count == 0) @@ -506,14 +508,20 @@ map_page(VMArea* area, vm_page* page, addr_t address, uint32 protection) DEBUG_PAGE_ACCESS_CHECK(page); map->Lock(); - map->Map(address, page->physical_page_number * B_PAGE_SIZE, protection); + map->Map(address, page->physical_page_number * B_PAGE_SIZE, protection, + reservation); map->Unlock(); increment_page_wired_count(page); } - if (page->usage_count < 0) - page->usage_count = 1; + if (page->state == PAGE_STATE_CACHED) { + if (page->usage_count == 0 && !activatePage) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else + vm_page_set_state(page, PAGE_STATE_ACTIVE); + } else if (page->state == PAGE_STATE_INACTIVE && activatePage) + vm_page_set_state(page, PAGE_STATE_ACTIVE); return B_OK; } @@ -526,7 +534,7 @@ static inline bool unmap_page(VMArea* area, addr_t virtualAddress) { return area->address_space->TranslationMap()->UnmapPage(area, - virtualAddress); + virtualAddress, true); } @@ -536,7 +544,7 @@ unmap_page(VMArea* area, addr_t virtualAddress) static inline void unmap_pages(VMArea* area, addr_t base, size_t size) { - area->address_space->TranslationMap()->UnmapPages(area, base, size); + area->address_space->TranslationMap()->UnmapPages(area, base, size, true); } @@ -1029,15 +1037,18 @@ vm_create_anonymous_area(team_id team, const char* name, void** address, page_num_t reservedPages = reservedMapPages; if (wiring == B_FULL_LOCK) reservedPages += size / B_PAGE_SIZE; + + vm_page_reservation reservation; if (reservedPages > 0) { if ((flags & CREATE_AREA_DONT_WAIT) != 0) { - if (!vm_page_try_reserve_pages(reservedPages, priority)) { + if (!vm_page_try_reserve_pages(&reservation, reservedPages, + priority)) { reservedPages = 0; status = B_WOULD_BLOCK; goto err0; } } else - vm_page_reserve_pages(reservedPages, priority); + vm_page_reserve_pages(&reservation, reservedPages, priority); } status = locker.SetTo(team); @@ -1122,29 +1133,12 @@ vm_create_anonymous_area(team_id team, const char* name, void** address, # endif continue; #endif - vm_page* page = vm_page_allocate_page( - PAGE_STATE_ACTIVE | pageAllocFlags); -// PAGE_STATE_WIRED | pageAllocFlags); - // TODO: The pages should be PAGE_STATE_WIRED, since there's - // no need for the page daemon to play with them (the same - // should be considered in vm_soft_fault()). ATM doing that - // will result in bad thrashing in systems with little - // memory due to the current tuning of the page daemon. It - // will age pages way too fast (since it just skips - // PAGE_STATE_WIRED pages, while it processes - // PAGE_STATE_ACTIVE with wired_count > 0). + vm_page* page = vm_page_allocate_page(&reservation, + PAGE_STATE_WIRED | pageAllocFlags); cache->InsertPage(page, offset); - map_page(area, page, address, protection); + map_page(area, page, address, protection, &reservation, false); DEBUG_PAGE_ACCESS_END(page); - - // Periodically unreserve pages we've already allocated, so that - // we don't unnecessarily increase the pressure on the VM. - if (offset > 0 && offset % (128 * B_PAGE_SIZE) == 0) { - page_num_t toUnreserve = 128; - vm_page_unreserve_pages(toUnreserve); - reservedPages -= toUnreserve; - } } break; @@ -1211,7 +1205,8 @@ vm_create_anonymous_area(team_id team, const char* name, void** address, if (page == NULL) panic("couldn't lookup physical page just allocated\n"); - status = map->Map(virtualAddress, physicalAddress, protection); + status = map->Map(virtualAddress, physicalAddress, protection, + &reservation); if (status < B_OK) panic("couldn't map physical page in page run\n"); @@ -1232,7 +1227,7 @@ vm_create_anonymous_area(team_id team, const char* name, void** address, cache->Unlock(); if (reservedPages > 0) - vm_page_unreserve_pages(reservedPages); + vm_page_unreserve_pages(&reservation); TRACE(("vm_create_anonymous_area: done\n")); @@ -1255,7 +1250,7 @@ err1: err0: if (reservedPages > 0) - vm_page_unreserve_pages(reservedPages); + vm_page_unreserve_pages(&reservation); if (reservedMemory > 0) vm_unreserve_memory(reservedMemory); @@ -1326,18 +1321,19 @@ vm_map_physical_memory(team_id team, const char* name, void** _address, size_t reservePages = map->MaxPagesNeededToMap(area->Base(), area->Base() + (size - 1)); - vm_page_reserve_pages(reservePages, + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, reservePages, team == VMAddressSpace::KernelID() ? VM_PRIORITY_SYSTEM : VM_PRIORITY_USER); map->Lock(); for (addr_t offset = 0; offset < size; offset += B_PAGE_SIZE) { map->Map(area->Base() + offset, physicalAddress + offset, - protection); + protection, &reservation); } map->Unlock(); - vm_page_unreserve_pages(reservePages); + vm_page_unreserve_pages(&reservation); } if (status < B_OK) @@ -1414,7 +1410,8 @@ vm_map_physical_memory_vecs(team_id team, const char* name, void** _address, size_t reservePages = map->MaxPagesNeededToMap(area->Base(), area->Base() + (size - 1)); - vm_page_reserve_pages(reservePages, + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, reservePages, team == VMAddressSpace::KernelID() ? VM_PRIORITY_SYSTEM : VM_PRIORITY_USER); map->Lock(); @@ -1431,13 +1428,14 @@ vm_map_physical_memory_vecs(team_id team, const char* name, void** _address, break; map->Map(area->Base() + offset, - (addr_t)vecs[vecIndex].iov_base + vecOffset, protection); + (addr_t)vecs[vecIndex].iov_base + vecOffset, protection, + &reservation); vecOffset += B_PAGE_SIZE; } map->Unlock(); - vm_page_unreserve_pages(reservePages); + vm_page_unreserve_pages(&reservation); if (_size != NULL) *_size = size; @@ -1502,7 +1500,8 @@ vm_create_vnode_cache(struct vnode* vnode, struct VMCache** cache) /*! \a cache must be locked. The area's address space must be read-locked. */ static void -pre_map_area_pages(VMArea* area, VMCache* cache) +pre_map_area_pages(VMArea* area, VMCache* cache, + vm_page_reservation* reservation) { addr_t baseAddress = area->Base(); addr_t cacheOffset = area->cache_offset; @@ -1515,14 +1514,14 @@ pre_map_area_pages(VMArea* area, VMCache* cache) if (page->cache_offset >= endPage) break; - // skip inactive pages - if (page->busy || page->usage_count <= 0) + // skip busy and inactive pages + if (page->busy || page->usage_count == 0) continue; DEBUG_PAGE_ACCESS_START(page); map_page(area, page, baseAddress + (page->cache_offset * B_PAGE_SIZE - cacheOffset), - B_READ_AREA | B_KERNEL_READ_AREA); + B_READ_AREA | B_KERNEL_READ_AREA, reservation, false); DEBUG_PAGE_ACCESS_END(page); } } @@ -1584,6 +1583,7 @@ _vm_map_file(team_id team, const char* name, void** _address, // If we're going to pre-map pages, we need to reserve the pages needed by // the mapping backend upfront. page_num_t reservedPreMapPages = 0; + vm_page_reservation reservation; if ((protection & B_READ_AREA) != 0) { AddressSpaceWriteLocker locker; status = locker.SetTo(team); @@ -1595,25 +1595,26 @@ _vm_map_file(team_id team, const char* name, void** _address, locker.Unlock(); - vm_page_reserve_pages(reservedPreMapPages, + vm_page_reserve_pages(&reservation, reservedPreMapPages, team == VMAddressSpace::KernelID() ? VM_PRIORITY_SYSTEM : VM_PRIORITY_USER); } struct PageUnreserver { - PageUnreserver(page_num_t count) - : fCount(count) + PageUnreserver(vm_page_reservation* reservation) + : + fReservation(reservation) { } ~PageUnreserver() { - if (fCount > 0) - vm_page_unreserve_pages(fCount); + if (fReservation != NULL) + vm_page_unreserve_pages(fReservation); } - page_num_t fCount; - } pageUnreserver(reservedPreMapPages); + vm_page_reservation* fReservation; + } pageUnreserver(reservedPreMapPages > 0 ? &reservation : NULL); AddressSpaceWriteLocker locker(team); if (!locker.IsLocked()) @@ -1638,7 +1639,7 @@ _vm_map_file(team_id team, const char* name, void** _address, } if (status == B_OK && (protection & B_READ_AREA) != 0) - pre_map_area_pages(area, cache); + pre_map_area_pages(area, cache, &reservation); cache->Unlock(); @@ -1798,7 +1799,8 @@ vm_clone_area(team_id team, const char* name, void** address, size_t reservePages = map->MaxPagesNeededToMap(newArea->Base(), newArea->Base() + (newArea->Size() - 1)); - vm_page_reserve_pages(reservePages, + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, reservePages, targetAddressSpace == VMAddressSpace::Kernel() ? VM_PRIORITY_SYSTEM : VM_PRIORITY_USER); map->Lock(); @@ -1806,16 +1808,17 @@ vm_clone_area(team_id team, const char* name, void** address, for (addr_t offset = 0; offset < newArea->Size(); offset += B_PAGE_SIZE) { map->Map(newArea->Base() + offset, physicalAddress + offset, - protection); + protection, &reservation); } map->Unlock(); - vm_page_unreserve_pages(reservePages); + vm_page_unreserve_pages(&reservation); } else { VMTranslationMap* map = targetAddressSpace->TranslationMap(); size_t reservePages = map->MaxPagesNeededToMap( newArea->Base(), newArea->Base() + (newArea->Size() - 1)); - vm_page_reserve_pages(reservePages, + vm_page_reservation reservation; + vm_page_reserve_pages(&reservation, reservePages, targetAddressSpace == VMAddressSpace::Kernel() ? VM_PRIORITY_SYSTEM : VM_PRIORITY_USER); @@ -1827,14 +1830,14 @@ vm_clone_area(team_id team, const char* name, void** address, map_page(newArea, page, newArea->Base() + ((page->cache_offset << PAGE_SHIFT) - newArea->cache_offset), - protection); + protection, &reservation, false); DEBUG_PAGE_ACCESS_END(page); } } // TODO: B_FULL_LOCK means that all pages are locked. We are not // ensuring that! - vm_page_unreserve_pages(reservePages); + vm_page_unreserve_pages(&reservation); } } if (status == B_OK) @@ -2206,42 +2209,6 @@ vm_test_map_modification(vm_page* page) } -/*! The page's cache must be locked. -*/ -int32 -vm_test_map_activation(vm_page* page, bool* _modified) -{ - int32 activation = 0; - bool modified = false; - - vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); - vm_page_mapping* mapping; - while ((mapping = iterator.Next()) != NULL) { - VMArea* area = mapping->area; - VMTranslationMap* map = area->address_space->TranslationMap(); - - addr_t physicalAddress; - uint32 flags; - map->Lock(); - map->Query(virtual_page_address(area, page), &physicalAddress, &flags); - map->Unlock(); - - if ((flags & PAGE_ACCESSED) != 0) - activation++; - if ((flags & PAGE_MODIFIED) != 0) - modified = true; - } - - if (_modified != NULL) - *_modified = modified || page->modified; - - if (page->accessed) - activation++; - - return activation; -} - - /*! The page's cache must be locked. */ void @@ -2266,26 +2233,84 @@ vm_clear_map_flags(vm_page* page, uint32 flags) /*! Removes all mappings from a page. - After you've called this function, the page is unmapped from memory. - The accumulated page flags of all mappings can be found in \a _flags. + After you've called this function, the page is unmapped from memory and + the page's \c accessed and \c modified flags have been updated according + to the state of the mappings. The page's cache must be locked. */ void -vm_remove_all_page_mappings(vm_page* page, uint32* _flags) +vm_remove_all_page_mappings(vm_page* page) { while (vm_page_mapping* mapping = page->mappings.Head()) { VMArea* area = mapping->area; VMTranslationMap* map = area->address_space->TranslationMap(); addr_t address = virtual_page_address(area, page); - map->UnmapPage(area, address); + map->UnmapPage(area, address, false); + } +} + + +int32 +vm_clear_page_mapping_accessed_flags(struct vm_page *page) +{ + int32 count = 0; + + vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); + vm_page_mapping* mapping; + while ((mapping = iterator.Next()) != NULL) { + VMArea* area = mapping->area; + VMTranslationMap* map = area->address_space->TranslationMap(); + + bool modified; + if (map->ClearAccessedAndModified(area, + virtual_page_address(area, page), false, modified)) { + count++; + } + + page->modified |= modified; } - if (_flags != NULL) { - *_flags = (page->modified ? PAGE_MODIFIED : 0) - | (page->accessed ? PAGE_ACCESSED : 0); - // TODO: This return value is obviously not particularly useful, as - // the caller could simply check the page's flags. + + if (page->accessed) { + count++; + page->accessed = false; } + + return count; +} + + +/*! Removes all mappings of a page and/or clears the accessed bits of the + mappings. + The function iterates through the page mappings and removes them until + encountering one that has been accessed. From then on it will continue to + iterate, but only clear the accessed flag of the mapping. The page's + \c modified bit will be updated accordingly, the \c accessed bit will be + cleared. + \return The number of mapping accessed bits encountered, including the + \c accessed bit of the page itself. If \c 0 is returned, all mappings + of the page have been removed. +*/ +int32 +vm_remove_all_page_mappings_if_unaccessed(struct vm_page *page) +{ + if (page->accessed) + return vm_clear_page_mapping_accessed_flags(page); + + while (vm_page_mapping* mapping = page->mappings.Head()) { + VMArea* area = mapping->area; + VMTranslationMap* map = area->address_space->TranslationMap(); + addr_t address = virtual_page_address(area, page); + bool modified = false; + if (map->ClearAccessedAndModified(area, address, true, modified)) { + page->accessed = true; + page->modified |= modified; + return vm_clear_page_mapping_accessed_flags(page); + } + page->modified |= modified; + } + + return 0; } @@ -3429,7 +3454,6 @@ status_t vm_init_post_thread(kernel_args* args) { vm_page_init_post_thread(args); - vm_daemon_init(); slab_init_post_thread(); return heap_init_post_thread(); } @@ -3630,6 +3654,7 @@ struct PageFaultContext { VMTranslationMap* map; VMCache* topCache; off_t cacheOffset; + vm_page_reservation reservation; bool isWrite; // return values @@ -3648,6 +3673,7 @@ struct PageFaultContext { ~PageFaultContext() { UnlockAll(); + vm_page_unreserve_pages(&reservation); } void Prepare(VMCache* topCache, off_t cacheOffset) @@ -3718,7 +3744,7 @@ fault_get_page(PageFaultContext& context) // see if the backing store has it if (cache->HasPage(context.cacheOffset)) { // insert a fresh page and mark it busy -- we're going to read it in - page = vm_page_allocate_page( + page = vm_page_allocate_page(&context.reservation, PAGE_STATE_ACTIVE | VM_PAGE_ALLOC_BUSY); cache->InsertPage(page, context.cacheOffset); @@ -3773,7 +3799,8 @@ fault_get_page(PageFaultContext& context) cache = context.isWrite ? context.topCache : lastCache; // allocate a clean page - page = vm_page_allocate_page(PAGE_STATE_ACTIVE | VM_PAGE_ALLOC_CLEAR); + page = vm_page_allocate_page(&context.reservation, + PAGE_STATE_ACTIVE | VM_PAGE_ALLOC_CLEAR); FTRACE(("vm_soft_fault: just allocated page 0x%lx\n", page->physical_page_number)); @@ -3787,7 +3814,7 @@ fault_get_page(PageFaultContext& context) // TODO: If memory is low, it might be a good idea to steal the page // from our source cache -- if possible, that is. FTRACE(("get new page, copy it, and put it into the topmost cache\n")); - page = vm_page_allocate_page(PAGE_STATE_ACTIVE); + page = vm_page_allocate_page(&context.reservation, PAGE_STATE_ACTIVE); // To not needlessly kill concurrency we unlock all caches but the top // one while copying the page. Lacking another mechanism to ensure that @@ -3832,7 +3859,7 @@ vm_soft_fault(VMAddressSpace* addressSpace, addr_t originalAddress, size_t reservePages = 2 + context.map->MaxPagesNeededToMap(originalAddress, originalAddress); context.addressSpaceLocker.Unlock(); - vm_page_reserve_pages(reservePages, + vm_page_reserve_pages(&context.reservation, reservePages, addressSpace == VMAddressSpace::Kernel() ? VM_PRIORITY_SYSTEM : VM_PRIORITY_USER); @@ -3960,7 +3987,8 @@ vm_soft_fault(VMAddressSpace* addressSpace, addr_t originalAddress, } if (mapPage) { - if (map_page(area, context.page, address, newProtection) != B_OK) { + if (map_page(area, context.page, address, newProtection, + &context.reservation, true) != B_OK) { // Mapping can only fail, when the page mapping object couldn't // be allocated. Save for the missing mapping everything is // fine, though. We'll simply leave and probably fault again. @@ -3979,15 +4007,14 @@ vm_soft_fault(VMAddressSpace* addressSpace, addr_t originalAddress, break; } - } + } else if (context.page->state == PAGE_STATE_INACTIVE) + vm_page_set_state(context.page, PAGE_STATE_ACTIVE); DEBUG_PAGE_ACCESS_END(context.page); break; } - vm_page_unreserve_pages(reservePages); - return status; } @@ -5335,10 +5362,8 @@ _user_set_memory_protection(void* _address, size_t size, int protection) bool unmapPage = page->Cache() != topCache && (protection & B_WRITE_AREA) != 0; - if (!unmapPage) { - map->Unmap(pageAddress, pageAddress + B_PAGE_SIZE - 1); - map->Map(pageAddress, physicalAddress, actualProtection); - } + if (!unmapPage) + map->ProtectPage(area, pageAddress, actualProtection); map->Unlock(); diff --git a/src/system/kernel/vm/vm_daemons.cpp b/src/system/kernel/vm/vm_daemons.cpp deleted file mode 100644 index 3a0a9b9828..0000000000 --- a/src/system/kernel/vm/vm_daemons.cpp +++ /dev/null @@ -1,473 +0,0 @@ -/* - * Copyright 2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved. - * Distributed under the terms of the MIT License. - */ - - -#include "PageCacheLocker.h" - -#include - -#include - -#include - -#include -#include -#include -#include -#include - -#include "VMAnonymousCache.h" - - -//#define TRACK_PAGE_USAGE_STATS 1 - -//#define TRACE_VM_DAEMONS -#ifdef TRACE_VM_DAEMONS -#define TRACE(x) dprintf x -#else -#define TRACE(x) ; -#endif - -const static uint32 kMinScanPagesCount = 512; -const static uint32 kMaxScanPagesCount = 8192; -const static bigtime_t kMinScanWaitInterval = 50000LL; // 50 ms -const static bigtime_t kMaxScanWaitInterval = 1000000LL; // 1 sec - -static uint32 sLowPagesCount; -static sem_id sPageDaemonSem; -static uint32 sNumPages; - -#ifdef TRACK_PAGE_USAGE_STATS -static page_num_t sPageUsageArrays[512]; -static page_num_t* sPageUsage = sPageUsageArrays; -static page_num_t sPageUsagePageCount; -static page_num_t* sNextPageUsage = sPageUsageArrays + 256; -static page_num_t sNextPageUsagePageCount; -#endif - - -PageCacheLocker::PageCacheLocker(vm_page* page, bool dontWait) - : - fPage(NULL) -{ - Lock(page, dontWait); -} - - -PageCacheLocker::~PageCacheLocker() -{ - Unlock(); -} - - -bool -PageCacheLocker::_IgnorePage(vm_page* page) -{ - if (page->busy || page->state == PAGE_STATE_WIRED - || page->state == PAGE_STATE_FREE || page->state == PAGE_STATE_CLEAR - || page->state == PAGE_STATE_UNUSED || page->wired_count > 0) - return true; - - return false; -} - - -bool -PageCacheLocker::Lock(vm_page* page, bool dontWait) -{ - if (_IgnorePage(page)) - return false; - - // Grab a reference to this cache. - VMCache* cache = vm_cache_acquire_locked_page_cache(page, dontWait); - if (cache == NULL) - return false; - - if (_IgnorePage(page)) { - cache->ReleaseRefAndUnlock(); - return false; - } - - fPage = page; - return true; -} - - -void -PageCacheLocker::Unlock() -{ - if (fPage == NULL) - return; - - fPage->Cache()->ReleaseRefAndUnlock(); - - fPage = NULL; -} - - -// #pragma mark - - - -#if PAGE_DAEMON_TRACING - -namespace PageDaemonTracing { - -class ActivatePage : public AbstractTraceEntry { - public: - ActivatePage(vm_page* page) - : - fCache(page->cache), - fPage(page) - { - Initialized(); - } - - virtual void AddDump(TraceOutput& out) - { - out.Print("page activated: %p, cache: %p", fPage, fCache); - } - - private: - VMCache* fCache; - vm_page* fPage; -}; - - -class DeactivatePage : public AbstractTraceEntry { - public: - DeactivatePage(vm_page* page) - : - fCache(page->cache), - fPage(page) - { - Initialized(); - } - - virtual void AddDump(TraceOutput& out) - { - out.Print("page deactivated: %p, cache: %p", fPage, fCache); - } - - private: - VMCache* fCache; - vm_page* fPage; -}; - - -class FreedPageSwap : public AbstractTraceEntry { - public: - FreedPageSwap(vm_page* page) - : - fCache(page->cache), - fPage(page) - { - Initialized(); - } - - virtual void AddDump(TraceOutput& out) - { - out.Print("page swap freed: %p, cache: %p", fPage, fCache); - } - - private: - VMCache* fCache; - vm_page* fPage; -}; - -} // namespace PageDaemonTracing - -# define T(x) new(std::nothrow) PageDaemonTracing::x - -#else -# define T(x) -#endif // PAGE_DAEMON_TRACING - - -// #pragma mark - - - -#ifdef TRACK_PAGE_USAGE_STATS - -static void -track_page_usage(vm_page* page) -{ - if (page->wired_count == 0) { - sNextPageUsage[(int32)page->usage_count + 128]++; - sNextPageUsagePageCount++; - } -} - - -static void -update_page_usage_stats() -{ - std::swap(sPageUsage, sNextPageUsage); - sPageUsagePageCount = sNextPageUsagePageCount; - - memset(sNextPageUsage, 0, sizeof(page_num_t) * 256); - sNextPageUsagePageCount = 0; - - // compute average - if (sPageUsagePageCount > 0) { - int64 sum = 0; - for (int32 i = 0; i < 256; i++) - sum += (int64)sPageUsage[i] * (i - 128); - - TRACE(("average page usage: %f (%lu pages)\n", - (float)sum / sPageUsagePageCount, sPageUsagePageCount)); - } -} - - -static int -dump_page_usage_stats(int argc, char** argv) -{ - kprintf("distribution of page usage counts (%lu pages):", - sPageUsagePageCount); - - int64 sum = 0; - for (int32 i = 0; i < 256; i++) { - if (i % 8 == 0) - kprintf("\n%4ld:", i - 128); - - int64 count = sPageUsage[i]; - sum += count * (i - 128); - - kprintf(" %9llu", count); - } - - kprintf("\n\n"); - - kprintf("average usage count: %f\n", - sPageUsagePageCount > 0 ? (float)sum / sPageUsagePageCount : 0); - - return 0; -} - -#endif - - -static void -clear_page_activation(int32 index) -{ - vm_page *page = vm_page_at_index(index); - PageCacheLocker locker(page); - if (!locker.IsLocked()) - return; - - if (!page->busy && page->state == PAGE_STATE_ACTIVE) - vm_clear_map_flags(page, PAGE_ACCESSED); -} - - -static bool -check_page_activation(int32 index) -{ - vm_page *page = vm_page_at_index(index); - PageCacheLocker locker(page); - if (!locker.IsLocked()) - return false; - - DEBUG_PAGE_ACCESS_START(page); - - bool modified; - int32 activation = vm_test_map_activation(page, &modified); - if (modified && page->state != PAGE_STATE_MODIFIED) { - TRACE(("page %p -> move to modified\n", page); - vm_page_set_state(page, PAGE_STATE_MODIFIED)); - } - - if (activation > 0) { - // page is still in active use - if (page->usage_count < 0) { - if (page->state != PAGE_STATE_MODIFIED) - vm_page_set_state(page, PAGE_STATE_ACTIVE); - page->usage_count = 1; - TRACE(("page %p -> move to active\n", page)); - T(ActivatePage(page)); - } else if (page->usage_count < 127) - page->usage_count++; - -#ifdef TRACK_PAGE_USAGE_STATS - track_page_usage(page); -#endif - - DEBUG_PAGE_ACCESS_END(page); - return false; - } - - if (page->usage_count > -128) - page->usage_count--; - -#ifdef TRACK_PAGE_USAGE_STATS - track_page_usage(page); -#endif - - if (page->usage_count < 0) { - uint32 flags; - vm_remove_all_page_mappings(page, &flags); - - // recheck eventual last minute changes - if ((flags & PAGE_MODIFIED) != 0 && page->state != PAGE_STATE_MODIFIED) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - if ((flags & PAGE_ACCESSED) != 0 && ++page->usage_count >= 0) { - DEBUG_PAGE_ACCESS_END(page); - return false; - } - - if (page->state == PAGE_STATE_MODIFIED) - vm_page_schedule_write_page(page); - else - vm_page_set_state(page, PAGE_STATE_INACTIVE); - TRACE(("page %p -> move to inactive\n", page)); - T(DeactivatePage(page)); - } - - DEBUG_PAGE_ACCESS_END(page); - return true; -} - - -#if ENABLE_SWAP_SUPPORT -static bool -free_page_swap_space(int32 index) -{ - vm_page *page = vm_page_at_index(index); - PageCacheLocker locker(page); - if (!locker.IsLocked()) - return false; - - DEBUG_PAGE_ACCESS_START(page); - - VMCache* cache = page->Cache(); - if (cache->temporary && page->wired_count == 0 - && cache->HasPage(page->cache_offset << PAGE_SHIFT) - && page->usage_count > 0) { - // TODO: how to judge a page is highly active? - if (swap_free_page_swap_space(page)) { - // We need to mark the page modified, since otherwise it could be - // stolen and we'd lose its data. - vm_page_set_state(page, PAGE_STATE_MODIFIED); - T(FreedPageSwap(page)); - DEBUG_PAGE_ACCESS_END(page); - return true; - } - } - DEBUG_PAGE_ACCESS_END(page); - return false; -} -#endif - - -static status_t -page_daemon(void* /*unused*/) -{ - bigtime_t scanWaitInterval = kMaxScanWaitInterval; - uint32 scanPagesCount = kMinScanPagesCount; - uint32 clearPage = 0; - uint32 checkPage = sNumPages / 2; - -#if ENABLE_SWAP_SUPPORT - uint32 swapPage = 0; -#endif - - while (true) { - acquire_sem_etc(sPageDaemonSem, 1, B_RELATIVE_TIMEOUT, - scanWaitInterval); - - // Compute next run time - uint32 pagesLeft = vm_page_num_free_pages(); - if (pagesLeft > sLowPagesCount) { - // don't do anything if we have enough free memory left - continue; - } - - scanWaitInterval = kMinScanWaitInterval - + (kMaxScanWaitInterval - kMinScanWaitInterval) - * pagesLeft / sLowPagesCount; - scanPagesCount = kMaxScanPagesCount - - (kMaxScanPagesCount - kMinScanPagesCount) - * pagesLeft / sLowPagesCount; - uint32 leftToFree = sLowPagesCount - pagesLeft; - TRACE(("wait interval %Ld, scan pages %lu, free %lu, " - "target %lu\n", scanWaitInterval, scanPagesCount, - pagesLeft, leftToFree)); - - for (uint32 i = 0; i < scanPagesCount && leftToFree > 0; i++) { - if (clearPage == 0) - TRACE(("clear through\n")); - if (checkPage == 0) { - TRACE(("check through\n")); -#ifdef TRACK_PAGE_USAGE_STATS - update_page_usage_stats(); -#endif - } - clear_page_activation(clearPage); - - if (check_page_activation(checkPage)) - leftToFree--; - - if (++clearPage == sNumPages) - clearPage = 0; - if (++checkPage == sNumPages) - checkPage = 0; - } - -#if ENABLE_SWAP_SUPPORT - uint32 availableSwapPages = swap_available_pages(); - if (swap_total_swap_pages() > 0 && availableSwapPages < sNumPages / 8) { - uint32 swapPagesToFree = sNumPages / 8 - availableSwapPages; - uint32 swapScanCount = kMaxScanPagesCount - - (kMaxScanPagesCount - kMinScanPagesCount) - * availableSwapPages / (sNumPages / 8); - - for (uint32 i = 0; i < swapScanCount && swapPagesToFree > 0; i++) { - if (free_page_swap_space(swapPage)) - swapPagesToFree--; - if (++swapPage == sNumPages) - swapPage = 0; - } - } -#endif - } - - return B_OK; -} - - -void -vm_schedule_page_scanner(uint32 target) -{ - release_sem(sPageDaemonSem); -} - - -status_t -vm_daemon_init() -{ -#ifdef TRACK_PAGE_USAGE_STATS - add_debugger_command_etc("page_usage", &dump_page_usage_stats, - "Dumps statistics about page usage counts", - "\n" - "Dumps statistics about page usage counts.\n", - B_KDEBUG_DONT_PARSE_ARGUMENTS); -#endif - - sPageDaemonSem = create_sem(0, "page daemon"); - - sNumPages = vm_page_num_pages(); - - sLowPagesCount = sNumPages / 16; - if (sLowPagesCount < 1024) - sLowPagesCount = 1024; - - // create a kernel thread to select pages for pageout - thread_id thread = spawn_kernel_thread(&page_daemon, "page daemon", - B_NORMAL_PRIORITY, NULL); - send_signal_etc(thread, SIGCONT, B_DO_NOT_RESCHEDULE); - - return B_OK; -} - diff --git a/src/system/kernel/vm/vm_page.cpp b/src/system/kernel/vm/vm_page.cpp index 8b0092f062..0b86655b63 100644 --- a/src/system/kernel/vm/vm_page.cpp +++ b/src/system/kernel/vm/vm_page.cpp @@ -51,6 +51,15 @@ # define TRACE(x) ; #endif +//#define TRACE_VM_DAEMONS +#ifdef TRACE_VM_DAEMONS +#define TRACE_DAEMON(x...) dprintf(x) +#else +#define TRACE_DAEMON(x...) do {} while (false) +#endif + +//#define TRACK_PAGE_USAGE_STATS 1 + #define SCRUB_SIZE 16 // this many pages will be cleared at once in the page scrubber thread @@ -68,8 +77,25 @@ static const size_t kPageReserveForPriority[] = { 0 // VIP }; -static const uint32 kMinimumSystemReserve = VM_PAGE_RESERVE_USER; +// Minimum number of free pages the page daemon will try to achieve. +static uint32 sFreePagesTarget; +static uint32 sFreeOrCachedPagesTarget; +static uint32 sInactivePagesTarget; +// Wait interval between page daemon runs. +static const bigtime_t kIdleScanWaitInterval = 1000000LL; // 1 sec +static const bigtime_t kBusyScanWaitInterval = 500000LL; // 0.5 sec + +// Number of idle runs after which we want to have processed the full active +// queue. +static const uint32 kIdleRunsForFullQueue = 20; + +// Maximum limit for the vm_page::usage_count. +static const int32 kPageUsageMax = 64; +// vm_page::usage_count buff an accessed page receives in a scan. +static const int32 kPageUsageAdvance = 3; +// vm_page::usage_count debuff an unaccessed page receives in a scan. +static const int32 kPageUsageDecline = 1; int32 gMappedPagesCount; @@ -86,8 +112,7 @@ static vm_page *sPages; static addr_t sPhysicalPageOffset; static size_t sNumPages; static vint32 sUnreservedFreePages; -static vint32 sSystemReservedPages; -static vint32 sPageDeficit; +static vint32 sUnsatisfiedPageReservations; static vint32 sModifiedTemporaryPages; static ConditionVariable sFreePageCondition; @@ -96,7 +121,95 @@ static mutex sPageDeficitLock = MUTEX_INITIALIZER("page deficit"); static rw_lock sFreePageQueuesLock = RW_LOCK_INITIALIZER("free/clear page queues"); -static sem_id sWriterWaitSem; +#ifdef TRACK_PAGE_USAGE_STATS +static page_num_t sPageUsageArrays[512]; +static page_num_t* sPageUsage = sPageUsageArrays; +static page_num_t sPageUsagePageCount; +static page_num_t* sNextPageUsage = sPageUsageArrays + 256; +static page_num_t sNextPageUsagePageCount; +#endif + + +struct PageReservationWaiter + : public DoublyLinkedListLinkImpl { + struct thread* thread; + uint32 dontTouch; // reserve not to touch + uint32 missing; // pages missing for the reservation + int32 threadPriority; + + bool operator<(const PageReservationWaiter& other) const + { + // Implies an order by descending VM priority (ascending dontTouch) + // and (secondarily) descending thread priority. + if (dontTouch != other.dontTouch) + return dontTouch < other.dontTouch; + return threadPriority > other.threadPriority; + } +}; + +typedef DoublyLinkedList PageReservationWaiterList; +static PageReservationWaiterList sPageReservationWaiters; + + +struct DaemonCondition { + void Init(const char* name) + { + mutex_init(&fLock, "daemon condition"); + fCondition.Init(this, name); + fActivated = false; + } + + bool Lock() + { + return mutex_lock(&fLock) == B_OK; + } + + void Unlock() + { + mutex_unlock(&fLock); + } + + bool Wait(bigtime_t timeout, bool clearActivated) + { + MutexLocker locker(fLock); + if (clearActivated) + fActivated = false; + else if (fActivated) + return true; + + ConditionVariableEntry entry; + fCondition.Add(&entry); + + locker.Unlock(); + + return entry.Wait(B_RELATIVE_TIMEOUT, timeout) == B_OK; + } + + void WakeUp() + { + if (fActivated) + return; + + MutexLocker locker(fLock); + fActivated = true; + fCondition.NotifyOne(); + } + + void ClearActivated() + { + MutexLocker locker(fLock); + fActivated = false; + } + +private: + mutex fLock; + ConditionVariable fCondition; + bool fActivated; +}; + + +static DaemonCondition sPageWriterCondition; +static DaemonCondition sPageDaemonCondition; #if PAGE_ALLOCATION_TRACING @@ -241,13 +354,88 @@ class StolenPage : public AbstractTraceEntry { } // namespace PageAllocationTracing -# define T(x) new(std::nothrow) PageAllocationTracing::x +# define TA(x) new(std::nothrow) PageAllocationTracing::x #else -# define T(x) +# define TA(x) #endif // PAGE_ALLOCATION_TRACING +#if PAGE_DAEMON_TRACING + +namespace PageDaemonTracing { + +class ActivatePage : public AbstractTraceEntry { + public: + ActivatePage(vm_page* page) + : + fCache(page->cache), + fPage(page) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("page activated: %p, cache: %p", fPage, fCache); + } + + private: + VMCache* fCache; + vm_page* fPage; +}; + + +class DeactivatePage : public AbstractTraceEntry { + public: + DeactivatePage(vm_page* page) + : + fCache(page->cache), + fPage(page) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("page deactivated: %p, cache: %p", fPage, fCache); + } + + private: + VMCache* fCache; + vm_page* fPage; +}; + + +class FreedPageSwap : public AbstractTraceEntry { + public: + FreedPageSwap(vm_page* page) + : + fCache(page->cache), + fPage(page) + { + Initialized(); + } + + virtual void AddDump(TraceOutput& out) + { + out.Print("page swap freed: %p, cache: %p", fPage, fCache); + } + + private: + VMCache* fCache; + vm_page* fPage; +}; + +} // namespace PageDaemonTracing + +# define TD(x) new(std::nothrow) PageDaemonTracing::x + +#else +# define TD(x) +#endif // PAGE_DAEMON_TRACING + + #if PAGE_WRITER_TRACING namespace PageWriterTracing { @@ -569,7 +757,7 @@ dump_page_stats(int argc, char **argv) if (sPages[i].state == PAGE_STATE_MODIFIED && sPages[i].Cache() != NULL && sPages[i].Cache()->temporary && sPages[i].wired_count == 0) { swappableModified++; - if (sPages[i].usage_count < 0) + if (sPages[i].usage_count == 0) swappableModifiedInactive++; } } @@ -593,10 +781,19 @@ dump_page_stats(int argc, char **argv) kprintf("clear: %" B_PRIuSIZE "\n", counter[PAGE_STATE_CLEAR]); kprintf("unreserved free pages: %" B_PRId32 "\n", sUnreservedFreePages); - kprintf("system reserved pages: %" B_PRId32 "\n", sSystemReservedPages); - kprintf("page deficit: %lu\n", sPageDeficit); + kprintf("unsatisfied page reservations: %" B_PRId32 "\n", + sUnsatisfiedPageReservations); kprintf("mapped pages: %lu\n", gMappedPagesCount); + kprintf("waiting threads:\n"); + for (PageReservationWaiterList::Iterator it + = sPageReservationWaiters.GetIterator(); + PageReservationWaiter* waiter = it.Next();) { + kprintf(" %6" B_PRId32 ": missing: %6" B_PRIu32 + ", don't touch: %6" B_PRIu32 "\n", waiter->thread->id, + waiter->missing, waiter->dontTouch); + } + kprintf("\nfree queue: %p, count = %ld\n", &sFreePageQueue, sFreePageQueue.Count()); kprintf("clear queue: %p, count = %ld\n", &sClearPageQueue, @@ -614,32 +811,150 @@ dump_page_stats(int argc, char **argv) } -// #pragma mark - +#ifdef TRACK_PAGE_USAGE_STATS + +static void +track_page_usage(vm_page* page) +{ + if (page->wired_count == 0) { + sNextPageUsage[(int32)page->usage_count + 128]++; + sNextPageUsagePageCount++; + } +} static void -unreserve_page() +update_page_usage_stats() { - int32 systemReserve = sSystemReservedPages; - if (systemReserve >= (int32)kMinimumSystemReserve) { - atomic_add(&sUnreservedFreePages, 1); - } else { - // Note: Due to the race condition, we might increment - // sSystemReservedPages beyond its desired count. That doesn't matter - // all that much, though, since its only about a single page and - // vm_page_reserve_pages() will correct this when the general reserve - // is running low. - atomic_add(&sSystemReservedPages, 1); + std::swap(sPageUsage, sNextPageUsage); + sPageUsagePageCount = sNextPageUsagePageCount; + + memset(sNextPageUsage, 0, sizeof(page_num_t) * 256); + sNextPageUsagePageCount = 0; + + // compute average + if (sPageUsagePageCount > 0) { + int64 sum = 0; + for (int32 i = 0; i < 256; i++) + sum += (int64)sPageUsage[i] * (i - 128); + + TRACE_DAEMON("average page usage: %f (%lu pages)\n", + (float)sum / sPageUsagePageCount, sPageUsagePageCount); + } +} + + +static int +dump_page_usage_stats(int argc, char** argv) +{ + kprintf("distribution of page usage counts (%lu pages):", + sPageUsagePageCount); + + int64 sum = 0; + for (int32 i = 0; i < 256; i++) { + if (i % 8 == 0) + kprintf("\n%4ld:", i - 128); + + int64 count = sPageUsage[i]; + sum += count * (i - 128); + + kprintf(" %9llu", count); } - if (sPageDeficit > 0) { - MutexLocker pageDeficitLocker(sPageDeficitLock); - if (sPageDeficit > 0) - sFreePageCondition.NotifyAll(); + kprintf("\n\n"); + + kprintf("average usage count: %f\n", + sPageUsagePageCount > 0 ? (float)sum / sPageUsagePageCount : 0); + + return 0; +} + +#endif // TRACK_PAGE_USAGE_STATS + + +// #pragma mark - + + +struct page_stats { + int32 totalFreePages; + int32 unsatisfiedReservations; + int32 cachedPages; +}; + +static void +get_page_stats(page_stats& pageStats) +{ + pageStats.totalFreePages = sUnreservedFreePages; + pageStats.cachedPages = sCachedPageQueue.Count(); + pageStats.unsatisfiedReservations = sUnsatisfiedPageReservations; + // TODO: We don't get an actual snapshot here! +} + + +/*! Reserves as many pages as possible from \c sUnreservedFreePages up to + \a count. Doesn't touch the last \a dontTouch pages of + \c sUnreservedFreePages, though. + \return The number of actually reserved pages. +*/ +static uint32 +reserve_some_pages(uint32 count, uint32 dontTouch) +{ + while (true) { + int32 freePages = sUnreservedFreePages; + if (freePages <= (int32)dontTouch) + return 0; + + int32 toReserve = std::min(count, freePages - dontTouch); + if (atomic_test_and_set(&sUnreservedFreePages, + freePages - toReserve, freePages) + == freePages) { + return toReserve; + } + + // the count changed in the meantime -- retry } } +static void +wake_up_page_reservation_waiters() +{ + MutexLocker pageDeficitLocker(sPageDeficitLock); + + // TODO: If this is a low priority thread, we might want to disable + // interrupts or otherwise ensure that we aren't unscheduled. Otherwise + // high priority threads wait be kept waiting while a medium priority thread + // prevents us from running. + + while (PageReservationWaiter* waiter = sPageReservationWaiters.Head()) { + int32 reserved = reserve_some_pages(waiter->missing, + waiter->dontTouch); + if (reserved == 0) + return; + + atomic_add(&sUnsatisfiedPageReservations, -reserved); + waiter->missing -= reserved; + + if (waiter->missing > 0) + return; + + sPageReservationWaiters.Remove(waiter); + + InterruptsSpinLocker threadLocker(gThreadSpinlock); + thread_unblock_locked(waiter->thread, B_OK); + } +} + + +static inline void +unreserve_pages(uint32 count) +{ + atomic_add(&sUnreservedFreePages, count); + if (sUnsatisfiedPageReservations != 0) + wake_up_page_reservation_waiters(); +} + + static void free_page(vm_page* page, bool clear) { @@ -679,16 +994,10 @@ free_page(vm_page* page, bool clear) if (!page->mappings.IsEmpty() || page->wired_count > 0) panic("to be freed page %p has mappings", page); - if (sPageDeficit > 0) { - MutexLocker pageDeficitLocker(sPageDeficitLock); - if (sPageDeficit > 0) - sFreePageCondition.NotifyOne(); - } - if (fromQueue != NULL) fromQueue->RemoveUnlocked(page); - T(FreePage()); + TA(FreePage()); ReadLocker locker(sFreePageQueuesLock); @@ -704,7 +1013,7 @@ free_page(vm_page* page, bool clear) locker.Unlock(); - unreserve_page(); + unreserve_pages(1); } @@ -762,6 +1071,7 @@ set_page_state(vm_page *page, int pageState) toQueue = &sModifiedPageQueue; break; case PAGE_STATE_CACHED: + ASSERT(page->wired_count == 0 && page->mappings.IsEmpty()); toQueue = &sCachedPageQueue; break; case PAGE_STATE_FREE: @@ -777,12 +1087,6 @@ set_page_state(vm_page *page, int pageState) return; } - if (pageState == PAGE_STATE_INACTIVE && sPageDeficit > 0) { - MutexLocker pageDeficitLocker(sPageDeficitLock); - if (sPageDeficit > 0) - sFreePageCondition.NotifyOne(); - } - VMCache* cache = page->Cache(); if (cache != NULL && cache->temporary) { if (pageState == PAGE_STATE_MODIFIED) @@ -825,18 +1129,18 @@ move_page_to_active_or_inactive_queue(vm_page *page, bool dequeued) // Note, this logic must be in sync with what the page daemon does int32 state; - if (!page->mappings.IsEmpty() || page->usage_count >= 0 - || page->wired_count) + if (page->mappings.IsEmpty() && page->wired_count == 0) + state = PAGE_STATE_CACHED; + else if (page->usage_count > 0) state = PAGE_STATE_ACTIVE; else state = PAGE_STATE_INACTIVE; -// TODO: Cached queue! +// TODO: If free + cached pages are low, we might directly want to free the +// page. if (dequeued) { page->state = state; - VMPageQueue& queue = state == PAGE_STATE_ACTIVE - ? sActivePageQueue : sInactivePageQueue; - queue.AppendUnlocked(page); + sPageQueues[state].AppendUnlocked(page); if (page->Cache()->temporary) atomic_add(&sModifiedTemporaryPages, -1); } else @@ -868,15 +1172,19 @@ page_scrubber(void *unused) for (;;) { snooze(100000); // 100ms - if (sFreePageQueue.Count() == 0) + if (sFreePageQueue.Count() == 0 + || sUnreservedFreePages < (int32)sFreePagesTarget) { continue; + } // Since we temporarily remove pages from the free pages reserve, // we must make sure we don't cause a violation of the page // reservation warranty. The following is usually stricter than // necessary, because we don't have information on how many of the // reserved pages have already been allocated. - if (!vm_page_try_reserve_pages(SCRUB_SIZE, VM_PRIORITY_USER)) + int32 reserved = reserve_some_pages(SCRUB_SIZE, + kPageReserveForPriority[VM_PRIORITY_USER]); + if (reserved == 0) continue; // get some pages from the free queue @@ -884,7 +1192,7 @@ page_scrubber(void *unused) vm_page *page[SCRUB_SIZE]; int32 scrubCount = 0; - for (int32 i = 0; i < SCRUB_SIZE; i++) { + for (int32 i = 0; i < reserved; i++) { page[i] = sFreePageQueue.RemoveHeadUnlocked(); if (page[i] == NULL) break; @@ -899,11 +1207,11 @@ page_scrubber(void *unused) locker.Unlock(); if (scrubCount == 0) { - vm_page_unreserve_pages(SCRUB_SIZE); + unreserve_pages(reserved); continue; } - T(ScrubbingPages(scrubCount)); + TA(ScrubbingPages(scrubCount)); // clear them for (int32 i = 0; i < scrubCount; i++) @@ -921,44 +1229,37 @@ page_scrubber(void *unused) locker.Unlock(); - vm_page_unreserve_pages(SCRUB_SIZE); + unreserve_pages(reserved); - T(ScrubbedPages(scrubCount)); + TA(ScrubbedPages(scrubCount)); } return 0; } +static void +init_page_marker(vm_page &marker) +{ + marker.SetCacheRef(NULL); + marker.state = PAGE_STATE_UNUSED; + marker.busy = true; +#if DEBUG_PAGE_QUEUE + marker.queue = NULL; +#endif +#if DEBUG_PAGE_ACCESS + marker.accessing_thread = thread_get_current_thread_id(); +#endif +} + + static void remove_page_marker(struct vm_page &marker) { - if (marker.state == PAGE_STATE_UNUSED) - return; - DEBUG_PAGE_ACCESS_CHECK(&marker); - VMPageQueue *queue; - - switch (marker.state) { - case PAGE_STATE_ACTIVE: - queue = &sActivePageQueue; - break; - case PAGE_STATE_INACTIVE: - queue = &sInactivePageQueue; - break; - case PAGE_STATE_MODIFIED: - queue = &sModifiedPageQueue; - break; - case PAGE_STATE_CACHED: - queue = &sCachedPageQueue; - break; - - default: - return; - } - - queue->RemoveUnlocked(&marker); + if (marker.state < PAGE_STATE_FIRST_UNQUEUED) + sPageQueues[marker.state].RemoveUnlocked(&marker); marker.state = PAGE_STATE_UNUSED; } @@ -1129,7 +1430,7 @@ PageWriteWrapper::CheckRemoveFromShrunkenCache() if (fPage->busy_writing) return; - vm_remove_all_page_mappings(fPage, NULL); + vm_remove_all_page_mappings(fPage); fCache->RemovePage(fPage); } @@ -1426,31 +1727,17 @@ page_writer(void* /*unused*/) } vm_page marker; - marker.SetCacheRef(NULL); - marker.state = PAGE_STATE_UNUSED; - marker.busy = true; -#if DEBUG_PAGE_QUEUE - marker.queue = NULL; -#endif -#if DEBUG_PAGE_ACCESS - marker.accessing_thread = thread_get_current_thread_id(); -#endif + init_page_marker(marker); while (true) { - if ((int32)sModifiedPageQueue.Count() - sModifiedTemporaryPages - < 1024) { - int32 count = 0; - get_sem_count(sWriterWaitSem, &count); - if (count == 0) - count = 1; - - acquire_sem_etc(sWriterWaitSem, count, B_RELATIVE_TIMEOUT, 3000000); +// TODO: Maybe wait shorter when memory is low! + if (sModifiedPageQueue.Count() < kNumPages) { + sPageWriterCondition.Wait(3000000, true); // all 3 seconds when no one triggers us } - int32 modifiedPages = sModifiedPageQueue.Count() - - sModifiedTemporaryPages; - if (modifiedPages <= 0) + int32 modifiedPages = sModifiedPageQueue.Count(); + if (modifiedPages == 0) continue; // depending on how urgent it becomes to get pages to disk, we adjust @@ -1567,38 +1854,75 @@ page_writer(void* /*unused*/) } +// #pragma mark - + + +// TODO: This should be done in the page daemon! +#if 0 +#if ENABLE_SWAP_SUPPORT +static bool +free_page_swap_space(int32 index) +{ + vm_page *page = vm_page_at_index(index); + PageCacheLocker locker(page); + if (!locker.IsLocked()) + return false; + + DEBUG_PAGE_ACCESS_START(page); + + VMCache* cache = page->Cache(); + if (cache->temporary && page->wired_count == 0 + && cache->HasPage(page->cache_offset << PAGE_SHIFT) + && page->usage_count > 0) { + // TODO: how to judge a page is highly active? + if (swap_free_page_swap_space(page)) { + // We need to mark the page modified, since otherwise it could be + // stolen and we'd lose its data. + vm_page_set_state(page, PAGE_STATE_MODIFIED); + TD(FreedPageSwap(page)); + DEBUG_PAGE_ACCESS_END(page); + return true; + } + } + DEBUG_PAGE_ACCESS_END(page); + return false; +} +#endif +#endif // 0 + + static vm_page * -find_page_candidate(struct vm_page &marker) +find_cached_page_candidate(struct vm_page &marker) { DEBUG_PAGE_ACCESS_CHECK(&marker); - InterruptsSpinLocker locker(sInactivePageQueue.GetLock()); + InterruptsSpinLocker locker(sCachedPageQueue.GetLock()); vm_page *page; if (marker.state == PAGE_STATE_UNUSED) { // Get the first free pages of the (in)active queue - page = sInactivePageQueue.Head(); + page = sCachedPageQueue.Head(); } else { // Get the next page of the current queue - if (marker.state != PAGE_STATE_INACTIVE) { + if (marker.state != PAGE_STATE_CACHED) { panic("invalid marker %p state", &marker); return NULL; } - page = sInactivePageQueue.Next(&marker); - sInactivePageQueue.Remove(&marker); + page = sCachedPageQueue.Next(&marker); + sCachedPageQueue.Remove(&marker); marker.state = PAGE_STATE_UNUSED; } while (page != NULL) { if (!page->busy) { // we found a candidate, insert marker - marker.state = PAGE_STATE_INACTIVE; - sInactivePageQueue.InsertAfter(page, &marker); + marker.state = PAGE_STATE_CACHED; + sCachedPageQueue.InsertAfter(page, &marker); return page; } - page = sInactivePageQueue.Next(page); + page = sCachedPageQueue.Next(page); } return NULL; @@ -1606,10 +1930,10 @@ find_page_candidate(struct vm_page &marker) static bool -steal_page(vm_page *page) +free_cached_page(vm_page *page, bool dontWait) { // try to lock the page's cache - if (vm_cache_acquire_locked_page_cache(page, false) == NULL) + if (vm_cache_acquire_locked_page_cache(page, dontWait) == NULL) return false; VMCache* cache = page->Cache(); @@ -1617,122 +1941,596 @@ steal_page(vm_page *page) MethodDeleter _2(cache, &VMCache::ReleaseRefLocked); // check again if that page is still a candidate - if (page->busy || page->state != PAGE_STATE_INACTIVE) + if (page->busy || page->state != PAGE_STATE_CACHED) return false; DEBUG_PAGE_ACCESS_START(page); - // recheck eventual last minute changes - uint32 flags; - vm_remove_all_page_mappings(page, &flags); - if ((flags & PAGE_MODIFIED) != 0) { - // page was modified, don't steal it - set_page_state(page, PAGE_STATE_MODIFIED); - DEBUG_PAGE_ACCESS_END(page); - return false; - } else if ((flags & PAGE_ACCESSED) != 0) { - // page is in active use, don't steal it - set_page_state(page, PAGE_STATE_ACTIVE); - DEBUG_PAGE_ACCESS_END(page); - return false; - } + ASSERT(page->wired_count == 0 && page->mappings.IsEmpty()); + ASSERT(!page->modified); // we can now steal this page - //dprintf(" steal page %p from cache %p%s\n", page, page->cache, - // page->state == PAGE_STATE_INACTIVE ? "" : " (ACTIVE)"); - cache->RemovePage(page); - sInactivePageQueue.RemoveUnlocked(page); + sCachedPageQueue.RemoveUnlocked(page); return true; } -static size_t -steal_pages(vm_page **pages, size_t count) +static uint32 +free_cached_pages(uint32 pagesToFree, bool dontWait) { - while (true) { - vm_page marker; - marker.SetCacheRef(NULL); - marker.state = PAGE_STATE_UNUSED; - marker.busy = true; -#if DEBUG_PAGE_QUEUE - marker.queue = NULL; -#endif -#if DEBUG_PAGE_ACCESS - marker.accessing_thread = thread_get_current_thread_id(); -#endif + vm_page marker; + init_page_marker(marker); - bool tried = false; - size_t stolen = 0; + uint32 pagesFreed = 0; - while (count > 0) { - vm_page *page = find_page_candidate(marker); - if (page == NULL) - break; + while (pagesFreed < pagesToFree) { + vm_page *page = find_cached_page_candidate(marker); + if (page == NULL) + break; - if (steal_page(page)) { - ReadLocker locker(sFreePageQueuesLock); - page->state = PAGE_STATE_FREE; - DEBUG_PAGE_ACCESS_END(page); - sFreePageQueue.PrependUnlocked(page); - locker.Unlock(); + if (free_cached_page(page, dontWait)) { + ReadLocker locker(sFreePageQueuesLock); + page->state = PAGE_STATE_FREE; + DEBUG_PAGE_ACCESS_END(page); + sFreePageQueue.PrependUnlocked(page); + locker.Unlock(); - unreserve_page(); + TA(StolenPage()); - T(StolenPage()); - - stolen++; - count--; - } else - tried = true; + pagesFreed++; } + } - remove_page_marker(marker); + remove_page_marker(marker); - if (count == 0 || sUnreservedFreePages >= 0) - return stolen; + return pagesFreed; +} - if (stolen && !tried && sInactivePageQueue.Count() > 0) { - count++; + +// TODO: Not needed when we remove the mappings of pages in the active scan when +// moved to the inactive queue. +#if 0 +static void +idle_scan_inactive_pages(page_stats& pageStats) +{ + VMPageQueue& queue = sInactivePageQueue; + + // We want to scan the whole queue in roughly kIdleRunsForFullQueue runs. + uint32 maxToScan = queue.Count() / kIdleRunsForFullQueue + 1; + + while (maxToScan > 0) { + maxToScan--; + + // Get the next page. Note that we don't bother to lock here. We go with + // the assumption that on all architectures reading/writing pointers is + // atomic. Beyond that it doesn't really matter. We have to unlock the + // queue anyway to lock the page's cache, and we'll recheck afterwards. + vm_page* page = queue.Head(); + if (page == NULL) + break; + + // lock the page's cache + VMCache* cache = vm_cache_acquire_locked_page_cache(page, true); + if (cache == NULL) + continue; + + if (page->state != PAGE_STATE_INACTIVE) { + // page is no longer in this queue + cache->ReleaseRefAndUnlock(); continue; } - // we need to wait for pages to become inactive + if (page->busy) { + // page is busy -- requeue at the end + vm_page_requeue(page, true); + cache->ReleaseRefAndUnlock(); + continue; + } + + DEBUG_PAGE_ACCESS_START(page); + + // Get the accessed count, clear the accessed/modified flags. + int32 usageCount = vm_clear_page_mapping_accessed_flags(page); + + // update usage count + if (usageCount > 0) { + usageCount += page->usage_count + kPageUsageAdvance; + if (usageCount > kPageUsageMax) + usageCount = kPageUsageMax; + } else { + usageCount += page->usage_count - (int32)kPageUsageDecline; + if (usageCount < 0) + usageCount = 0; + } + + page->usage_count = usageCount; + + // Move to fitting queue or requeue: + // * Active mapped pages go to the active queue. + // * Inactive mapped (i.e. wired) pages are requeued. + // * Inactive unmapped pages of a temporary cache are requeued. We don't + // want to page out memory until really necessary. + // * The remaining pages are cachable. Thus, if unmodified they go to + // the cached queue, otherwise to the modified queue. + bool isMapped = page->wired_count > 0 || !page->mappings.IsEmpty(); + if (usageCount > 0) { + if (isMapped) + set_page_state(page, PAGE_STATE_ACTIVE); + else + vm_page_requeue(page, true); + } else if (isMapped || cache->temporary) { + vm_page_requeue(page, true); + } else if (page->modified) { + set_page_state(page, PAGE_STATE_MODIFIED); + } else + set_page_state(page, PAGE_STATE_CACHED); + + DEBUG_PAGE_ACCESS_END(page); + + cache->ReleaseRefAndUnlock(); + } +} +#endif + + +static void +idle_scan_active_pages(page_stats& pageStats) +{ + VMPageQueue& queue = sActivePageQueue; + + // We want to scan the whole queue in roughly kIdleRunsForFullQueue runs. + uint32 maxToScan = queue.Count() / kIdleRunsForFullQueue + 1; + + while (maxToScan > 0) { + maxToScan--; + + // Get the next page. Note that we don't bother to lock here. We go with + // the assumption that on all architectures reading/writing pointers is + // atomic. Beyond that it doesn't really matter. We have to unlock the + // queue anyway to lock the page's cache, and we'll recheck afterwards. + vm_page* page = queue.Head(); + if (page == NULL) + break; + + // lock the page's cache + VMCache* cache = vm_cache_acquire_locked_page_cache(page, true); + if (cache == NULL) + continue; + + if (cache == NULL || page->state != PAGE_STATE_ACTIVE) { + // page is no longer in the cache or in this queue + cache->ReleaseRefAndUnlock(); + continue; + } + + if (page->busy) { + // page is busy -- requeue at the end + vm_page_requeue(page, true); + cache->ReleaseRefAndUnlock(); + continue; + } + + DEBUG_PAGE_ACCESS_START(page); + + // Get the page active/modified flags and update the page's usage count. + // We completely unmap inactive temporary pages. This saves us to + // iterate through the inactive list as well, since we'll be notified + // via page fault whenever such an inactive page is used again. + // We don't remove the mappings of non-temporary pages, since we + // wouldn't notice when those would become unused and could thus be + // moved to the cached list. + int32 usageCount; + if (page->wired_count > 0 || page->usage_count > 0 || !cache->temporary) + usageCount = vm_clear_page_mapping_accessed_flags(page); + else + usageCount = vm_remove_all_page_mappings_if_unaccessed(page); + + if (usageCount > 0) { + usageCount += page->usage_count + kPageUsageAdvance; + if (usageCount > kPageUsageMax) + usageCount = kPageUsageMax; +// TODO: This would probably also be the place to reclaim swap space. + } else { + usageCount += page->usage_count - (int32)kPageUsageDecline; + if (usageCount < 0) { + usageCount = 0; + set_page_state(page, PAGE_STATE_INACTIVE); + } + } + + page->usage_count = usageCount; + + DEBUG_PAGE_ACCESS_END(page); + + cache->ReleaseRefAndUnlock(); + } +} + + +static void +full_scan_inactive_pages(page_stats& pageStats, int32 despairLevel) +{ + int32 pagesToFree = pageStats.unsatisfiedReservations + + sFreeOrCachedPagesTarget + - (pageStats.totalFreePages + pageStats.cachedPages); + if (pagesToFree <= 0) + return; + + bigtime_t time = system_time(); + uint32 pagesScanned = 0; + uint32 pagesToCached = 0; + uint32 pagesToModified = 0; + uint32 pagesToActive = 0; + + // Determine how many pages at maximum to send to the modified queue. Since + // it is relatively expensive to page out pages, we do that on a grander + // scale only when things get desperate. + uint32 maxToFlush = despairLevel <= 1 ? 32 : 10000; + + vm_page marker; + init_page_marker(marker); + + VMPageQueue& queue = sInactivePageQueue; + InterruptsSpinLocker queueLocker(queue.GetLock()); + uint32 maxToScan = queue.Count(); + + vm_page* nextPage = queue.Head(); + + while (pagesToFree > 0 && maxToScan > 0) { + maxToScan--; + + // get the next page + vm_page* page = nextPage; + if (page == NULL) + break; + nextPage = queue.Next(page); + + if (page->busy) + continue; + + // mark the position + queue.InsertAfter(page, &marker); + queueLocker.Unlock(); + + // lock the page's cache + VMCache* cache = vm_cache_acquire_locked_page_cache(page, true); + if (cache == NULL || page->busy || page->state != PAGE_STATE_INACTIVE) { + if (cache != NULL) + cache->ReleaseRefAndUnlock(); + queueLocker.Lock(); + nextPage = queue.Next(&marker); + queue.Remove(&marker); + continue; + } + + pagesScanned++; + + DEBUG_PAGE_ACCESS_START(page); + + // Get the accessed count, clear the accessed/modified flags and + // unmap the page, if it hasn't been accessed. + int32 usageCount; + if (page->wired_count > 0) + usageCount = vm_clear_page_mapping_accessed_flags(page); + else + usageCount = vm_remove_all_page_mappings_if_unaccessed(page); + + // update usage count + if (usageCount > 0) { + usageCount += page->usage_count + kPageUsageAdvance; + if (usageCount > kPageUsageMax) + usageCount = kPageUsageMax; + } else { + usageCount += page->usage_count - (int32)kPageUsageDecline; + if (usageCount < 0) + usageCount = 0; + } + + page->usage_count = usageCount; + + // Move to fitting queue or requeue: + // * Active mapped pages go to the active queue. + // * Inactive mapped (i.e. wired) pages are requeued. + // * The remaining pages are cachable. Thus, if unmodified they go to + // the cached queue, otherwise to the modified queue (up to a limit). + // Note that until in the idle scanning we don't exempt pages of + // temporary caches. Apparently we really need memory, so we better + // page out memory as well. + bool isMapped = page->wired_count > 0 || !page->mappings.IsEmpty(); + if (usageCount > 0) { + if (isMapped) { + set_page_state(page, PAGE_STATE_ACTIVE); + pagesToActive++; + } else + vm_page_requeue(page, true); + } if (isMapped) { + vm_page_requeue(page, true); + } else if (!page->modified) { + set_page_state(page, PAGE_STATE_CACHED); + pagesToFree--; + pagesToCached++; + } else if (maxToFlush > 0) { + set_page_state(page, PAGE_STATE_MODIFIED); + maxToFlush--; + pagesToModified++; + } else + vm_page_requeue(page, true); + + DEBUG_PAGE_ACCESS_END(page); + + cache->ReleaseRefAndUnlock(); + + // remove the marker + queueLocker.Lock(); + nextPage = queue.Next(&marker); + queue.Remove(&marker); + } + + time = system_time() - time; + TRACE_DAEMON(" -> inactive scan (%7lld us): scanned: %7lu, " + "moved: %lu -> cached, %lu -> modified, %lu -> active\n", time, + pagesScanned, pagesToCached, pagesToModified, pagesToActive); + + // wake up the page writer, if we tossed it some pages + if (pagesToModified > 0) + sPageWriterCondition.WakeUp(); +} + + +static void +full_scan_active_pages(page_stats& pageStats, int32 despairLevel) +{ + vm_page marker; + init_page_marker(marker); + + VMPageQueue& queue = sActivePageQueue; + InterruptsSpinLocker queueLocker(queue.GetLock()); + uint32 maxToScan = queue.Count(); + + int32 pagesToDeactivate = pageStats.unsatisfiedReservations + + sFreeOrCachedPagesTarget + - (pageStats.totalFreePages + pageStats.cachedPages) + + std::max((int32)sInactivePagesTarget - (int32)maxToScan, (int32)0); + if (pagesToDeactivate <= 0) + return; + + bigtime_t time = system_time(); + uint32 pagesAccessed = 0; + uint32 pagesToInactive = 0; + uint32 pagesScanned = 0; + + vm_page* nextPage = queue.Head(); + + while (pagesToDeactivate > 0 && maxToScan > 0) { + maxToScan--; + + // get the next page + vm_page* page = nextPage; + if (page == NULL) + break; + nextPage = queue.Next(page); + + if (page->busy) + continue; + + // mark the position + queue.InsertAfter(page, &marker); + queueLocker.Unlock(); + + // lock the page's cache + VMCache* cache = vm_cache_acquire_locked_page_cache(page, true); + if (cache == NULL || page->busy || page->state != PAGE_STATE_ACTIVE) { + if (cache != NULL) + cache->ReleaseRefAndUnlock(); + queueLocker.Lock(); + nextPage = queue.Next(&marker); + queue.Remove(&marker); + continue; + } + + pagesScanned++; + + DEBUG_PAGE_ACCESS_START(page); + + // Get the page active/modified flags and update the page's usage count. + int32 usageCount = vm_clear_page_mapping_accessed_flags(page); + + if (usageCount > 0) { + usageCount += page->usage_count + kPageUsageAdvance; + if (usageCount > kPageUsageMax) + usageCount = kPageUsageMax; + pagesAccessed++; +// TODO: This would probably also be the place to reclaim swap space. + } else { + usageCount += page->usage_count - (int32)kPageUsageDecline; + if (usageCount <= 0) { + usageCount = 0; + set_page_state(page, PAGE_STATE_INACTIVE); + pagesToInactive++; + } + } + + page->usage_count = usageCount; + + DEBUG_PAGE_ACCESS_END(page); + + cache->ReleaseRefAndUnlock(); + + // remove the marker + queueLocker.Lock(); + nextPage = queue.Next(&marker); + queue.Remove(&marker); + } + + time = system_time() - time; + TRACE_DAEMON(" -> active scan (%7lld us): scanned: %7lu, " + "moved: %lu -> inactive, encountered %lu accessed ones\n", time, + pagesScanned, pagesToInactive, pagesAccessed); +} + + +static void +page_daemon_idle_scan(page_stats& pageStats) +{ + TRACE_DAEMON("page daemon: idle run\n"); + + if (pageStats.totalFreePages < (int32)sFreePagesTarget) { + // We want more actually free pages, so free some from the cached + // ones. + uint32 freed = free_cached_pages( + sFreePagesTarget - pageStats.totalFreePages, false); + if (freed > 0) + unreserve_pages(freed); + get_page_stats(pageStats); + } + + // Walk the inactive list and transfer pages to the cached and modified and + // active queues. +// get_page_stats(pageStats); +// idle_scan_inactive_pages(pageStats); + + // Walk the active list and move pages to the inactive queue. + get_page_stats(pageStats); + idle_scan_active_pages(pageStats); +} + + +static void +page_daemon_full_scan(page_stats& pageStats, int32 despairLevel) +{ + TRACE_DAEMON("page daemon: full run: free: %lu, cached: %lu, " + "to free: %lu\n", pageStats.totalFreePages, pageStats.cachedPages, + pageStats.unsatisfiedReservations + sFreeOrCachedPagesTarget + - (pageStats.totalFreePages + pageStats.cachedPages)); + + // Walk the inactive list and transfer pages to the cached and modified + // queues. + full_scan_inactive_pages(pageStats, despairLevel); + + // Free cached pages. Also wake up reservation waiters. + get_page_stats(pageStats); + int32 pagesToFree = pageStats.unsatisfiedReservations + sFreePagesTarget + - (pageStats.totalFreePages); + if (pagesToFree > 0) { + uint32 freed = free_cached_pages(pagesToFree, true); + if (freed > 0) + unreserve_pages(freed); + } + + // Walk the active list and move pages to the inactive queue. + get_page_stats(pageStats); + full_scan_active_pages(pageStats, despairLevel); +} + + +static status_t +page_daemon(void* /*unused*/) +{ + int32 despairLevel = 0; + + while (true) { + sPageDaemonCondition.ClearActivated(); + + // evaluate the free pages situation + page_stats pageStats; + get_page_stats(pageStats); + + if (pageStats.totalFreePages + pageStats.cachedPages + >= pageStats.unsatisfiedReservations + + (int32)sFreeOrCachedPagesTarget) { + // Things look good -- just maintain statistics and keep the pool + // of actually free pages full enough. + despairLevel = 0; + page_daemon_idle_scan(pageStats); + sPageDaemonCondition.Wait(kIdleScanWaitInterval, false); + } else { + // Not enough free pages. We need to do some real work. + despairLevel = std::max(despairLevel + 1, (int32)3); + page_daemon_full_scan(pageStats, despairLevel); + + // Don't wait after the first full scan, but rather immediately + // check whether we were successful in freeing enough pages and + // re-run with increased despair level. The first scan is + // conservative with respect to moving inactive modified pages to + // the modified list to avoid thrashing. The second scan, however, + // will not hold back. + if (despairLevel > 1) + snooze(kBusyScanWaitInterval); + } + } + + return B_OK; +} + + +/*! Returns how many pages could *not* be reserved. +*/ +static uint32 +reserve_pages(uint32 count, int priority, bool dontWait) +{ + int32 dontTouch = kPageReserveForPriority[priority]; + + while (true) { + count -= reserve_some_pages(count, dontTouch); + if (count == 0) + return 0; + + if (sUnsatisfiedPageReservations == 0) { + count -= free_cached_pages(count, dontWait); + if (count == 0) + return count; + } + + if (dontWait) + return count; + + // we need to wait for pages to become available MutexLocker pageDeficitLocker(sPageDeficitLock); - sPageDeficit++; - if (sUnreservedFreePages >= 0) { - // There are enough pages available now. No need to wait after all. - sPageDeficit--; - return stolen; + bool notifyDaemon = sUnsatisfiedPageReservations == 0; + sUnsatisfiedPageReservations += count; + + if (sUnreservedFreePages > dontTouch) { + // the situation changed + sUnsatisfiedPageReservations -= count; + continue; } - ConditionVariableEntry freeConditionEntry; - freeConditionEntry.Add(&sFreePageQueue); + PageReservationWaiter waiter; + waiter.dontTouch = dontTouch; + waiter.missing = count; + waiter.thread = thread_get_current_thread(); + waiter.threadPriority = waiter.thread->priority; + + // insert ordered (i.e. after all waiters with higher or equal priority) + PageReservationWaiter* otherWaiter = NULL; + for (PageReservationWaiterList::Iterator it + = sPageReservationWaiters.GetIterator(); + (otherWaiter = it.Next()) != NULL;) { + if (waiter < *otherWaiter) + break; + } + + sPageReservationWaiters.InsertBefore(otherWaiter, &waiter); + + thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_OTHER, + "waiting for pages"); + + if (notifyDaemon) + sPageDaemonCondition.WakeUp(); + pageDeficitLocker.Unlock(); - if (tried) { - // We tried all potential pages, but one or more couldn't be stolen - // at that time (likely because their cache was locked). No one - // else will have any better luck, so we'll just retry a little - // later. - freeConditionEntry.Wait(B_RELATIVE_TIMEOUT, 10000); - } else { - low_resource(B_KERNEL_RESOURCE_PAGES, count, B_RELATIVE_TIMEOUT, 0); - //snooze(50000); - // sleep for 50ms - - freeConditionEntry.Wait(); - } + low_resource(B_KERNEL_RESOURCE_PAGES, count, B_RELATIVE_TIMEOUT, 0); + thread_block(); pageDeficitLocker.Lock(); - sPageDeficit--; - if (sUnreservedFreePages >= 0) - return stolen; + return 0; } } @@ -1879,7 +2677,7 @@ vm_page_schedule_write_page(vm_page *page) vm_page_requeue(page, false); - release_sem_etc(sWriterWaitSem, 1, B_DO_NOT_RESCHEDULE); + sPageWriterCondition.WakeUp(); } @@ -1905,20 +2703,18 @@ vm_page_schedule_write_page_range(struct VMCache *cache, uint32 firstPage, } if (modified > 0) - release_sem_etc(sWriterWaitSem, 1, B_DO_NOT_RESCHEDULE); + sPageWriterCondition.WakeUp(); } void vm_page_init_num_pages(kernel_args *args) { - uint32 i; - // calculate the size of memory by looking at the physical_memory_range array addr_t physicalPagesEnd = 0; sPhysicalPageOffset = args->physical_memory_range[0].start / B_PAGE_SIZE; - for (i = 0; i < args->num_physical_memory_ranges; i++) { + for (uint32 i = 0; i < args->num_physical_memory_ranges; i++) { physicalPagesEnd = (args->physical_memory_range[i].start + args->physical_memory_range[i].size) / B_PAGE_SIZE; } @@ -1929,8 +2725,8 @@ vm_page_init_num_pages(kernel_args *args) sNumPages = physicalPagesEnd - sPhysicalPageOffset; #ifdef LIMIT_AVAILABLE_MEMORY - if (sNumPages > LIMIT_AVAILABLE_MEMORY * 256) - sNumPages = LIMIT_AVAILABLE_MEMORY * 256; + if (sNumPages > LIMIT_AVAILABLE_MEMORY * (1024 * 1024 / B_PAGE_SIZE)) + sNumPages = LIMIT_AVAILABLE_MEMORY * (1024 * 1024 / B_PAGE_SIZE); #endif } @@ -1948,6 +2744,8 @@ vm_page_init(kernel_args *args) sFreePageQueue.Init("free pages queue"); sClearPageQueue.Init("clear pages queue"); + new (&sPageReservationWaiters) PageReservationWaiterList; + // map in the new free page table sPages = (vm_page *)vm_allocate_early(args, sNumPages * sizeof(vm_page), ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, false); @@ -1977,21 +2775,31 @@ vm_page_init(kernel_args *args) TRACE(("initialized table\n")); - // mark some of the page ranges inuse + // mark some of the page ranges in use for (uint32 i = 0; i < args->num_physical_allocated_ranges; i++) { vm_mark_page_range_inuse(args->physical_allocated_range[i].start / B_PAGE_SIZE, args->physical_allocated_range[i].size / B_PAGE_SIZE); } - TRACE(("vm_page_init: exit\n")); + // The target of actually free pages. This must be at least the system + // reserve, but should be a few more pages, so we don't have to extract + // a cached page with each allocation. + sFreePagesTarget = VM_PAGE_RESERVE_USER + + std::max((uint32)32, sNumPages / 1024); - // reserve pages for the system, that user allocations will not touch - if (sUnreservedFreePages < (int32)kMinimumSystemReserve) { - panic("Less pages than the system reserve!"); - sSystemReservedPages = sUnreservedFreePages; - } else - sSystemReservedPages = kMinimumSystemReserve; - sUnreservedFreePages -= sSystemReservedPages; + // The target of free + cached and inactive pages. On low-memory machines + // keep things tight. free + cached is the pool of immediately allocatable + // pages. We want a few inactive pages, so when we're actually paging, we + // have a reasonably large set of pages to work with. + if (sUnreservedFreePages < 16 * 1024) { + sFreeOrCachedPagesTarget = sFreePagesTarget + 128; + sInactivePagesTarget = sFreePagesTarget / 3; + } else { + sFreeOrCachedPagesTarget = 2 * sFreePagesTarget; + sInactivePagesTarget = sFreePagesTarget / 2; + } + + TRACE(("vm_page_init: exit\n")); return B_OK; } @@ -2025,6 +2833,14 @@ vm_page_init_post_area(kernel_args *args) add_debugger_command("find_page", &find_page, "Find out which queue a page is actually in"); +#ifdef TRACK_PAGE_USAGE_STATS + add_debugger_command_etc("page_usage", &dump_page_usage_stats, + "Dumps statistics about page usage counts", + "\n" + "Dumps statistics about page usage counts.\n", + B_KDEBUG_DONT_PARSE_ARGUMENTS); +#endif + return B_OK; } @@ -2043,12 +2859,20 @@ vm_page_init_post_thread(kernel_args *args) // start page writer - sWriterWaitSem = create_sem(0, "page writer"); + sPageWriterCondition.Init("page writer"); thread = spawn_kernel_thread(&page_writer, "page writer", B_NORMAL_PRIORITY + 1, NULL); send_signal_etc(thread, SIGCONT, B_DO_NOT_RESCHEDULE); + // start page daemon + + sPageDaemonCondition.Init("page daemon"); + + thread = spawn_kernel_thread(&page_daemon, "page daemon", + B_NORMAL_PRIORITY, NULL); + send_signal_etc(thread, SIGCONT, B_DO_NOT_RESCHEDULE); + return B_OK; } @@ -2118,65 +2942,19 @@ vm_mark_page_range_inuse(addr_t startPage, addr_t length) /*! Unreserve pages previously reserved with vm_page_reserve_pages(). - Note, you specify the same \a count here that you specified when - reserving the pages - you don't need to keep track how many pages - you actually needed of that upfront allocation. */ void -vm_page_unreserve_pages(uint32 count) +vm_page_unreserve_pages(vm_page_reservation* reservation) { + uint32 count = reservation->count; + reservation->count = 0; + if (count == 0) return; - T(UnreservePages(count)); + TA(UnreservePages(count)); - while (true) { - int32 freePages = sUnreservedFreePages; - if (freePages >= 0) - break; - - int32 toUnreserve = std::min((int32)count, -freePages); - if (atomic_test_and_set(&sUnreservedFreePages, - freePages + toUnreserve, freePages) == freePages) { - count -= toUnreserve; - if (count == 0) { - // TODO: Notify waiting system priority reservers. - return; - } - break; - } - - // the count changed in the meantime -- retry - } - - while (true) { - int32 systemReserve = sSystemReservedPages; - if (systemReserve >= (int32)kMinimumSystemReserve) - break; - - int32 toUnreserve = std::min((int32)count, - (int32)kMinimumSystemReserve - systemReserve); - if (atomic_test_and_set(&sSystemReservedPages, - systemReserve + toUnreserve, systemReserve) - == systemReserve) { - count -= toUnreserve; - if (count == 0) { - // TODO: Notify waiting system priority reservers. - return; - } - break; - } - - // the count changed in the meantime -- retry - } - - atomic_add(&sUnreservedFreePages, count); - - if (sPageDeficit > 0) { - MutexLocker pageDeficitLocker(sPageDeficitLock); - if (sPageDeficit > 0) - sFreePageCondition.NotifyAll(); - } + unreserve_pages(count); } @@ -2187,131 +2965,52 @@ vm_page_unreserve_pages(uint32 count) The caller must not hold any cache lock or the function might deadlock. */ void -vm_page_reserve_pages(uint32 count, int priority) +vm_page_reserve_pages(vm_page_reservation* reservation, uint32 count, + int priority) { + reservation->count = count; + if (count == 0) return; - T(ReservePages(count)); + TA(ReservePages(count)); - while (true) { - // Of the requested count reserve as many pages as possible from the - // general reserve. - int32 freePages = sUnreservedFreePages; - if (freePages <= 0) - break; - - uint32 toReserve = std::min((int32)count, freePages); - if (atomic_test_and_set(&sUnreservedFreePages, - freePages - toReserve, freePages) - != freePages) { - // the count changed in the meantime -- retry - continue; - } - - count -= toReserve; - if (count == 0) - return; - - break; - } - - // Try to get the remaining pages from the system reserve. - uint32 systemReserve = kPageReserveForPriority[priority]; - while (true) { - int32 systemFreePages = sSystemReservedPages; - if (systemFreePages > (int32)systemReserve) { - uint32 toReserve = std::min(count, systemFreePages - systemReserve); - if (atomic_test_and_set(&sSystemReservedPages, - systemFreePages - toReserve, systemFreePages) - != systemFreePages) { - // the count changed in the meantime -- retry - continue; - } - - count -= toReserve; - if (count == 0) - return; - } - - break; - } - - // subtract the remaining pages - int32 oldFreePages = atomic_add(&sUnreservedFreePages, -(int32)count); - if (oldFreePages > 0) { - if ((int32)count <= oldFreePages) - return; - count -= oldFreePages; -// TODO: Activate low-memory handling/page daemon! - } - - steal_pages(NULL, count + 1); - // we get one more, just in case we can do something someone - // else can't + reserve_pages(count, priority, false); } bool -vm_page_try_reserve_pages(uint32 count, int priority) +vm_page_try_reserve_pages(vm_page_reservation* reservation, uint32 count, + int priority) { - if (count == 0) + if (count == 0) { + reservation->count = count; return true; - - uint32 reserved; - while (true) { - // From the requested count reserve as many pages as possible from the - // general reserve. - int32 freePages = sUnreservedFreePages; - if (freePages > 0) { - reserved = std::min((int32)count, freePages); - if (atomic_test_and_set(&sUnreservedFreePages, - freePages - reserved, freePages) - != freePages) { - // the count changed in the meantime -- retry - continue; - } - - if (reserved == count) { - T(ReservePages(count)); - return true; - } - } else - reserved = 0; - - break; } - // Try to get the remaining pages from the system reserve. - uint32 systemReserve = kPageReserveForPriority[priority]; - uint32 leftToReserve = count - reserved; - while (true) { - int32 systemFreePages = sSystemReservedPages; - if ((uint32)systemFreePages < leftToReserve + systemReserve) { - // no dice - vm_page_unreserve_pages(reserved); - return false; - } - - if (atomic_test_and_set(&sSystemReservedPages, - systemFreePages - leftToReserve, systemFreePages) - == systemFreePages) { - T(ReservePages(count)); - return true; - } - - // the count changed in the meantime -- retry + uint32 remaining = reserve_pages(count, priority, true); + if (remaining == 0) { + TA(ReservePages(count)); + reservation->count = count; + return true; } + + unreserve_pages(count - remaining); + + return false; } vm_page * -vm_page_allocate_page(uint32 flags) +vm_page_allocate_page(vm_page_reservation* reservation, uint32 flags) { uint32 pageState = flags & VM_PAGE_ALLOC_STATE; ASSERT(pageState != PAGE_STATE_FREE); ASSERT(pageState != PAGE_STATE_CLEAR); + ASSERT(reservation->count > 0); + reservation->count--; + VMPageQueue* queue; VMPageQueue* otherQueue; @@ -2323,9 +3022,7 @@ vm_page_allocate_page(uint32 flags) otherQueue = &sClearPageQueue; } - atomic_add(&sUnreservedFreePages, -1); - - T(AllocatePage()); + TA(AllocatePage()); ReadLocker locker(sFreePageQueuesLock); @@ -2364,7 +3061,7 @@ vm_page_allocate_page(uint32 flags) int oldPageState = page->state; page->state = pageState; page->busy = (flags & VM_PAGE_ALLOC_BUSY) != 0; - page->usage_count = 2; + page->usage_count = 0; page->accessed = false; page->modified = false; @@ -2390,7 +3087,7 @@ allocate_page_run(page_num_t start, page_num_t length, uint32 flags, ASSERT(pageState != PAGE_STATE_FREE); ASSERT(pageState != PAGE_STATE_CLEAR); - T(AllocatePageRun(length)); + TA(AllocatePageRun(length)); // pull the pages out of the appropriate queues VMPageQueue::PageList freePages; @@ -2408,7 +3105,7 @@ allocate_page_run(page_num_t start, page_num_t length, uint32 flags, page.state = flags & VM_PAGE_ALLOC_STATE; page.busy = flags & VM_PAGE_ALLOC_BUSY; - page.usage_count = 1; + page.usage_count = 0; page.accessed = false; page.modified = false; } @@ -2442,7 +3139,8 @@ vm_page_allocate_page_run(uint32 flags, addr_t base, addr_t length, { uint32 start = base >> PAGE_SHIFT; - if (!vm_page_try_reserve_pages(length, priority)) + vm_page_reservation reservation; + if (!vm_page_try_reserve_pages(&reservation, length, priority)) return NULL; // TODO: add more tries, ie. free some inactive, ... // no free space @@ -2453,7 +3151,7 @@ vm_page_allocate_page_run(uint32 flags, addr_t base, addr_t length, bool foundRun = true; if (start + length > sNumPages) { freeClearQueueLocker.Unlock(); - vm_page_unreserve_pages(length); + vm_page_unreserve_pages(&reservation); return NULL; } @@ -2489,7 +3187,8 @@ vm_page_allocate_page_run_no_base(uint32 flags, addr_t count, int priority) otherQueue = &sClearPageQueue; } - if (!vm_page_try_reserve_pages(count, priority)) + vm_page_reservation reservation; + if (!vm_page_try_reserve_pages(&reservation, count, priority)) return NULL; // TODO: add more tries, ie. free some inactive, ... // no free space @@ -2520,7 +3219,7 @@ vm_page_allocate_page_run_no_base(uint32 flags, addr_t count, int priority) if (queue == otherQueue) { freeClearQueueLocker.Unlock(); - vm_page_unreserve_pages(count); + vm_page_unreserve_pages(&reservation); } queue = otherQueue; @@ -2650,7 +3349,7 @@ vm_page_num_available_pages(void) size_t vm_page_num_free_pages(void) { - int32 count = sUnreservedFreePages + sInactivePageQueue.Count(); + int32 count = sUnreservedFreePages + sCachedPageQueue.Count(); return count > 0 ? count : 0; }