From 7651b97c0ab8111c1ddfd04073428c3269d1b2bf Mon Sep 17 00:00:00 2001 From: Augustin Cavalier Date: Sat, 28 Dec 2024 12:50:37 -0500 Subject: [PATCH] kernel/vm: Consolidate implementations of VMTranslationMap::UnmapArea. This adds some new parameters to UnmapPage and UnmapPages. The important one is a "_flags" pointer to UnmapPage, which if specified will be filled with the page flags instead of PageUnmapped() being called (and Flush() won't be invoked, either.) This removes the remaining PAGE_STATE_* changes from VM architecture code. Change-Id: Iacbc424dd8a75a79986edcd7f04d15a10f773c87 Reviewed-on: https://review.haiku-os.org/c/haiku/+/8728 Tested-by: Commit checker robot Reviewed-by: waddlesplash --- headers/private/kernel/vm/VMTranslationMap.h | 7 +- .../paging/32bit/ARMVMTranslationMap32Bit.cpp | 144 +++------------- .../paging/32bit/ARMVMTranslationMap32Bit.h | 9 +- .../arch/arm64/VMSAv8TranslationMap.cpp | 124 +++----------- .../kernel/arch/arm64/VMSAv8TranslationMap.h | 9 +- .../paging/040/M68KVMTranslationMap040.cpp | 154 +++-------------- .../m68k/paging/040/M68KVMTranslationMap040.h | 9 +- .../ppc/paging/460/PPCVMTranslationMap460.cpp | 131 ++------------- .../ppc/paging/460/PPCVMTranslationMap460.h | 9 +- .../classic/PPCVMTranslationMapClassic.cpp | 131 ++------------- .../classic/PPCVMTranslationMapClassic.h | 9 +- .../arch/riscv64/RISCV64VMTranslationMap.cpp | 145 +++------------- .../arch/riscv64/RISCV64VMTranslationMap.h | 9 +- .../paging/32bit/X86VMTranslationMap32Bit.cpp | 141 +++------------- .../paging/32bit/X86VMTranslationMap32Bit.h | 9 +- .../paging/64bit/X86VMTranslationMap64Bit.cpp | 131 +++------------ .../paging/64bit/X86VMTranslationMap64Bit.h | 9 +- .../x86/paging/pae/X86VMTranslationMapPAE.cpp | 157 ++---------------- .../x86/paging/pae/X86VMTranslationMapPAE.h | 9 +- src/system/kernel/vm/VMTranslationMap.cpp | 102 +++++++++--- 20 files changed, 321 insertions(+), 1127 deletions(-) diff --git a/headers/private/kernel/vm/VMTranslationMap.h b/headers/private/kernel/vm/VMTranslationMap.h index 2a29e0685f..bbb493426c 100644 --- a/headers/private/kernel/vm/VMTranslationMap.h +++ b/headers/private/kernel/vm/VMTranslationMap.h @@ -44,9 +44,12 @@ public: // map not locked virtual status_t UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue) = 0; + bool updatePageQueue, + bool deletingAddressSpace = false, + uint32* _flags = NULL) = 0; virtual void UnmapPages(VMArea* area, addr_t base, - size_t size, bool updatePageQueue); + size_t size, bool updatePageQueue, + bool deletingAddressSpace = false); virtual void UnmapArea(VMArea* area, bool deletingAddressSpace, bool ignoreTopCachePageFlags); diff --git a/src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.cpp b/src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.cpp index 1780a4d3f6..32421bb2a8 100644 --- a/src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.cpp +++ b/src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.cpp @@ -335,21 +335,19 @@ ARMVMTranslationMap32Bit::DebugMarkRangePresent(addr_t start, addr_t end, } -/*! Caller must have locked the cache of the page to be unmapped. - This object shouldn't be locked. -*/ status_t ARMVMTranslationMap32Bit::UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace, uint32* _flags) { ASSERT(address % B_PAGE_SIZE == 0); - - page_directory_entry* pd = fPagingStructures->pgdir_virt; + ASSERT(_flags == NULL || !updatePageQueue); TRACE("ARMVMTranslationMap32Bit::UnmapPage(%#" B_PRIxADDR ")\n", address); RecursiveLocker locker(fLock); + page_directory_entry* pd = fPagingStructures->pgdir_virt; + int index = VADDR_TO_PDENT(address); if ((pd[index] & ARM_PDE_TYPE_MASK) == 0) return B_ENTRY_NOT_FOUND; @@ -372,13 +370,15 @@ ARMVMTranslationMap32Bit::UnmapPage(VMArea* area, addr_t address, fMapCount--; - if ((oldEntry & ARM_MMU_L2_FLAG_AP0) != 0) { // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have been // in any TLB. - InvalidatePage(address); - Flush(); + if (!deletingAddressSpace) + InvalidatePage(address); + + if (_flags == NULL) + Flush(); // NOTE: Between clearing the page table entry and Flush() other // processors (actually even this processor with another thread of the @@ -392,12 +392,21 @@ ARMVMTranslationMap32Bit::UnmapPage(VMArea* area, addr_t address, // (cf. pmap_remove_all()), unless I've missed something. } - locker.Detach(); - // PageUnmapped() will unlock for us + if (_flags == NULL) { + locker.Detach(); + // PageUnmapped() will unlock for us - PageUnmapped(area, (oldEntry & ARM_PTE_ADDRESS_MASK) / B_PAGE_SIZE, - (oldEntry & ARM_MMU_L2_FLAG_AP0) != 0, false /*(oldEntry & ARM_PTE_DIRTY) != 0*/, - updatePageQueue); + PageUnmapped(area, (oldEntry & ARM_PTE_ADDRESS_MASK) / B_PAGE_SIZE, + (oldEntry & ARM_MMU_L2_FLAG_AP0) != 0, (oldEntry & ARM_MMU_L2_FLAG_AP2) != 0, + updatePageQueue); + } else { + uint32 flags = PAGE_PRESENT; + if ((oldEntry & ARM_MMU_L2_FLAG_AP0) != 0) + flags |= PAGE_ACCESSED; + if ((oldEntry & ARM_MMU_L2_FLAG_AP2) != 0) + flags |= PAGE_MODIFIED; + *_flags = flags; + } return B_OK; } @@ -405,7 +414,7 @@ ARMVMTranslationMap32Bit::UnmapPage(VMArea* area, addr_t address, void ARMVMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace) { if (size == 0) return; @@ -450,7 +459,8 @@ ARMVMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size, // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have // been in any TLB. - InvalidatePage(start); + if (!deletingAddressSpace) + InvalidatePage(start); } if (area->cache_type != CACHE_TYPE_DEVICE) { @@ -482,108 +492,6 @@ ARMVMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size, } -void -ARMVMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace, - bool ignoreTopCachePageFlags) -{ - if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - ARMVMTranslationMap32Bit::UnmapPages(area, area->Base(), area->Size(), - true); - return; - } - - bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; - - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - RecursiveLocker locker(fLock); - - VMAreaMappings mappings; - mappings.TakeFrom(&area->mappings); - - for (VMAreaMappings::Iterator it = mappings.GetIterator(); - vm_page_mapping* mapping = it.Next();) { - vm_page* page = mapping->page; - page->mappings.Remove(mapping); - - VMCache* cache = page->Cache(); - - bool pageFullyUnmapped = false; - if (!page->IsMapped()) { - atomic_add(&gMappedPagesCount, -1); - pageFullyUnmapped = true; - } - - if (unmapPages || cache != area->cache) { - addr_t address = area->Base() - + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); - - int index = VADDR_TO_PDENT(address); - if ((pd[index] & ARM_PDE_TYPE_MASK) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page dir entry", page, area, address); - continue; - } - - ThreadCPUPinner pinner(thread_get_current_thread()); - - page_table_entry* pt - = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & ARM_PDE_ADDRESS_MASK); - page_table_entry oldEntry - = ARMPagingMethod32Bit::ClearPageTableEntry( - &pt[VADDR_TO_PTENT(address)]); - - pinner.Unlock(); - - if ((oldEntry & ARM_PTE_TYPE_MASK) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page table entry", page, area, address); - continue; - } - - // transfer the accessed/dirty flags to the page and invalidate - // the mapping, if necessary - if ((oldEntry & ARM_MMU_L2_FLAG_AP0) != 0) { - page->accessed = true; - - if (!deletingAddressSpace) - InvalidatePage(address); - } - - if (false /*(oldEntry & ARM_PTE_DIRTY) != 0*/) - page->modified = true; - - if (pageFullyUnmapped) { - DEBUG_PAGE_ACCESS_START(page); - - if (cache->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - - DEBUG_PAGE_ACCESS_END(page); - } - } - - fMapCount--; - } - - Flush(); - // flush explicitely, since we directly use the lock - - locker.Unlock(); - - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - while (vm_page_mapping* mapping = mappings.RemoveHead()) - vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); -} - - status_t ARMVMTranslationMap32Bit::Query(addr_t va, phys_addr_t *_physical, uint32 *_flags) diff --git a/src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.h b/src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.h index cee270241b..997161df53 100644 --- a/src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.h +++ b/src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.h @@ -31,12 +31,11 @@ struct ARMVMTranslationMap32Bit : ARMVMTranslationMap { bool markPresent); virtual status_t UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue); + bool updatePageQueue, + bool deletingAddressSpace, uint32* _flags); virtual void UnmapPages(VMArea* area, addr_t base, - size_t size, bool updatePageQueue); - virtual void UnmapArea(VMArea* area, - bool deletingAddressSpace, - bool ignoreTopCachePageFlags); + size_t size, bool updatePageQueue, + bool deletingAddressSpace); virtual status_t Query(addr_t virtualAddress, phys_addr_t* _physicalAddress, diff --git a/src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp b/src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp index 248408fb87..c6ba9e7798 100644 --- a/src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp +++ b/src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp @@ -400,7 +400,8 @@ flush_va_if_accessed(uint64_t pte, addr_t va, int asid) } bool -VMSAv8TranslationMap::FlushVAIfAccessed(uint64_t pte, addr_t va) { +VMSAv8TranslationMap::FlushVAIfAccessed(uint64_t pte, addr_t va) +{ InterruptsSpinLocker locker(sAsidLock); return flush_va_if_accessed(pte, va, fASID); } @@ -633,8 +634,12 @@ VMSAv8TranslationMap::Unmap(addr_t start, addr_t end) status_t -VMSAv8TranslationMap::UnmapPage(VMArea* area, addr_t address, bool updatePageQueue) +VMSAv8TranslationMap::UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue, bool deletingAddressSpace, uint32* _flags) { + ASSERT(address % B_PAGE_SIZE == 0); + ASSERT(_flags == NULL || !updatePageQueue); + TRACE("VMSAv8TranslationMap::UnmapPage(0x%" B_PRIxADDR "(%s), 0x%" B_PRIxADDR ", %d)\n", (addr_t)area, area->name, address, updatePageQueue); @@ -647,23 +652,35 @@ VMSAv8TranslationMap::UnmapPage(VMArea* area, addr_t address, bool updatePageQue ProcessRange(fPageTable, fInitialLevel, address, B_PAGE_SIZE, nullptr, [=, &oldPte](uint64_t* ptePtr, uint64_t effectiveVa) { oldPte = atomic_get_and_set64((int64_t*)ptePtr, 0); - FlushVAIfAccessed(oldPte, effectiveVa); + if (!deletingAddressSpace) + FlushVAIfAccessed(oldPte, effectiveVa); }); if ((oldPte & kPteValidMask) == 0) return B_ENTRY_NOT_FOUND; pinner.Unlock(); - locker.Detach(); - PageUnmapped(area, (oldPte & kPteAddrMask) >> fPageBits, is_pte_accessed(oldPte), - is_pte_dirty(oldPte), updatePageQueue); + + if (_flags != NULL) { + locker.Detach(); + PageUnmapped(area, (oldPte & kPteAddrMask) >> fPageBits, is_pte_accessed(oldPte), + is_pte_dirty(oldPte), updatePageQueue); + } else { + uint32 flags = PAGE_PRESENT; + if (is_pte_accessed(oldPte)) + flags |= PAGE_ACCESSED; + if (is_pte_dirty(oldPte)) + flags |= PAGE_MODIFIED; + *_flags = flags; + } return B_OK; } void -VMSAv8TranslationMap::UnmapPages(VMArea* area, addr_t address, size_t size, bool updatePageQueue) +VMSAv8TranslationMap::UnmapPages(VMArea* area, addr_t address, size_t size, + bool updatePageQueue, bool deletingAddressSpace) { TRACE("VMSAv8TranslationMap::UnmapPages(0x%" B_PRIxADDR "(%s), 0x%" B_PRIxADDR ", 0x%" B_PRIxSIZE ", %d)\n", (addr_t)area, @@ -707,99 +724,6 @@ VMSAv8TranslationMap::UnmapPages(VMArea* area, addr_t address, size_t size, bool } -void -VMSAv8TranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace, - bool ignoreTopCachePageFlags) -{ - TRACE("VMSAv8TranslationMap::UnmapArea(0x%" B_PRIxADDR "(%s), 0x%" - B_PRIxADDR ", 0x%" B_PRIxSIZE ", %d, %d)\n", (addr_t)area, - area->name, area->Base(), area->Size(), deletingAddressSpace, - ignoreTopCachePageFlags); - - if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - UnmapPages(area, area->Base(), area->Size(), true); - return; - } - - bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; - - RecursiveLocker locker(fLock); - ThreadCPUPinner pinner(thread_get_current_thread()); - - VMAreaMappings mappings; - mappings.TakeFrom(&area->mappings); - - for (VMAreaMappings::Iterator it = mappings.GetIterator(); - vm_page_mapping* mapping = it.Next();) { - - vm_page* page = mapping->page; - page->mappings.Remove(mapping); - - VMCache* cache = page->Cache(); - - bool pageFullyUnmapped = false; - if (!page->IsMapped()) { - atomic_add(&gMappedPagesCount, -1); - pageFullyUnmapped = true; - } - - if (unmapPages || cache != area->cache) { - addr_t address = area->Base() - + ((page->cache_offset * B_PAGE_SIZE) - - area->cache_offset); - - uint64_t oldPte = 0; - ProcessRange(fPageTable, fInitialLevel, address, B_PAGE_SIZE, nullptr, - [=, &oldPte](uint64_t* ptePtr, uint64_t effectiveVa) { - oldPte = atomic_get_and_set64((int64_t*)ptePtr, 0); - if (!deletingAddressSpace) - FlushVAIfAccessed(oldPte, effectiveVa); - }); - - if ((oldPte & kPteValidMask) == 0) { - panic("page %p has mapping for area %p " - "(%#" B_PRIxADDR "), but has no " - "page table", page, area, address); - continue; - } - - // transfer the accessed/dirty flags to the page and - // invalidate the mapping, if necessary - if (is_pte_dirty(oldPte)) - page->modified = true; - if (is_pte_accessed(oldPte)) - page->accessed = true; - - if (pageFullyUnmapped) { - DEBUG_PAGE_ACCESS_START(page); - - if (cache->temporary) { - vm_page_set_state(page, - PAGE_STATE_INACTIVE); - } else if (page->modified) { - vm_page_set_state(page, - PAGE_STATE_MODIFIED); - } else { - vm_page_set_state(page, - PAGE_STATE_CACHED); - } - - DEBUG_PAGE_ACCESS_END(page); - } - } - } - - locker.Unlock(); - - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - - while (vm_page_mapping* mapping = mappings.RemoveHead()) - vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); -} - - bool VMSAv8TranslationMap::ValidateVa(addr_t va) { diff --git a/src/system/kernel/arch/arm64/VMSAv8TranslationMap.h b/src/system/kernel/arch/arm64/VMSAv8TranslationMap.h index 59fbba72c8..459f707c0b 100644 --- a/src/system/kernel/arch/arm64/VMSAv8TranslationMap.h +++ b/src/system/kernel/arch/arm64/VMSAv8TranslationMap.h @@ -57,12 +57,11 @@ public: virtual status_t Unmap(addr_t start, addr_t end); virtual status_t UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue); + bool updatePageQueue, + bool deletingAddressSpace, uint32* _flags); virtual void UnmapPages(VMArea* area, addr_t base, - size_t size, bool updatePageQueue); - virtual void UnmapArea(VMArea* area, - bool deletingAddressSpace, - bool ignoreTopCachePageFlags); + size_t size, bool updatePageQueue, + bool deletingAddressSpace); virtual status_t Query(addr_t virtualAddress, phys_addr_t* _physicalAddress, diff --git a/src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.cpp b/src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.cpp index 6a0b6b5745..01a446ea5e 100644 --- a/src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.cpp +++ b/src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.cpp @@ -418,14 +418,12 @@ M68KVMTranslationMap040::Unmap(addr_t start, addr_t end) } -/*! Caller must have locked the cache of the page to be unmapped. - This object shouldn't be locked. -*/ status_t M68KVMTranslationMap040::UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace, uint32* _flags) { ASSERT(address % B_PAGE_SIZE == 0); + ASSERT(_flags == NULL || !updatePageQueue); page_root_entry* pr = fPagingStructures->pgroot_virt; @@ -475,8 +473,11 @@ M68KVMTranslationMap040::UnmapPage(VMArea* area, addr_t address, // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have been // in any TLB. - InvalidatePage(address); - Flush(); + if (!deletingAddressSpace) + InvalidatePage(address); + + if (_flags == NULL) + Flush(); // NOTE: Between clearing the page table entry and Flush() other // processors (actually even this processor with another thread of the @@ -490,12 +491,21 @@ M68KVMTranslationMap040::UnmapPage(VMArea* area, addr_t address, // (cf. pmap_remove_all()), unless I've missed something. } - locker.Detach(); - // PageUnmapped() will unlock for us + if (_flags == NULL) { + locker.Detach(); + // PageUnmapped() will unlock for us - PageUnmapped(area, (oldEntry & M68K_PTE_ADDRESS_MASK) / B_PAGE_SIZE, - (oldEntry & M68K_PTE_ACCESSED) != 0, (oldEntry & M68K_PTE_DIRTY) != 0, - updatePageQueue); + PageUnmapped(area, (oldEntry & M68K_PTE_ADDRESS_MASK) / B_PAGE_SIZE, + (oldEntry & M68K_PTE_ACCESSED) != 0, (oldEntry & M68K_PTE_DIRTY) != 0, + updatePageQueue); + } else { + uint32 flags = PAGE_PRESENT; + if ((oldEntry & M68K_PTE_ACCESSED) != 0) + flags |= PAGE_ACCESSED; + if ((oldEntry & M68K_PTE_DIRTY) != 0) + flags |= PAGE_MODIFIED; + *_flags = flags; + } return B_OK; } @@ -503,7 +513,7 @@ M68KVMTranslationMap040::UnmapPage(VMArea* area, addr_t address, void M68KVMTranslationMap040::UnmapPages(VMArea* area, addr_t base, size_t size, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace) { int index; @@ -570,7 +580,8 @@ M68KVMTranslationMap040::UnmapPages(VMArea* area, addr_t base, size_t size, // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have // been in any TLB. - InvalidatePage(start); + if (!deletingAddressSpace) + InvalidatePage(start); } if (area->cache_type != CACHE_TYPE_DEVICE) { @@ -602,123 +613,6 @@ M68KVMTranslationMap040::UnmapPages(VMArea* area, addr_t base, size_t size, } -void -M68KVMTranslationMap040::UnmapArea(VMArea* area, bool deletingAddressSpace, - bool ignoreTopCachePageFlags) -{ - if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - M68KVMTranslationMap040::UnmapPages(area, area->Base(), area->Size(), - true); - return; - } - - bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; - - page_root_entry* pr = fPagingStructures->pgroot_virt; - - RecursiveLocker locker(fLock); - - VMAreaMappings mappings; - mappings.TakeFrom(&area->mappings); - - for (VMAreaMappings::Iterator it = mappings.GetIterator(); - vm_page_mapping* mapping = it.Next();) { - vm_page* page = mapping->page; - page->mappings.Remove(mapping); - - VMCache* cache = page->Cache(); - - bool pageFullyUnmapped = false; - if (!page->IsMapped()) { - atomic_add(&gMappedPagesCount, -1); - pageFullyUnmapped = true; - } - - if (unmapPages || cache != area->cache) { - addr_t address = area->Base() - + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); - - int index; - index = VADDR_TO_PRENT(address); - if (PRE_TYPE(pr[index]) != DT_ROOT) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page root entry", page, area, address); - continue; - } - - ThreadCPUPinner pinner(thread_get_current_thread()); - - page_directory_entry* pd - = (page_directory_entry*)MapperGetPageTableAt( - pr[index] & M68K_PRE_ADDRESS_MASK); - - index = VADDR_TO_PDENT(address); - if (PDE_TYPE(pr[index]) != DT_DIR) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page dir entry", page, area, address); - continue; - } - - page_table_entry* pt - = (page_table_entry*)MapperGetPageTableAt( - pd[index] & M68K_PDE_ADDRESS_MASK); - - //XXX:M68K: DT_INDIRECT here? - - page_table_entry oldEntry - = M68KPagingMethod040::ClearPageTableEntry( - &pt[VADDR_TO_PTENT(address)]); - - pinner.Unlock(); - - if (PTE_TYPE(oldEntry) != DT_PAGE) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page table entry", page, area, address); - continue; - } - - // transfer the accessed/dirty flags to the page and invalidate - // the mapping, if necessary - if ((oldEntry & M68K_PTE_ACCESSED) != 0) { - page->accessed = true; - - if (!deletingAddressSpace) - InvalidatePage(address); - } - - if ((oldEntry & M68K_PTE_DIRTY) != 0) - page->modified = true; - - if (pageFullyUnmapped) { - DEBUG_PAGE_ACCESS_START(page); - - if (cache->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - - DEBUG_PAGE_ACCESS_END(page); - } - } - - fMapCount--; - } - - Flush(); - // flush explicitely, since we directly use the lock - - locker.Unlock(); - - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - while (vm_page_mapping* mapping = mappings.RemoveHead()) - vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); -} - - status_t M68KVMTranslationMap040::Query(addr_t va, phys_addr_t *_physical, uint32 *_flags) diff --git a/src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.h b/src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.h index 5f372861ea..e830dbe8d3 100644 --- a/src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.h +++ b/src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.h @@ -27,12 +27,11 @@ struct M68KVMTranslationMap040 : M68KVMTranslationMap { virtual status_t Unmap(addr_t start, addr_t end); virtual status_t UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue); + bool updatePageQueue, + bool deletingAddressSpace, uint32* _flags); virtual void UnmapPages(VMArea* area, addr_t base, - size_t size, bool updatePageQueue); - virtual void UnmapArea(VMArea* area, - bool deletingAddressSpace, - bool ignoreTopCachePageFlags); + size_t size, bool updatePageQueue, + bool deletingAddressSpace); virtual status_t Query(addr_t virtualAddress, phys_addr_t* _physicalAddress, diff --git a/src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.cpp b/src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.cpp index 6c4010e7fb..7e43db1822 100644 --- a/src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.cpp +++ b/src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.cpp @@ -644,14 +644,12 @@ PPCVMTranslationMap460::DebugMarkRangePresent(addr_t start, addr_t end, } -/*! Caller must have locked the cache of the page to be unmapped. - This object shouldn't be locked. -*/ status_t PPCVMTranslationMap460::UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace, uint32* _flags) { ASSERT(address % B_PAGE_SIZE == 0); + ASSERT(_flags == NULL || !updatePageQueue); RecursiveLocker locker(fLock); @@ -675,10 +673,19 @@ PPCVMTranslationMap460::UnmapPage(VMArea* area, addr_t address, fMapCount--; - locker.Detach(); - // PageUnmapped() will unlock for us + if (_flags == NULL) { + locker.Detach(); + // PageUnmapped() will unlock for us - PageUnmapped(area, pageNumber, accessed, modified, updatePageQueue); + PageUnmapped(area, pageNumber, accessed, modified, updatePageQueue); + } else { + uint32 flags = PAGE_PRESENT; + if (accessed) + flags |= PAGE_ACCESSED; + if (modified) + flags |= PAGE_MODIFIED; + *_flags = flags; + } return B_OK; @@ -747,7 +754,7 @@ PPCVMTranslationMap460::UnmapPage(VMArea* area, addr_t address, void PPCVMTranslationMap460::UnmapPages(VMArea* area, addr_t base, size_t size, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace) { panic("%s: UNIMPLEMENTED", __FUNCTION__); #if 0//X86 @@ -794,7 +801,8 @@ PPCVMTranslationMap460::UnmapPages(VMArea* area, addr_t base, size_t size, // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have // been in any TLB. - InvalidatePage(start); + if (!deletingAddressSpace) + InvalidatePage(start); } if (area->cache_type != CACHE_TYPE_DEVICE) { @@ -827,111 +835,6 @@ PPCVMTranslationMap460::UnmapPages(VMArea* area, addr_t base, size_t size, } -void -PPCVMTranslationMap460::UnmapArea(VMArea* area, bool deletingAddressSpace, - bool ignoreTopCachePageFlags) -{ - panic("%s: UNIMPLEMENTED", __FUNCTION__); -#if 0//X86 - if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - PPCVMTranslationMap460::UnmapPages(area, area->Base(), area->Size(), - true); - return; - } - - bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; - - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - RecursiveLocker locker(fLock); - - VMAreaMappings mappings; - mappings.TakeFrom(&area->mappings); - - for (VMAreaMappings::Iterator it = mappings.GetIterator(); - vm_page_mapping* mapping = it.Next();) { - vm_page* page = mapping->page; - page->mappings.Remove(mapping); - - VMCache* cache = page->Cache(); - - bool pageFullyUnmapped = false; - if (!page->IsMapped()) { - atomic_add(&gMappedPagesCount, -1); - pageFullyUnmapped = true; - } - - if (unmapPages || cache != area->cache) { - addr_t address = area->Base() - + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); - - int index = VADDR_TO_PDENT(address); - if ((pd[index] & PPC_PDE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page dir entry", page, area, address); - continue; - } - - ThreadCPUPinner pinner(thread_get_current_thread()); - - page_table_entry* pt - = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & PPC_PDE_ADDRESS_MASK); - page_table_entry oldEntry - = PPCPagingMethod460::ClearPageTableEntry( - &pt[VADDR_TO_PTENT(address)]); - - pinner.Unlock(); - - if ((oldEntry & PPC_PTE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page table entry", page, area, address); - continue; - } - - // transfer the accessed/dirty flags to the page and invalidate - // the mapping, if necessary - if ((oldEntry & PPC_PTE_ACCESSED) != 0) { - page->accessed = true; - - if (!deletingAddressSpace) - InvalidatePage(address); - } - - if ((oldEntry & PPC_PTE_DIRTY) != 0) - page->modified = true; - - if (pageFullyUnmapped) { - DEBUG_PAGE_ACCESS_START(page); - - if (cache->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - - DEBUG_PAGE_ACCESS_END(page); - } - } - - fMapCount--; - } - - Flush(); - // flush explicitely, since we directly use the lock - - locker.Unlock(); - - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - while (vm_page_mapping* mapping = mappings.RemoveHead()) - vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); -#endif -} - - status_t PPCVMTranslationMap460::Query(addr_t va, phys_addr_t *_outPhysical, uint32 *_outFlags) diff --git a/src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.h b/src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.h index e0d9e171a7..f8889e263b 100644 --- a/src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.h +++ b/src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.h @@ -42,12 +42,11 @@ struct PPCVMTranslationMap460 : PPCVMTranslationMap { bool markPresent); virtual status_t UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue); + bool updatePageQueue, + bool deletingAddressSpace, uint32* _flags); virtual void UnmapPages(VMArea* area, addr_t base, - size_t size, bool updatePageQueue); - virtual void UnmapArea(VMArea* area, - bool deletingAddressSpace, - bool ignoreTopCachePageFlags); + size_t size, bool updatePageQueue, + bool deletingAddressSpace); virtual status_t Query(addr_t virtualAddress, phys_addr_t* _physicalAddress, diff --git a/src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.cpp b/src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.cpp index dafd347dfd..7da3607737 100644 --- a/src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.cpp +++ b/src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.cpp @@ -644,14 +644,12 @@ PPCVMTranslationMapClassic::DebugMarkRangePresent(addr_t start, addr_t end, } -/*! Caller must have locked the cache of the page to be unmapped. - This object shouldn't be locked. -*/ status_t PPCVMTranslationMapClassic::UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace, uint32* _flags) { ASSERT(address % B_PAGE_SIZE == 0); + ASSERT(_flags == NULL || !updatePageQueue); RecursiveLocker locker(fLock); @@ -675,10 +673,19 @@ PPCVMTranslationMapClassic::UnmapPage(VMArea* area, addr_t address, fMapCount--; - locker.Detach(); - // PageUnmapped() will unlock for us + if (_flags == NULL) { + locker.Detach(); + // PageUnmapped() will unlock for us - PageUnmapped(area, pageNumber, accessed, modified, updatePageQueue); + PageUnmapped(area, pageNumber, accessed, modified, updatePageQueue); + } else { + uint32 flags = PAGE_PRESENT; + if (accessed) + flags |= PAGE_ACCESSED; + if (modified) + flags |= PAGE_MODIFIED; + *_flags = flags; + } return B_OK; @@ -747,7 +754,7 @@ PPCVMTranslationMapClassic::UnmapPage(VMArea* area, addr_t address, void PPCVMTranslationMapClassic::UnmapPages(VMArea* area, addr_t base, size_t size, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace) { panic("%s: UNIMPLEMENTED", __FUNCTION__); #if 0//X86 @@ -794,7 +801,8 @@ PPCVMTranslationMapClassic::UnmapPages(VMArea* area, addr_t base, size_t size, // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have // been in any TLB. - InvalidatePage(start); + if (!deletingAddressSpace) + InvalidatePage(start); } if (area->cache_type != CACHE_TYPE_DEVICE) { @@ -827,111 +835,6 @@ PPCVMTranslationMapClassic::UnmapPages(VMArea* area, addr_t base, size_t size, } -void -PPCVMTranslationMapClassic::UnmapArea(VMArea* area, bool deletingAddressSpace, - bool ignoreTopCachePageFlags) -{ - panic("%s: UNIMPLEMENTED", __FUNCTION__); -#if 0//X86 - if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - PPCVMTranslationMapClassic::UnmapPages(area, area->Base(), area->Size(), - true); - return; - } - - bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; - - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - RecursiveLocker locker(fLock); - - VMAreaMappings mappings; - mappings.TakeFrom(&area->mappings); - - for (VMAreaMappings::Iterator it = mappings.GetIterator(); - vm_page_mapping* mapping = it.Next();) { - vm_page* page = mapping->page; - page->mappings.Remove(mapping); - - VMCache* cache = page->Cache(); - - bool pageFullyUnmapped = false; - if (!page->IsMapped()) { - atomic_add(&gMappedPagesCount, -1); - pageFullyUnmapped = true; - } - - if (unmapPages || cache != area->cache) { - addr_t address = area->Base() - + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); - - int index = VADDR_TO_PDENT(address); - if ((pd[index] & PPC_PDE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page dir entry", page, area, address); - continue; - } - - ThreadCPUPinner pinner(thread_get_current_thread()); - - page_table_entry* pt - = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & PPC_PDE_ADDRESS_MASK); - page_table_entry oldEntry - = PPCPagingMethodClassic::ClearPageTableEntry( - &pt[VADDR_TO_PTENT(address)]); - - pinner.Unlock(); - - if ((oldEntry & PPC_PTE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page table entry", page, area, address); - continue; - } - - // transfer the accessed/dirty flags to the page and invalidate - // the mapping, if necessary - if ((oldEntry & PPC_PTE_ACCESSED) != 0) { - page->accessed = true; - - if (!deletingAddressSpace) - InvalidatePage(address); - } - - if ((oldEntry & PPC_PTE_DIRTY) != 0) - page->modified = true; - - if (pageFullyUnmapped) { - DEBUG_PAGE_ACCESS_START(page); - - if (cache->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - - DEBUG_PAGE_ACCESS_END(page); - } - } - - fMapCount--; - } - - Flush(); - // flush explicitely, since we directly use the lock - - locker.Unlock(); - - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - while (vm_page_mapping* mapping = mappings.RemoveHead()) - vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); -#endif -} - - status_t PPCVMTranslationMapClassic::Query(addr_t va, phys_addr_t *_outPhysical, uint32 *_outFlags) diff --git a/src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.h b/src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.h index f23791ea7a..86bb41a6c5 100644 --- a/src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.h +++ b/src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.h @@ -42,12 +42,11 @@ struct PPCVMTranslationMapClassic : PPCVMTranslationMap { bool markPresent); virtual status_t UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue); + bool updatePageQueue, + bool deletingAddressSpace, uint32* _flags); virtual void UnmapPages(VMArea* area, addr_t base, - size_t size, bool updatePageQueue); - virtual void UnmapArea(VMArea* area, - bool deletingAddressSpace, - bool ignoreTopCachePageFlags); + size_t size, bool updatePageQueue, + bool deletingAddressSpace); virtual status_t Query(addr_t virtualAddress, phys_addr_t* _physicalAddress, diff --git a/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp b/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp index b479353227..718cf5c892 100644 --- a/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp +++ b/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp @@ -320,23 +320,13 @@ RISCV64VMTranslationMap::DebugMarkRangePresent(addr_t start, addr_t end, } -/* -Things need to be done when unmapping VMArea pages - update vm_page::accessed, modified - MMIO pages: - just unmap - wired pages: - decrement wired count - non-wired pages: - remove from VMArea and vm_page `mappings` list - wired and non-wird pages - vm_page_set_state -*/ - status_t RISCV64VMTranslationMap::UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace, uint32* _flags) { + ASSERT(address % B_PAGE_SIZE == 0); + ASSERT(_flags == NULL || !updatePageQueue); + TRACE("RISCV64VMTranslationMap::UnmapPage(0x%" B_PRIxADDR "(%s), 0x%" B_PRIxADDR ", %d)\n", (addr_t)area, area->name, address, updatePageQueue); @@ -353,20 +343,35 @@ RISCV64VMTranslationMap::UnmapPage(VMArea* area, addr_t address, fMapCount--; pinner.Unlock(); - if (oldPte.isAccessed) - InvalidatePage(address); + if (oldPte.isAccessed) { + if (!deletingAddressSpace) + InvalidatePage(address); - Flush(); + if (_flags == NULL) + Flush(); + } + + if (_flags == NULL) { + locker.Detach(); + // PageUnmapped() will unlock for us + + PageUnmapped(area, oldPte.ppn, oldPte.isAccessed, oldPte.isDirty, updatePageQueue); + } else { + uint32 flags = PAGE_PRESENT; + if (oldPte.isAccessed) + flags |= PAGE_ACCESSED; + if (oldPte.isDirty) + flags |= PAGE_MODIFIED; + *_flags = flags; + } - locker.Detach(); // PageUnmapped takes ownership - PageUnmapped(area, oldPte.ppn, oldPte.isAccessed, oldPte.isDirty, updatePageQueue); return B_OK; } void RISCV64VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace) { TRACE("RISCV64VMTranslationMap::UnmapPages(0x%" B_PRIxADDR "(%s), 0x%" B_PRIxADDR ", 0x%" B_PRIxSIZE ", %d)\n", (addr_t)area, @@ -392,7 +397,7 @@ RISCV64VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size, fMapCount--; - if (oldPte.isAccessed) + if (oldPte.isAccessed && !deletingAddressSpace) InvalidatePage(start); if (area->cache_type != CACHE_TYPE_DEVICE) { @@ -421,104 +426,6 @@ RISCV64VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size, } -void -RISCV64VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace, - bool ignoreTopCachePageFlags) -{ - TRACE("RISCV64VMTranslationMap::UnmapArea(0x%" B_PRIxADDR "(%s), 0x%" - B_PRIxADDR ", 0x%" B_PRIxSIZE ", %d, %d)\n", (addr_t)area, - area->name, area->Base(), area->Size(), deletingAddressSpace, - ignoreTopCachePageFlags); - - if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - UnmapPages(area, area->Base(), area->Size(), true); - return; - } - - bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; - - RecursiveLocker locker(fLock); - ThreadCPUPinner pinner(thread_get_current_thread()); - - VMAreaMappings mappings; - mappings.TakeFrom(&area->mappings); - - for (VMAreaMappings::Iterator it = mappings.GetIterator(); - vm_page_mapping* mapping = it.Next();) { - - vm_page* page = mapping->page; - page->mappings.Remove(mapping); - - VMCache* cache = page->Cache(); - - bool pageFullyUnmapped = false; - if (!page->IsMapped()) { - atomic_add(&gMappedPagesCount, -1); - pageFullyUnmapped = true; - } - - if (unmapPages || cache != area->cache) { - addr_t address = area->Base() - + ((page->cache_offset * B_PAGE_SIZE) - - area->cache_offset); - - std::atomic* pte = LookupPte(address, false, NULL); - if (pte == NULL || !pte->load().isValid) { - panic("page %p has mapping for area %p " - "(%#" B_PRIxADDR "), but has no " - "page table", page, area, address); - continue; - } - - Pte oldPte = pte->exchange({}); - - // transfer the accessed/dirty flags to the page and - // invalidate the mapping, if necessary - if (oldPte.isAccessed) { - page->accessed = true; - - if (!deletingAddressSpace) - InvalidatePage(address); - } - - if (oldPte.isDirty) - page->modified = true; - - if (pageFullyUnmapped) { - DEBUG_PAGE_ACCESS_START(page); - - if (cache->temporary) { - vm_page_set_state(page, - PAGE_STATE_INACTIVE); - } else if (page->modified) { - vm_page_set_state(page, - PAGE_STATE_MODIFIED); - } else { - vm_page_set_state(page, - PAGE_STATE_CACHED); - } - - DEBUG_PAGE_ACCESS_END(page); - } - } - - fMapCount--; - } - - Flush(); - // flush explicitely, since we directly use the lock - - locker.Unlock(); - - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - - while (vm_page_mapping* mapping = mappings.RemoveHead()) - vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); -} - - status_t RISCV64VMTranslationMap::Query(addr_t virtualAddress, phys_addr_t* _physicalAddress, uint32* _flags) diff --git a/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.h b/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.h index 4c8aceecc0..ff3ce86595 100644 --- a/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.h +++ b/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.h @@ -43,12 +43,11 @@ struct RISCV64VMTranslationMap: public VMTranslationMap { bool markPresent); virtual status_t UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue); + bool updatePageQueue, + bool deletingAddressSpace, uint32* _flags); virtual void UnmapPages(VMArea* area, addr_t base, - size_t size, bool updatePageQueue); - virtual void UnmapArea(VMArea* area, - bool deletingAddressSpace, - bool ignoreTopCachePageFlags); + size_t size, bool updatePageQueue, + bool deletingAddressSpace); virtual status_t Query(addr_t virtualAddress, phys_addr_t* _physicalAddress, diff --git a/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp b/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp index 4bd2760066..a24ac98400 100644 --- a/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp +++ b/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp @@ -319,14 +319,12 @@ X86VMTranslationMap32Bit::DebugMarkRangePresent(addr_t start, addr_t end, } -/*! Caller must have locked the cache of the page to be unmapped. - This object shouldn't be locked. -*/ status_t X86VMTranslationMap32Bit::UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace, uint32* _flags) { ASSERT(address % B_PAGE_SIZE == 0); + ASSERT(_flags == NULL || !updatePageQueue); page_directory_entry* pd = fPagingStructures->pgdir_virt; @@ -360,8 +358,13 @@ X86VMTranslationMap32Bit::UnmapPage(VMArea* area, addr_t address, // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have been // in any TLB. - InvalidatePage(address); - Flush(); + if (!deletingAddressSpace) + InvalidatePage(address); + + if (_flags == NULL) { + Flush(); + // flush explicitly, since we directly use the lock + } // NOTE: Between clearing the page table entry and Flush() other // processors (actually even this processor with another thread of the @@ -375,12 +378,21 @@ X86VMTranslationMap32Bit::UnmapPage(VMArea* area, addr_t address, // (cf. pmap_remove_all()), unless I've missed something. } - locker.Detach(); - // PageUnmapped() will unlock for us + if (_flags == NULL) { + locker.Detach(); + // PageUnmapped() will unlock for us - PageUnmapped(area, (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE, - (oldEntry & X86_PTE_ACCESSED) != 0, (oldEntry & X86_PTE_DIRTY) != 0, - updatePageQueue); + PageUnmapped(area, (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE, + (oldEntry & X86_PTE_ACCESSED) != 0, + (oldEntry & X86_PTE_DIRTY) != 0, updatePageQueue); + } else { + uint32 flags = PAGE_PRESENT; + if ((oldEntry & X86_PTE_ACCESSED) != 0) + flags |= PAGE_ACCESSED; + if ((oldEntry & X86_PTE_DIRTY) != 0) + flags |= PAGE_MODIFIED; + *_flags = flags; + } return B_OK; } @@ -388,7 +400,7 @@ X86VMTranslationMap32Bit::UnmapPage(VMArea* area, addr_t address, void X86VMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace) { if (size == 0) return; @@ -433,7 +445,8 @@ X86VMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size, // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have // been in any TLB. - InvalidatePage(start); + if (!deletingAddressSpace) + InvalidatePage(start); } if (area->cache_type != CACHE_TYPE_DEVICE) { @@ -465,108 +478,6 @@ X86VMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size, } -void -X86VMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace, - bool ignoreTopCachePageFlags) -{ - if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - X86VMTranslationMap32Bit::UnmapPages(area, area->Base(), area->Size(), - true); - return; - } - - bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; - - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - RecursiveLocker locker(fLock); - - VMAreaMappings mappings; - mappings.TakeFrom(&area->mappings); - - for (VMAreaMappings::Iterator it = mappings.GetIterator(); - vm_page_mapping* mapping = it.Next();) { - vm_page* page = mapping->page; - page->mappings.Remove(mapping); - - VMCache* cache = page->Cache(); - - bool pageFullyUnmapped = false; - if (!page->IsMapped()) { - atomic_add(&gMappedPagesCount, -1); - pageFullyUnmapped = true; - } - - if (unmapPages || cache != area->cache) { - addr_t address = area->Base() - + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); - - int index = VADDR_TO_PDENT(address); - if ((pd[index] & X86_PDE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page dir entry", page, area, address); - continue; - } - - ThreadCPUPinner pinner(thread_get_current_thread()); - - page_table_entry* pt - = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - page_table_entry oldEntry - = X86PagingMethod32Bit::ClearPageTableEntry( - &pt[VADDR_TO_PTENT(address)]); - - pinner.Unlock(); - - if ((oldEntry & X86_PTE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page table entry", page, area, address); - continue; - } - - // transfer the accessed/dirty flags to the page and invalidate - // the mapping, if necessary - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - page->accessed = true; - - if (!deletingAddressSpace) - InvalidatePage(address); - } - - if ((oldEntry & X86_PTE_DIRTY) != 0) - page->modified = true; - - if (pageFullyUnmapped) { - DEBUG_PAGE_ACCESS_START(page); - - if (cache->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - - DEBUG_PAGE_ACCESS_END(page); - } - } - - fMapCount--; - } - - Flush(); - // flush explicitely, since we directly use the lock - - locker.Unlock(); - - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - while (vm_page_mapping* mapping = mappings.RemoveHead()) - vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); -} - - status_t X86VMTranslationMap32Bit::Query(addr_t va, phys_addr_t *_physical, uint32 *_flags) diff --git a/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.h b/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.h index 1ea2130fb3..f43ed4885b 100644 --- a/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.h +++ b/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.h @@ -31,12 +31,11 @@ struct X86VMTranslationMap32Bit final : X86VMTranslationMap { bool markPresent); virtual status_t UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue); + bool updatePageQueue, + bool deletingAddressSpace, uint32* _flags); virtual void UnmapPages(VMArea* area, addr_t base, - size_t size, bool updatePageQueue); - virtual void UnmapArea(VMArea* area, - bool deletingAddressSpace, - bool ignoreTopCachePageFlags); + size_t size, bool updatePageQueue, + bool deletingAddressSpace); virtual status_t Query(addr_t virtualAddress, phys_addr_t* _physicalAddress, diff --git a/src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.cpp b/src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.cpp index e11a5ac797..466e4d7e61 100644 --- a/src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.cpp +++ b/src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.cpp @@ -337,9 +337,10 @@ X86VMTranslationMap64Bit::DebugMarkRangePresent(addr_t start, addr_t end, status_t X86VMTranslationMap64Bit::UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace, uint32* _flags) { ASSERT(address % B_PAGE_SIZE == 0); + ASSERT(_flags == NULL || !updatePageQueue); TRACE("X86VMTranslationMap64Bit::UnmapPage(%#" B_PRIxADDR ")\n", address); @@ -367,9 +368,13 @@ X86VMTranslationMap64Bit::UnmapPage(VMArea* area, addr_t address, // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have been // in any TLB. - InvalidatePage(address); + if (!deletingAddressSpace) + InvalidatePage(address); - Flush(); + if (_flags == NULL) { + Flush(); + // flush explicitly, since we directly use the lock + } // NOTE: Between clearing the page table entry and Flush() other // processors (actually even this processor with another thread of the @@ -383,12 +388,21 @@ X86VMTranslationMap64Bit::UnmapPage(VMArea* area, addr_t address, // (cf. pmap_remove_all()), unless I've missed something. } - locker.Detach(); - // PageUnmapped() will unlock for us + if (_flags == NULL) { + locker.Detach(); + // PageUnmapped() will unlock for us - PageUnmapped(area, (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE, - (oldEntry & X86_64_PTE_ACCESSED) != 0, - (oldEntry & X86_64_PTE_DIRTY) != 0, updatePageQueue); + PageUnmapped(area, (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE, + (oldEntry & X86_64_PTE_ACCESSED) != 0, + (oldEntry & X86_64_PTE_DIRTY) != 0, updatePageQueue); + } else { + uint32 flags = PAGE_PRESENT; + if ((oldEntry & X86_64_PTE_ACCESSED) != 0) + flags |= PAGE_ACCESSED; + if ((oldEntry & X86_64_PTE_DIRTY) != 0) + flags |= PAGE_MODIFIED; + *_flags = flags; + } return B_OK; } @@ -396,7 +410,7 @@ X86VMTranslationMap64Bit::UnmapPage(VMArea* area, addr_t address, void X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace) { if (size == 0) return; @@ -436,7 +450,8 @@ X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size, // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have // been in any TLB. - InvalidatePage(start); + if (!deletingAddressSpace) + InvalidatePage(start); } if (area->cache_type != CACHE_TYPE_DEVICE) { @@ -468,102 +483,6 @@ X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size, } -void -X86VMTranslationMap64Bit::UnmapArea(VMArea* area, bool deletingAddressSpace, - bool ignoreTopCachePageFlags) -{ - TRACE("X86VMTranslationMap64Bit::UnmapArea(%p)\n", area); - - if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - X86VMTranslationMap64Bit::UnmapPages(area, area->Base(), area->Size(), - true); - return; - } - - bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; - - RecursiveLocker locker(fLock); - ThreadCPUPinner pinner(thread_get_current_thread()); - - VMAreaMappings mappings; - mappings.TakeFrom(&area->mappings); - - for (VMAreaMappings::Iterator it = mappings.GetIterator(); - vm_page_mapping* mapping = it.Next();) { - vm_page* page = mapping->page; - page->mappings.Remove(mapping); - - VMCache* cache = page->Cache(); - - bool pageFullyUnmapped = false; - if (!page->IsMapped()) { - atomic_add(&gMappedPagesCount, -1); - pageFullyUnmapped = true; - } - - if (unmapPages || cache != area->cache) { - addr_t address = area->Base() - + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); - - uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress( - fPagingStructures->VirtualPMLTop(), address, fIsKernelMap, - false, NULL, fPageMapper, fMapCount); - if (entry == NULL) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page table", page, area, address); - continue; - } - - uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry); - - if ((oldEntry & X86_64_PTE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page table entry", page, area, address); - continue; - } - - // transfer the accessed/dirty flags to the page and invalidate - // the mapping, if necessary - if ((oldEntry & X86_64_PTE_ACCESSED) != 0) { - page->accessed = true; - - if (!deletingAddressSpace) - InvalidatePage(address); - } - - if ((oldEntry & X86_64_PTE_DIRTY) != 0) - page->modified = true; - - if (pageFullyUnmapped) { - DEBUG_PAGE_ACCESS_START(page); - - if (cache->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - - DEBUG_PAGE_ACCESS_END(page); - } - } - - fMapCount--; - } - - Flush(); - // flush explicitely, since we directly use the lock - - locker.Unlock(); - - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - while (vm_page_mapping* mapping = mappings.RemoveHead()) - vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); -} - - status_t X86VMTranslationMap64Bit::Query(addr_t virtualAddress, phys_addr_t* _physicalAddress, uint32* _flags) diff --git a/src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.h b/src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.h index b7d5732930..44747f93a9 100644 --- a/src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.h +++ b/src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.h @@ -32,12 +32,11 @@ struct X86VMTranslationMap64Bit final : X86VMTranslationMap { bool markPresent); virtual status_t UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue); + bool updatePageQueue, + bool deletingAddressSpace, uint32* _flags); virtual void UnmapPages(VMArea* area, addr_t base, - size_t size, bool updatePageQueue); - virtual void UnmapArea(VMArea* area, - bool deletingAddressSpace, - bool ignoreTopCachePageFlags); + size_t size, bool updatePageQueue, + bool deletingAddressSpace); virtual status_t Query(addr_t virtualAddress, phys_addr_t* _physicalAddress, diff --git a/src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.cpp b/src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.cpp index 2f6eed1f21..89ffd0dbba 100644 --- a/src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.cpp +++ b/src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.cpp @@ -558,27 +558,25 @@ X86VMTranslationMapPAE::DebugMarkRangePresent(addr_t start, addr_t end, } -/*! Caller must have locked the cache of the page to be unmapped. - This object shouldn't be locked. -*/ status_t X86VMTranslationMapPAE::UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace, uint32* _flags) { ASSERT(address % B_PAGE_SIZE == 0); + ASSERT(_flags == NULL || !updatePageQueue); + + TRACE("X86VMTranslationMapPAE::UnmapPage(%#" B_PRIxADDR ")\n", address); + + ThreadCPUPinner pinner(thread_get_current_thread()); pae_page_directory_entry* pageDirEntry = X86PagingMethodPAE::PageDirEntryForAddress( fPagingStructures->VirtualPageDirs(), address); - TRACE("X86VMTranslationMapPAE::UnmapPage(%#" B_PRIxADDR ")\n", address); - - RecursiveLocker locker(fLock); - if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) return B_ENTRY_NOT_FOUND; - ThreadCPUPinner pinner(thread_get_current_thread()); + RecursiveLocker locker(fLock); pae_page_table_entry* pageTable = (pae_page_table_entry*)fPageMapper->GetPageTableAt( @@ -602,9 +600,13 @@ X86VMTranslationMapPAE::UnmapPage(VMArea* area, addr_t address, // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have been // in any TLB. - InvalidatePage(address); + if (!deletingAddressSpace) + InvalidatePage(address); - Flush(); + if (_flags == NULL) { + Flush(); + // flush explicitly, since we directly use the lock + } // NOTE: Between clearing the page table entry and Flush() other // processors (actually even this processor with another thread of the @@ -631,7 +633,7 @@ X86VMTranslationMapPAE::UnmapPage(VMArea* area, addr_t address, void X86VMTranslationMapPAE::UnmapPages(VMArea* area, addr_t base, size_t size, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace) { if (size == 0) return; @@ -680,7 +682,8 @@ X86VMTranslationMapPAE::UnmapPages(VMArea* area, addr_t base, size_t size, // Note, that we only need to invalidate the address, if the // accessed flags was set, since only then the entry could have // been in any TLB. - InvalidatePage(start); + if (!deletingAddressSpace) + InvalidatePage(start); } if (area->cache_type != CACHE_TYPE_DEVICE) { @@ -712,134 +715,6 @@ X86VMTranslationMapPAE::UnmapPages(VMArea* area, addr_t base, size_t size, } -void -X86VMTranslationMapPAE::UnmapArea(VMArea* area, bool deletingAddressSpace, - bool ignoreTopCachePageFlags) -{ - if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - X86VMTranslationMapPAE::UnmapPages(area, area->Base(), area->Size(), - true); - return; - } - - bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; - - RecursiveLocker locker(fLock); - - VMAreaMappings mappings; - mappings.TakeFrom(&area->mappings); - - for (VMAreaMappings::Iterator it = mappings.GetIterator(); - vm_page_mapping* mapping = it.Next();) { - vm_page* page = mapping->page; - page->mappings.Remove(mapping); - - VMCache* cache = page->Cache(); - - bool pageFullyUnmapped = false; - if (!page->IsMapped()) { - atomic_add(&gMappedPagesCount, -1); - pageFullyUnmapped = true; - } - - if (unmapPages || cache != area->cache) { - addr_t address = area->Base() - + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); - - pae_page_directory_entry* pageDirEntry - = X86PagingMethodPAE::PageDirEntryForAddress( - fPagingStructures->VirtualPageDirs(), address); - if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page dir entry", page, area, address); - continue; - } - - ThreadCPUPinner pinner(thread_get_current_thread()); - - pae_page_table_entry* pageTable - = (pae_page_table_entry*)fPageMapper->GetPageTableAt( - *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); - pae_page_table_entry oldEntry - = X86PagingMethodPAE::ClearTableEntry( - &pageTable[address / B_PAGE_SIZE - % kPAEPageTableEntryCount]); - - pinner.Unlock(); - - if ((oldEntry & X86_PAE_PTE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page table entry", page, area, address); - continue; - } - - T(Unmap(this, address, oldEntry)); - - // transfer the accessed/dirty flags to the page and invalidate - // the mapping, if necessary - if ((oldEntry & X86_PAE_PTE_ACCESSED) != 0) { - page->accessed = true; - - if (!deletingAddressSpace) - InvalidatePage(address); - } - - if ((oldEntry & X86_PAE_PTE_DIRTY) != 0) - page->modified = true; - - if (pageFullyUnmapped) { - DEBUG_PAGE_ACCESS_START(page); - - if (cache->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - - DEBUG_PAGE_ACCESS_END(page); - } - } else { -#if TRANSLATION_MAP_TRACING - addr_t address = area->Base() - + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); - - ThreadCPUPinner pinner(thread_get_current_thread()); - - pae_page_directory_entry* pageDirEntry - = X86PagingMethodPAE::PageDirEntryForAddress( - fPagingStructures->VirtualPageDirs(), address); - if ((*pageDirEntry & X86_PAE_PDE_PRESENT) != 0) { - pae_page_table_entry* pageTable - = (pae_page_table_entry*)fPageMapper->GetPageTableAt( - *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK); - pae_page_table_entry oldEntry = pageTable[ - address / B_PAGE_SIZE % kPAEPageTableEntryCount]; - - pinner.Unlock(); - - if ((oldEntry & X86_PAE_PTE_PRESENT) != 0) - T(Unmap(this, address, oldEntry)); - } -#endif - } - - fMapCount--; - } - - Flush(); - // flush explicitely, since we directly use the lock - - locker.Unlock(); - - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - while (vm_page_mapping* mapping = mappings.RemoveHead()) - vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); -} - - status_t X86VMTranslationMapPAE::Query(addr_t virtualAddress, phys_addr_t* _physicalAddress, uint32* _flags) diff --git a/src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.h b/src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.h index c630be1619..c55422486f 100644 --- a/src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.h +++ b/src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.h @@ -34,12 +34,11 @@ struct X86VMTranslationMapPAE final : X86VMTranslationMap { bool markPresent); virtual status_t UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue); + bool updatePageQueue, + bool deletingAddressSpace, uint32* _flags); virtual void UnmapPages(VMArea* area, addr_t base, - size_t size, bool updatePageQueue); - virtual void UnmapArea(VMArea* area, - bool deletingAddressSpace, - bool ignoreTopCachePageFlags); + size_t size, bool updatePageQueue, + bool deletingAddressSpace); virtual status_t Query(addr_t virtualAddress, phys_addr_t* _physicalAddress, diff --git a/src/system/kernel/vm/VMTranslationMap.cpp b/src/system/kernel/vm/VMTranslationMap.cpp index 1cd4663a88..e01bef0083 100644 --- a/src/system/kernel/vm/VMTranslationMap.cpp +++ b/src/system/kernel/vm/VMTranslationMap.cpp @@ -46,7 +46,7 @@ VMTranslationMap::DebugMarkRangePresent(addr_t start, addr_t end, */ void VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size, - bool updatePageQueue) + bool updatePageQueue, bool deletingAddressSpace) { ASSERT(base % B_PAGE_SIZE == 0); ASSERT(size % B_PAGE_SIZE == 0); @@ -62,57 +62,113 @@ VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size, vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE); if (page != NULL) { DEBUG_PAGE_ACCESS_START(page); - UnmapPage(area, address, updatePageQueue); + UnmapPage(area, address, updatePageQueue, deletingAddressSpace); DEBUG_PAGE_ACCESS_END(page); } else - UnmapPage(area, address, updatePageQueue); + UnmapPage(area, address, updatePageQueue, deletingAddressSpace); } } #else for (; address != end; address += B_PAGE_SIZE) - UnmapPage(area, address, updatePageQueue); + UnmapPage(area, address, updatePageQueue, deletingAddressSpace); #endif } /*! Unmaps all of an area's pages. + If \a deletingAddressSpace is \c true, the address space the area belongs to is in the process of being destroyed and isn't used by anyone anymore. For some architectures this can be used for optimizations (e.g. not unmapping pages or at least not needing to invalidate TLB entries). + If \a ignoreTopCachePageFlags is \c true, the area is in the process of being destroyed and its top cache is otherwise unreferenced. I.e. all mapped pages that live in the top cache area going to be freed and the page accessed and modified flags don't need to be propagated. - The default implementation just iterates over all virtual pages of the - area and calls UnmapPage(). This is obviously not particularly efficient. + The default implementation iterates over all mapping objects, and calls + UnmapPage() (with \c _flags specified to avoid calls to PageUnmapped). + It skips unmapping pages owned by the top cache if \a deletingAddressSpace + is \c true, or if \a ignoreTopCachePageFlags is set. This behavior should + be sufficient for most (if not all) architectures. */ void VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace, bool ignoreTopCachePageFlags) { - addr_t address = area->Base(); - addr_t end = address + area->Size(); -#if DEBUG_PAGE_ACCESS - for (; address != end; address += B_PAGE_SIZE) { - phys_addr_t physicalAddress; - uint32 flags; - if (Query(address, &physicalAddress, &flags) == B_OK - && (flags & PAGE_PRESENT) != 0) { - vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE); - if (page != NULL) { + if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { + UnmapPages(area, area->Base(), area->Size(), true, deletingAddressSpace); + return; + } + + const bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; + + Lock(); + + VMAreaMappings mappings; + mappings.TakeFrom(&area->mappings); + + for (VMAreaMappings::Iterator it = mappings.GetIterator(); + vm_page_mapping* mapping = it.Next();) { + vm_page* page = mapping->page; + page->mappings.Remove(mapping); + + VMCache* cache = page->Cache(); + + bool pageFullyUnmapped = false; + if (!page->IsMapped()) { + atomic_add(&gMappedPagesCount, -1); + pageFullyUnmapped = true; + } + + if (unmapPages || cache != area->cache) { + const addr_t address = area->Base() + + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); + + // UnmapPage should skip flushing and calling PageUnmapped when we pass &flags. + uint32 flags = 0; + status_t status = UnmapPage(area, address, false, deletingAddressSpace, &flags); + if (status == B_ENTRY_NOT_FOUND) { + panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " + "has no translation map entry", page, area, address); + continue; + } + if (status != B_OK) { + panic("unmapping page %p for area %p (%#" B_PRIxADDR ") failed: %x", + page, area, address, status); + continue; + } + + // Transfer the accessed/dirty flags to the page. + if ((flags & PAGE_ACCESSED) != 0) + page->accessed = true; + if ((flags & PAGE_MODIFIED) != 0) + page->modified = true; + + if (pageFullyUnmapped) { DEBUG_PAGE_ACCESS_START(page); - UnmapPage(area, address, true); + + if (cache->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + DEBUG_PAGE_ACCESS_END(page); - } else - UnmapPage(area, address, true); + } } } -#else - for (; address != end; address += B_PAGE_SIZE) - UnmapPage(area, address, true); -#endif + + // This should Flush(), if necessary. + Unlock(); + + bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); + uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY + | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); + while (vm_page_mapping* mapping = mappings.RemoveHead()) + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); }