From c6caf520ca20ba155cbcf24ab28209c9cf028961 Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Mon, 7 Jun 2010 20:36:33 +0000 Subject: [PATCH] * Added a level of indirection for the arch_vm_translation_map functions. Introduced the interface X86PagingMethod which is used by those. ATM there's one implementing class, X86PagingMethod32Bit. * Made X86PagingStructures a base class, with one derived class, X86PagingStructures32Bit. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@37050 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- src/system/kernel/arch/x86/Jamfile | 2 + .../kernel/arch/x86/X86PagingMethod.cpp | 12 + src/system/kernel/arch/x86/X86PagingMethod.h | 41 + .../kernel/arch/x86/X86PagingMethod32Bit.cpp | 1503 +++++++++++++++++ .../kernel/arch/x86/X86PagingMethod32Bit.h | 46 + .../arch/x86/arch_vm_translation_map.cpp | 1420 +--------------- src/system/kernel/arch/x86/x86_paging.h | 2 +- 7 files changed, 1614 insertions(+), 1412 deletions(-) create mode 100644 src/system/kernel/arch/x86/X86PagingMethod.cpp create mode 100644 src/system/kernel/arch/x86/X86PagingMethod.h create mode 100644 src/system/kernel/arch/x86/X86PagingMethod32Bit.cpp create mode 100644 src/system/kernel/arch/x86/X86PagingMethod32Bit.h diff --git a/src/system/kernel/arch/x86/Jamfile b/src/system/kernel/arch/x86/Jamfile index a63b066284..39b4d70590 100644 --- a/src/system/kernel/arch/x86/Jamfile +++ b/src/system/kernel/arch/x86/Jamfile @@ -40,6 +40,8 @@ KernelMergeObject kernel_arch_x86.o : x86_physical_page_mapper.cpp x86_physical_page_mapper_large_memory.cpp x86_syscalls.cpp + X86PagingMethod.cpp + X86PagingMethod32Bit.cpp x86_apic.cpp x86_hpet.cpp diff --git a/src/system/kernel/arch/x86/X86PagingMethod.cpp b/src/system/kernel/arch/x86/X86PagingMethod.cpp new file mode 100644 index 0000000000..8491c2d83e --- /dev/null +++ b/src/system/kernel/arch/x86/X86PagingMethod.cpp @@ -0,0 +1,12 @@ +/* + * Copyright 2010, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ + + +#include "X86PagingMethod.h" + + +X86PagingMethod::~X86PagingMethod() +{ +} diff --git a/src/system/kernel/arch/x86/X86PagingMethod.h b/src/system/kernel/arch/x86/X86PagingMethod.h new file mode 100644 index 0000000000..d3a574384d --- /dev/null +++ b/src/system/kernel/arch/x86/X86PagingMethod.h @@ -0,0 +1,41 @@ +/* + * Copyright 2010, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef KERNEL_ARCH_X86_X86_PAGING_METHOD_H +#define KERNEL_ARCH_X86_X86_PAGING_METHOD_H + + +#include + + +struct kernel_args; +struct VMPhysicalPageMapper; +struct VMTranslationMap; + + +class X86PagingMethod { +public: + virtual ~X86PagingMethod(); + + virtual status_t Init(kernel_args* args, + VMPhysicalPageMapper** _physicalPageMapper) + = 0; + virtual status_t InitPostArea(kernel_args* args) = 0; + + virtual status_t CreateTranslationMap(bool kernel, + VMTranslationMap** _map) = 0; + + virtual status_t MapEarly(kernel_args* args, + addr_t virtualAddress, + phys_addr_t physicalAddress, + uint8 attributes, + phys_addr_t (*get_free_page)(kernel_args*)) + = 0; + + virtual bool IsKernelPageAccessible(addr_t virtualAddress, + uint32 protection) = 0; +}; + + +#endif // KERNEL_ARCH_X86_X86_PAGING_METHOD_H diff --git a/src/system/kernel/arch/x86/X86PagingMethod32Bit.cpp b/src/system/kernel/arch/x86/X86PagingMethod32Bit.cpp new file mode 100644 index 0000000000..b7a4fcdd4d --- /dev/null +++ b/src/system/kernel/arch/x86/X86PagingMethod32Bit.cpp @@ -0,0 +1,1503 @@ +/* + * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. + * Distributed under the terms of the NewOS License. + */ + + +#include "X86PagingMethod32Bit.h" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "x86_physical_page_mapper.h" +#include "X86VMTranslationMap.h" + + +//#define TRACE_X86_PAGING_METHOD_32_BIT +#ifdef TRACE_X86_PAGING_METHOD_32_BIT +# define TRACE(x...) dprintf(x) +#else +# define TRACE(x...) ; +#endif + + +static X86PagingMethod32Bit sMethod; + +static page_table_entry *sPageHole = NULL; +static page_directory_entry *sPageHolePageDir = NULL; +static uint32 sKernelPhysicalPageDirectory = 0; +static page_directory_entry *sKernelVirtualPageDirectory = NULL; + +static X86PhysicalPageMapper* sPhysicalPageMapper; +static TranslationMapPhysicalPageMapper* sKernelPhysicalPageMapper; + + +// Accessor class to reuse the SinglyLinkedListLink of DeferredDeletable for +// X86PagingStructures32Bit. +struct PagingStructuresGetLink { +private: + typedef SinglyLinkedListLink Link; + +public: + inline Link* operator()(X86PagingStructures32Bit* element) const + { + return (Link*)element->GetSinglyLinkedListLink(); + } + + inline const Link* operator()( + const X86PagingStructures32Bit* element) const + { + return (const Link*)element->GetSinglyLinkedListLink(); + } +}; + + +typedef SinglyLinkedList + PagingStructuresList; + + +static PagingStructuresList sPagingStructuresList; +static spinlock sPagingStructuresListLock; + + +#define FIRST_USER_PGDIR_ENT (VADDR_TO_PDENT(USER_BASE)) +#define NUM_USER_PGDIR_ENTS (VADDR_TO_PDENT(ROUNDUP(USER_SIZE, \ + B_PAGE_SIZE * 1024))) +#define FIRST_KERNEL_PGDIR_ENT (VADDR_TO_PDENT(KERNEL_BASE)) +#define NUM_KERNEL_PGDIR_ENTS (VADDR_TO_PDENT(KERNEL_SIZE)) +#define IS_KERNEL_MAP(map) (fPagingStructures->pgdir_phys \ + == sKernelPhysicalPageDirectory) + + +X86PagingStructures32Bit::X86PagingStructures32Bit() +{ +} + + +X86PagingStructures32Bit::~X86PagingStructures32Bit() +{ +} + + +void +X86PagingStructures32Bit::Delete() +{ + // remove from global list + InterruptsSpinLocker locker(sPagingStructuresListLock); + sPagingStructuresList.Remove(this); + locker.Unlock(); + +#if 0 + // this sanity check can be enabled when corruption due to + // overwriting an active page directory is suspected + uint32 activePageDirectory; + read_cr3(activePageDirectory); + if (activePageDirectory == pgdir_phys) + panic("deleting a still active page directory\n"); +#endif + + if (are_interrupts_enabled()) + delete this; + else + deferred_delete(this); +} + + +// #pragma mark - + + +//! TODO: currently assumes this translation map is active +static status_t +early_query(addr_t va, phys_addr_t *_physicalAddress) +{ + if ((sPageHolePageDir[VADDR_TO_PDENT(va)] & X86_PDE_PRESENT) == 0) { + // no pagetable here + return B_ERROR; + } + + page_table_entry* pentry = sPageHole + va / B_PAGE_SIZE; + if ((*pentry & X86_PTE_PRESENT) == 0) { + // page mapping not valid + return B_ERROR; + } + + *_physicalAddress = *pentry & X86_PTE_ADDRESS_MASK; + return B_OK; +} + + +static inline uint32 +memory_type_to_pte_flags(uint32 memoryType) +{ + // ATM we only handle the uncacheable and write-through type explicitly. For + // all other types we rely on the MTRRs to be set up correctly. Since we set + // the default memory type to write-back and since the uncacheable type in + // the PTE overrides any MTRR attribute (though, as per the specs, that is + // not recommended for performance reasons), this reduces the work we + // actually *have* to do with the MTRRs to setting the remaining types + // (usually only write-combining for the frame buffer). + switch (memoryType) { + case B_MTR_UC: + return X86_PTE_CACHING_DISABLED | X86_PTE_WRITE_THROUGH; + + case B_MTR_WC: + // X86_PTE_WRITE_THROUGH would be closer, but the combination with + // MTRR WC is "implementation defined" for Pentium Pro/II. + return 0; + + case B_MTR_WT: + return X86_PTE_WRITE_THROUGH; + + case B_MTR_WP: + case B_MTR_WB: + default: + return 0; + } +} + + +static void +put_page_table_entry_in_pgtable(page_table_entry* entry, + phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType, + bool globalPage) +{ + page_table_entry page = (physicalAddress & X86_PTE_ADDRESS_MASK) + | X86_PTE_PRESENT | (globalPage ? X86_PTE_GLOBAL : 0) + | memory_type_to_pte_flags(memoryType); + + // if the page is user accessible, it's automatically + // accessible in kernel space, too (but with the same + // protection) + if ((attributes & B_USER_PROTECTION) != 0) { + page |= X86_PTE_USER; + if ((attributes & B_WRITE_AREA) != 0) + page |= X86_PTE_WRITABLE; + } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) + page |= X86_PTE_WRITABLE; + + // put it in the page table + *(volatile page_table_entry*)entry = page; +} + + +// #pragma mark - + + +uint32 +i386_translation_map_get_pgdir(VMTranslationMap* map) +{ + return static_cast(map)->PhysicalPageDir(); +} + + +void +x86_update_all_pgdirs(int index, page_directory_entry e) +{ + unsigned int state = disable_interrupts(); + + acquire_spinlock(&sPagingStructuresListLock); + + PagingStructuresList::Iterator it = sPagingStructuresList.GetIterator(); + while (X86PagingStructures32Bit* info = it.Next()) + info->pgdir_virt[index] = e; + + release_spinlock(&sPagingStructuresListLock); + restore_interrupts(state); +} + + +void +x86_put_pgtable_in_pgdir(page_directory_entry *entry, + phys_addr_t pgtablePhysical, uint32 attributes) +{ + *entry = (pgtablePhysical & X86_PDE_ADDRESS_MASK) + | X86_PDE_PRESENT + | X86_PDE_WRITABLE + | X86_PDE_USER; + // TODO: we ignore the attributes of the page table - for compatibility + // with BeOS we allow having user accessible areas in the kernel address + // space. This is currently being used by some drivers, mainly for the + // frame buffer. Our current real time data implementation makes use of + // this fact, too. + // We might want to get rid of this possibility one day, especially if + // we intend to port it to a platform that does not support this. +} + + +void +x86_early_prepare_page_tables(page_table_entry* pageTables, addr_t address, + size_t size) +{ + memset(pageTables, 0, B_PAGE_SIZE * (size / (B_PAGE_SIZE * 1024))); + + // put the array of pgtables directly into the kernel pagedir + // these will be wired and kept mapped into virtual space to be easy to get + // to + { + addr_t virtualTable = (addr_t)pageTables; + + for (size_t i = 0; i < (size / (B_PAGE_SIZE * 1024)); + i++, virtualTable += B_PAGE_SIZE) { + phys_addr_t physicalTable = 0; + early_query(virtualTable, &physicalTable); + page_directory_entry* entry = &sPageHolePageDir[ + (address / (B_PAGE_SIZE * 1024)) + i]; + x86_put_pgtable_in_pgdir(entry, physicalTable, + B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + } + } +} + + +// #pragma mark - VM ops + + +X86VMTranslationMap::X86VMTranslationMap() + : + fPagingStructures(NULL), + fPageMapper(NULL), + fInvalidPagesCount(0) +{ +} + + +X86VMTranslationMap::~X86VMTranslationMap() +{ + if (fPagingStructures == NULL) + return; + + if (fPageMapper != NULL) + fPageMapper->Delete(); + + if (fPagingStructures->pgdir_virt != NULL) { + // cycle through and free all of the user space pgtables + for (uint32 i = VADDR_TO_PDENT(USER_BASE); + i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) { + if ((fPagingStructures->pgdir_virt[i] & X86_PDE_PRESENT) != 0) { + addr_t address = fPagingStructures->pgdir_virt[i] + & X86_PDE_ADDRESS_MASK; + vm_page* page = vm_lookup_page(address / B_PAGE_SIZE); + if (!page) + panic("destroy_tmap: didn't find pgtable page\n"); + DEBUG_PAGE_ACCESS_START(page); + vm_page_set_state(page, PAGE_STATE_FREE); + } + } + } + + fPagingStructures->RemoveReference(); +} + + +status_t +X86VMTranslationMap::Init(bool kernel) +{ + TRACE("X86VMTranslationMap::Init()\n"); + + fPagingStructures = new(std::nothrow) X86PagingStructures32Bit; + if (fPagingStructures == NULL) + return B_NO_MEMORY; + + fPagingStructures->active_on_cpus = 0; + + if (!kernel) { + // user + // allocate a physical page mapper + status_t error = sPhysicalPageMapper + ->CreateTranslationMapPhysicalPageMapper(&fPageMapper); + if (error != B_OK) + return error; + + // allocate a pgdir + fPagingStructures->pgdir_virt = (page_directory_entry *)memalign( + B_PAGE_SIZE, B_PAGE_SIZE); + if (fPagingStructures->pgdir_virt == NULL) + return B_NO_MEMORY; + + phys_addr_t physicalPageDir; + vm_get_page_mapping(VMAddressSpace::KernelID(), + (addr_t)fPagingStructures->pgdir_virt, + &physicalPageDir); + fPagingStructures->pgdir_phys = physicalPageDir; + } else { + // kernel + // get the physical page mapper + fPageMapper = sKernelPhysicalPageMapper; + + // we already know the kernel pgdir mapping + fPagingStructures->pgdir_virt = sKernelVirtualPageDirectory; + fPagingStructures->pgdir_phys = sKernelPhysicalPageDirectory; + } + + // zero out the bottom portion of the new pgdir + memset(fPagingStructures->pgdir_virt + FIRST_USER_PGDIR_ENT, 0, + NUM_USER_PGDIR_ENTS * sizeof(page_directory_entry)); + + // insert this new map into the map list + { + int state = disable_interrupts(); + acquire_spinlock(&sPagingStructuresListLock); + + // copy the top portion of the pgdir from the current one + memcpy(fPagingStructures->pgdir_virt + FIRST_KERNEL_PGDIR_ENT, + sKernelVirtualPageDirectory + FIRST_KERNEL_PGDIR_ENT, + NUM_KERNEL_PGDIR_ENTS * sizeof(page_directory_entry)); + + sPagingStructuresList.Add( + static_cast(fPagingStructures)); + + release_spinlock(&sPagingStructuresListLock); + restore_interrupts(state); + } + + return B_OK; +} + + +/*! Acquires the map's recursive lock, and resets the invalidate pages counter + in case it's the first locking recursion. +*/ +bool +X86VMTranslationMap::Lock() +{ + TRACE("%p->X86VMTranslationMap::Lock()\n", this); + + recursive_lock_lock(&fLock); + if (recursive_lock_get_recursion(&fLock) == 1) { + // we were the first one to grab the lock + TRACE("clearing invalidated page count\n"); + fInvalidPagesCount = 0; + } + + return true; +} + + +/*! Unlocks the map, and, if we are actually losing the recursive lock, + flush all pending changes of this map (ie. flush TLB caches as + needed). +*/ +void +X86VMTranslationMap::Unlock() +{ + TRACE("%p->X86VMTranslationMap::Unlock()\n", this); + + if (recursive_lock_get_recursion(&fLock) == 1) { + // we're about to release it for the last time + X86VMTranslationMap::Flush(); + } + + recursive_lock_unlock(&fLock); +} + + +size_t +X86VMTranslationMap::MaxPagesNeededToMap(addr_t start, addr_t end) const +{ + // If start == 0, the actual base address is not yet known to the caller and + // we shall assume the worst case. + if (start == 0) { + // offset the range so it has the worst possible alignment + start = 1023 * B_PAGE_SIZE; + end += 1023 * B_PAGE_SIZE; + } + + return VADDR_TO_PDENT(end) + 1 - VADDR_TO_PDENT(start); +} + + +status_t +X86VMTranslationMap::Map(addr_t va, phys_addr_t pa, uint32 attributes, + uint32 memoryType, vm_page_reservation* reservation) +{ + TRACE("map_tmap: entry pa 0x%lx va 0x%lx\n", pa, va); + +/* + dprintf("pgdir at 0x%x\n", pgdir); + dprintf("index is %d\n", va / B_PAGE_SIZE / 1024); + dprintf("final at 0x%x\n", &pgdir[va / B_PAGE_SIZE / 1024]); + dprintf("value is 0x%x\n", *(int *)&pgdir[va / B_PAGE_SIZE / 1024]); + dprintf("present bit is %d\n", pgdir[va / B_PAGE_SIZE / 1024].present); + dprintf("addr is %d\n", pgdir[va / B_PAGE_SIZE / 1024].addr); +*/ + page_directory_entry* pd = fPagingStructures->pgdir_virt; + + // check to see if a page table exists for this range + uint32 index = VADDR_TO_PDENT(va); + if ((pd[index] & X86_PDE_PRESENT) == 0) { + phys_addr_t pgtable; + vm_page *page; + + // we need to allocate a pgtable + page = vm_page_allocate_page(reservation, + PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); + + DEBUG_PAGE_ACCESS_END(page); + + pgtable = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE; + + TRACE("map_tmap: asked for free page for pgtable. 0x%lx\n", pgtable); + + // put it in the pgdir + x86_put_pgtable_in_pgdir(&pd[index], pgtable, attributes + | ((attributes & B_USER_PROTECTION) != 0 + ? B_WRITE_AREA : B_KERNEL_WRITE_AREA)); + + // update any other page directories, if it maps kernel space + if (index >= FIRST_KERNEL_PGDIR_ENT + && index < (FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS)) + x86_update_all_pgdirs(index, pd[index]); + + fMapCount++; + } + + // now, fill in the pentry + struct thread* thread = thread_get_current_thread(); + ThreadCPUPinner pinner(thread); + + page_table_entry* pt = fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + index = VADDR_TO_PTENT(va); + + ASSERT_PRINT((pt[index] & X86_PTE_PRESENT) == 0, + "virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx32, va, + pt[index]); + + put_page_table_entry_in_pgtable(&pt[index], pa, attributes, memoryType, + IS_KERNEL_MAP(map)); + + pinner.Unlock(); + + // Note: We don't need to invalidate the TLB for this address, as previously + // the entry was not present and the TLB doesn't cache those entries. + + fMapCount++; + + return 0; +} + + +status_t +X86VMTranslationMap::Unmap(addr_t start, addr_t end) +{ + page_directory_entry *pd = fPagingStructures->pgdir_virt; + + start = ROUNDDOWN(start, B_PAGE_SIZE); + end = ROUNDUP(end, B_PAGE_SIZE); + + TRACE("unmap_tmap: asked to free pages 0x%lx to 0x%lx\n", start, end); + +restart: + if (start >= end) + return B_OK; + + int index = VADDR_TO_PDENT(start); + if ((pd[index] & X86_PDE_PRESENT) == 0) { + // no pagetable here, move the start up to access the next page table + start = ROUNDUP(start + 1, B_PAGE_SIZE * 1024); + if (start == 0) + return B_OK; + goto restart; + } + + struct thread* thread = thread_get_current_thread(); + ThreadCPUPinner pinner(thread); + + page_table_entry* pt = fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + + for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end); + index++, start += B_PAGE_SIZE) { + if ((pt[index] & X86_PTE_PRESENT) == 0) { + // page mapping not valid + continue; + } + + TRACE("unmap_tmap: removing page 0x%lx\n", start); + + page_table_entry oldEntry = clear_page_table_entry_flags(&pt[index], + X86_PTE_PRESENT); + fMapCount--; + + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + // Note, that we only need to invalidate the address, if the + // accessed flags was set, since only then the entry could have been + // in any TLB. + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = start; + + fInvalidPagesCount++; + } + } + + pinner.Unlock(); + + goto restart; +} + + +/*! Caller must have locked the cache of the page to be unmapped. + This object shouldn't be locked. +*/ +status_t +X86VMTranslationMap::UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue) +{ + ASSERT(address % B_PAGE_SIZE == 0); + + page_directory_entry* pd = fPagingStructures->pgdir_virt; + + TRACE("X86VMTranslationMap::UnmapPage(%#" B_PRIxADDR ")\n", address); + + RecursiveLocker locker(fLock); + + int index = VADDR_TO_PDENT(address); + if ((pd[index] & X86_PDE_PRESENT) == 0) + return B_ENTRY_NOT_FOUND; + + ThreadCPUPinner pinner(thread_get_current_thread()); + + page_table_entry* pt = fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + + index = VADDR_TO_PTENT(address); + page_table_entry oldEntry = clear_page_table_entry(&pt[index]); + + pinner.Unlock(); + + if ((oldEntry & X86_PTE_PRESENT) == 0) { + // page mapping not valid + return B_ENTRY_NOT_FOUND; + } + + fMapCount--; + + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + // Note, that we only need to invalidate the address, if the + // accessed flags was set, since only then the entry could have been + // in any TLB. + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = address; + + fInvalidPagesCount++; + + Flush(); + + // NOTE: Between clearing the page table entry and Flush() other + // processors (actually even this processor with another thread of the + // same team) could still access the page in question via their cached + // entry. We can obviously lose a modified flag in this case, with the + // effect that the page looks unmodified (and might thus be recycled), + // but is actually modified. + // In most cases this is harmless, but for vm_remove_all_page_mappings() + // this is actually a problem. + // Interestingly FreeBSD seems to ignore this problem as well + // (cf. pmap_remove_all()), unless I've missed something. + } + + if (area->cache_type == CACHE_TYPE_DEVICE) + return B_OK; + + // get the page + vm_page* page = vm_lookup_page( + (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); + ASSERT(page != NULL); + + // transfer the accessed/dirty flags to the page + if ((oldEntry & X86_PTE_ACCESSED) != 0) + page->accessed = true; + if ((oldEntry & X86_PTE_DIRTY) != 0) + page->modified = true; + + // remove the mapping object/decrement the wired_count of the page + vm_page_mapping* mapping = NULL; + if (area->wiring == B_NO_LOCK) { + vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); + while ((mapping = iterator.Next()) != NULL) { + if (mapping->area == area) { + area->mappings.Remove(mapping); + page->mappings.Remove(mapping); + break; + } + } + + ASSERT(mapping != NULL); + } else + page->wired_count--; + + locker.Unlock(); + + if (page->wired_count == 0 && page->mappings.IsEmpty()) { + atomic_add(&gMappedPagesCount, -1); + + if (updatePageQueue) { + if (page->Cache()->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + } + } + + if (mapping != NULL) { + bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); + object_cache_free(gPageMappingsObjectCache, mapping, + CACHE_DONT_WAIT_FOR_MEMORY + | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0)); + } + + return B_OK; +} + + +void +X86VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size, + bool updatePageQueue) +{ + page_directory_entry* pd = fPagingStructures->pgdir_virt; + + addr_t start = base; + addr_t end = base + size; + + TRACE("X86VMTranslationMap::UnmapPages(%p, %#" B_PRIxADDR ", %#" + B_PRIxADDR ")\n", area, start, end); + + VMAreaMappings queue; + + RecursiveLocker locker(fLock); + + while (start < end) { + int index = VADDR_TO_PDENT(start); + if ((pd[index] & X86_PDE_PRESENT) == 0) { + // no page table here, move the start up to access the next page + // table + start = ROUNDUP(start + 1, B_PAGE_SIZE * 1024); + if (start == 0) + break; + continue; + } + + struct thread* thread = thread_get_current_thread(); + ThreadCPUPinner pinner(thread); + + page_table_entry* pt = fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + + for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end); + index++, start += B_PAGE_SIZE) { + page_table_entry oldEntry = clear_page_table_entry(&pt[index]); + if ((oldEntry & X86_PTE_PRESENT) == 0) + continue; + + fMapCount--; + + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + // Note, that we only need to invalidate the address, if the + // accessed flags was set, since only then the entry could have + // been in any TLB. + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = start; + + fInvalidPagesCount++; + } + + if (area->cache_type != CACHE_TYPE_DEVICE) { + // get the page + vm_page* page = vm_lookup_page( + (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); + ASSERT(page != NULL); + + DEBUG_PAGE_ACCESS_START(page); + + // transfer the accessed/dirty flags to the page + if ((oldEntry & X86_PTE_ACCESSED) != 0) + page->accessed = true; + if ((oldEntry & X86_PTE_DIRTY) != 0) + page->modified = true; + + // remove the mapping object/decrement the wired_count of the + // page + if (area->wiring == B_NO_LOCK) { + vm_page_mapping* mapping = NULL; + vm_page_mappings::Iterator iterator + = page->mappings.GetIterator(); + while ((mapping = iterator.Next()) != NULL) { + if (mapping->area == area) + break; + } + + ASSERT(mapping != NULL); + + area->mappings.Remove(mapping); + page->mappings.Remove(mapping); + queue.Add(mapping); + } else + page->wired_count--; + + if (page->wired_count == 0 && page->mappings.IsEmpty()) { + atomic_add(&gMappedPagesCount, -1); + + if (updatePageQueue) { + if (page->Cache()->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + } + } + + DEBUG_PAGE_ACCESS_END(page); + } + } + + Flush(); + // flush explicitly, since we directly use the lock + + pinner.Unlock(); + } + + // TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not + // really critical here, as in all cases this method is used, the unmapped + // area range is unmapped for good (resized/cut) and the pages will likely + // be freed. + + locker.Unlock(); + + // free removed mappings + bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); + uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY + | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); + while (vm_page_mapping* mapping = queue.RemoveHead()) + object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); +} + + +void +X86VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace, + bool ignoreTopCachePageFlags) +{ + if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { + X86VMTranslationMap::UnmapPages(area, area->Base(), area->Size(), true); + return; + } + + bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; + + page_directory_entry* pd = fPagingStructures->pgdir_virt; + + RecursiveLocker locker(fLock); + + VMAreaMappings mappings; + mappings.MoveFrom(&area->mappings); + + for (VMAreaMappings::Iterator it = mappings.GetIterator(); + vm_page_mapping* mapping = it.Next();) { + vm_page* page = mapping->page; + page->mappings.Remove(mapping); + + VMCache* cache = page->Cache(); + + bool pageFullyUnmapped = false; + if (page->wired_count == 0 && page->mappings.IsEmpty()) { + atomic_add(&gMappedPagesCount, -1); + pageFullyUnmapped = true; + } + + if (unmapPages || cache != area->cache) { + addr_t address = area->Base() + + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); + + int index = VADDR_TO_PDENT(address); + if ((pd[index] & X86_PDE_PRESENT) == 0) { + panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " + "has no page dir entry", page, area, address); + continue; + } + + ThreadCPUPinner pinner(thread_get_current_thread()); + + page_table_entry* pt = fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + page_table_entry oldEntry = clear_page_table_entry( + &pt[VADDR_TO_PTENT(address)]); + + pinner.Unlock(); + + if ((oldEntry & X86_PTE_PRESENT) == 0) { + panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " + "has no page table entry", page, area, address); + continue; + } + + // transfer the accessed/dirty flags to the page and invalidate + // the mapping, if necessary + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + page->accessed = true; + + if (!deletingAddressSpace) { + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = address; + + fInvalidPagesCount++; + } + } + + if ((oldEntry & X86_PTE_DIRTY) != 0) + page->modified = true; + + if (pageFullyUnmapped) { + DEBUG_PAGE_ACCESS_START(page); + + if (cache->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + + DEBUG_PAGE_ACCESS_END(page); + } + } + + fMapCount--; + } + + Flush(); + // flush explicitely, since we directly use the lock + + locker.Unlock(); + + bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); + uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY + | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); + while (vm_page_mapping* mapping = mappings.RemoveHead()) + object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); +} + + +status_t +X86VMTranslationMap::Query(addr_t va, phys_addr_t *_physical, uint32 *_flags) +{ + // default the flags to not present + *_flags = 0; + *_physical = 0; + + int index = VADDR_TO_PDENT(va); + page_directory_entry *pd = fPagingStructures->pgdir_virt; + if ((pd[index] & X86_PDE_PRESENT) == 0) { + // no pagetable here + return B_OK; + } + + struct thread* thread = thread_get_current_thread(); + ThreadCPUPinner pinner(thread); + + page_table_entry* pt = fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + page_table_entry entry = pt[VADDR_TO_PTENT(va)]; + + *_physical = entry & X86_PDE_ADDRESS_MASK; + + // read in the page state flags + if ((entry & X86_PTE_USER) != 0) { + *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0) + | B_READ_AREA; + } + + *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0) + | B_KERNEL_READ_AREA + | ((entry & X86_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0) + | ((entry & X86_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0) + | ((entry & X86_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0); + + pinner.Unlock(); + + TRACE("query_tmap: returning pa 0x%lx for va 0x%lx\n", *_physical, va); + + return B_OK; +} + + +status_t +X86VMTranslationMap::QueryInterrupt(addr_t va, phys_addr_t *_physical, + uint32 *_flags) +{ + *_flags = 0; + *_physical = 0; + + int index = VADDR_TO_PDENT(va); + page_directory_entry* pd = fPagingStructures->pgdir_virt; + if ((pd[index] & X86_PDE_PRESENT) == 0) { + // no pagetable here + return B_OK; + } + + // map page table entry + page_table_entry* pt = sPhysicalPageMapper->InterruptGetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + page_table_entry entry = pt[VADDR_TO_PTENT(va)]; + + *_physical = entry & X86_PDE_ADDRESS_MASK; + + // read in the page state flags + if ((entry & X86_PTE_USER) != 0) { + *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0) + | B_READ_AREA; + } + + *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0) + | B_KERNEL_READ_AREA + | ((entry & X86_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0) + | ((entry & X86_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0) + | ((entry & X86_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0); + + return B_OK; +} + + +addr_t +X86VMTranslationMap::MappedSize() const +{ + return fMapCount; +} + + +status_t +X86VMTranslationMap::Protect(addr_t start, addr_t end, uint32 attributes, + uint32 memoryType) +{ + page_directory_entry *pd = fPagingStructures->pgdir_virt; + + start = ROUNDDOWN(start, B_PAGE_SIZE); + + TRACE("protect_tmap: pages 0x%lx to 0x%lx, attributes %lx\n", start, end, + attributes); + + // compute protection flags + uint32 newProtectionFlags = 0; + if ((attributes & B_USER_PROTECTION) != 0) { + newProtectionFlags = X86_PTE_USER; + if ((attributes & B_WRITE_AREA) != 0) + newProtectionFlags |= X86_PTE_WRITABLE; + } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) + newProtectionFlags = X86_PTE_WRITABLE; + +restart: + if (start >= end) + return B_OK; + + int index = VADDR_TO_PDENT(start); + if ((pd[index] & X86_PDE_PRESENT) == 0) { + // no pagetable here, move the start up to access the next page table + start = ROUNDUP(start + 1, B_PAGE_SIZE * 1024); + if (start == 0) + return B_OK; + goto restart; + } + + struct thread* thread = thread_get_current_thread(); + ThreadCPUPinner pinner(thread); + + page_table_entry* pt = fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + + for (index = VADDR_TO_PTENT(start); index < 1024 && start < end; + index++, start += B_PAGE_SIZE) { + page_table_entry entry = pt[index]; + if ((entry & X86_PTE_PRESENT) == 0) { + // page mapping not valid + continue; + } + + TRACE("protect_tmap: protect page 0x%lx\n", start); + + // set the new protection flags -- we want to do that atomically, + // without changing the accessed or dirty flag + page_table_entry oldEntry; + while (true) { + oldEntry = test_and_set_page_table_entry(&pt[index], + (entry & ~(X86_PTE_PROTECTION_MASK | X86_PTE_MEMORY_TYPE_MASK)) + | newProtectionFlags | memory_type_to_pte_flags(memoryType), + entry); + if (oldEntry == entry) + break; + entry = oldEntry; + } + + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + // Note, that we only need to invalidate the address, if the + // accessed flag was set, since only then the entry could have been + // in any TLB. + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = start; + + fInvalidPagesCount++; + } + } + + pinner.Unlock(); + + goto restart; +} + + +status_t +X86VMTranslationMap::ClearFlags(addr_t va, uint32 flags) +{ + int index = VADDR_TO_PDENT(va); + page_directory_entry* pd = fPagingStructures->pgdir_virt; + if ((pd[index] & X86_PDE_PRESENT) == 0) { + // no pagetable here + return B_OK; + } + + uint32 flagsToClear = ((flags & PAGE_MODIFIED) ? X86_PTE_DIRTY : 0) + | ((flags & PAGE_ACCESSED) ? X86_PTE_ACCESSED : 0); + + struct thread* thread = thread_get_current_thread(); + ThreadCPUPinner pinner(thread); + + page_table_entry* pt = fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + index = VADDR_TO_PTENT(va); + + // clear out the flags we've been requested to clear + page_table_entry oldEntry + = clear_page_table_entry_flags(&pt[index], flagsToClear); + + pinner.Unlock(); + + if ((oldEntry & flagsToClear) != 0) { + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = va; + + fInvalidPagesCount++; + } + + return B_OK; +} + + +bool +X86VMTranslationMap::ClearAccessedAndModified(VMArea* area, addr_t address, + bool unmapIfUnaccessed, bool& _modified) +{ + ASSERT(address % B_PAGE_SIZE == 0); + + page_directory_entry* pd = fPagingStructures->pgdir_virt; + + TRACE("X86VMTranslationMap::ClearAccessedAndModified(%#" B_PRIxADDR ")\n", + address); + + RecursiveLocker locker(fLock); + + int index = VADDR_TO_PDENT(address); + if ((pd[index] & X86_PDE_PRESENT) == 0) + return false; + + ThreadCPUPinner pinner(thread_get_current_thread()); + + page_table_entry* pt = fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + + index = VADDR_TO_PTENT(address); + + // perform the deed + page_table_entry oldEntry; + + if (unmapIfUnaccessed) { + while (true) { + oldEntry = pt[index]; + if ((oldEntry & X86_PTE_PRESENT) == 0) { + // page mapping not valid + return false; + } + + if (oldEntry & X86_PTE_ACCESSED) { + // page was accessed -- just clear the flags + oldEntry = clear_page_table_entry_flags(&pt[index], + X86_PTE_ACCESSED | X86_PTE_DIRTY); + break; + } + + // page hasn't been accessed -- unmap it + if (test_and_set_page_table_entry(&pt[index], 0, oldEntry) + == oldEntry) { + break; + } + + // something changed -- check again + } + } else { + oldEntry = clear_page_table_entry_flags(&pt[index], + X86_PTE_ACCESSED | X86_PTE_DIRTY); + } + + pinner.Unlock(); + + _modified = (oldEntry & X86_PTE_DIRTY) != 0; + + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + // Note, that we only need to invalidate the address, if the + // accessed flags was set, since only then the entry could have been + // in any TLB. + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = address; + + fInvalidPagesCount++; + + Flush(); + + return true; + } + + if (!unmapIfUnaccessed) + return false; + + // We have unmapped the address. Do the "high level" stuff. + + fMapCount--; + + if (area->cache_type == CACHE_TYPE_DEVICE) + return false; + + // get the page + vm_page* page = vm_lookup_page( + (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); + ASSERT(page != NULL); + + // remove the mapping object/decrement the wired_count of the page + vm_page_mapping* mapping = NULL; + if (area->wiring == B_NO_LOCK) { + vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); + while ((mapping = iterator.Next()) != NULL) { + if (mapping->area == area) { + area->mappings.Remove(mapping); + page->mappings.Remove(mapping); + break; + } + } + + ASSERT(mapping != NULL); + } else + page->wired_count--; + + locker.Unlock(); + + if (page->wired_count == 0 && page->mappings.IsEmpty()) + atomic_add(&gMappedPagesCount, -1); + + if (mapping != NULL) { + object_cache_free(gPageMappingsObjectCache, mapping, + CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); + // Since this is called by the page daemon, we never want to lock + // the kernel address space. + } + + return false; +} + + +void +X86VMTranslationMap::Flush() +{ + if (fInvalidPagesCount <= 0) + return; + + struct thread* thread = thread_get_current_thread(); + thread_pin_to_current_cpu(thread); + + if (fInvalidPagesCount > PAGE_INVALIDATE_CACHE_SIZE) { + // invalidate all pages + TRACE("flush_tmap: %d pages to invalidate, invalidate all\n", + fInvalidPagesCount); + + if (IS_KERNEL_MAP(map)) { + arch_cpu_global_TLB_invalidate(); + smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, + NULL, SMP_MSG_FLAG_SYNC); + } else { + cpu_status state = disable_interrupts(); + arch_cpu_user_TLB_invalidate(); + restore_interrupts(state); + + int cpu = smp_get_current_cpu(); + uint32 cpuMask = fPagingStructures->active_on_cpus + & ~((uint32)1 << cpu); + if (cpuMask != 0) { + smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES, + 0, 0, 0, NULL, SMP_MSG_FLAG_SYNC); + } + } + } else { + TRACE("flush_tmap: %d pages to invalidate, invalidate list\n", + fInvalidPagesCount); + + arch_cpu_invalidate_TLB_list(fInvalidPages, fInvalidPagesCount); + + if (IS_KERNEL_MAP(map)) { + smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, + (uint32)fInvalidPages, fInvalidPagesCount, 0, NULL, + SMP_MSG_FLAG_SYNC); + } else { + int cpu = smp_get_current_cpu(); + uint32 cpuMask = fPagingStructures->active_on_cpus + & ~((uint32)1 << cpu); + if (cpuMask != 0) { + smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST, + (uint32)fInvalidPages, fInvalidPagesCount, 0, NULL, + SMP_MSG_FLAG_SYNC); + } + } + } + fInvalidPagesCount = 0; + + thread_unpin_from_current_cpu(thread); +} + + +// #pragma mark - X86PagingMethod32Bit + + +X86PagingMethod32Bit::X86PagingMethod32Bit() +{ +} + + +X86PagingMethod32Bit::~X86PagingMethod32Bit() +{ +} + + +/*static*/ X86PagingMethod* +X86PagingMethod32Bit::Create() +{ + return new(&sMethod) X86PagingMethod32Bit; +} + + +status_t +X86PagingMethod32Bit::Init(kernel_args* args, + VMPhysicalPageMapper** _physicalPageMapper) +{ + TRACE("vm_translation_map_init: entry\n"); + + // page hole set up in stage2 + sPageHole = (page_table_entry *)args->arch_args.page_hole; + // calculate where the pgdir would be + sPageHolePageDir = (page_directory_entry*) + (((addr_t)args->arch_args.page_hole) + + (B_PAGE_SIZE * 1024 - B_PAGE_SIZE)); + // clear out the bottom 2 GB, unmap everything + memset(sPageHolePageDir + FIRST_USER_PGDIR_ENT, 0, + sizeof(page_directory_entry) * NUM_USER_PGDIR_ENTS); + + sKernelPhysicalPageDirectory = args->arch_args.phys_pgdir; + sKernelVirtualPageDirectory = (page_directory_entry*) + args->arch_args.vir_pgdir; + +#ifdef TRACE_X86_PAGING_METHOD_32_BIT + TRACE("page hole: %p, page dir: %p\n", sPageHole, sPageHolePageDir); + TRACE("page dir: %p (physical: %#" B_PRIx32 ")\n", + sKernelVirtualPageDirectory, sKernelPhysicalPageDirectory); +#endif + + B_INITIALIZE_SPINLOCK(&sPagingStructuresListLock); + new (&sPagingStructuresList) PagingStructuresList; + + large_memory_physical_page_ops_init(args, sPhysicalPageMapper, + sKernelPhysicalPageMapper); + // TODO: Select the best page mapper! + + // enable global page feature if available + if (x86_check_feature(IA32_FEATURE_PGE, FEATURE_COMMON)) { + // this prevents kernel pages from being flushed from TLB on + // context-switch + x86_write_cr4(x86_read_cr4() | IA32_CR4_GLOBAL_PAGES); + } + + TRACE("vm_translation_map_init: done\n"); + + *_physicalPageMapper = sPhysicalPageMapper; + return B_OK; +} + + +status_t +X86PagingMethod32Bit::InitPostArea(kernel_args* args) +{ + // now that the vm is initialized, create an area that represents + // the page hole + void *temp; + status_t error; + area_id area; + + // unmap the page hole hack we were using before + sKernelVirtualPageDirectory[1023] = 0; + sPageHolePageDir = NULL; + sPageHole = NULL; + + temp = (void *)sKernelVirtualPageDirectory; + area = create_area("kernel_pgdir", &temp, B_EXACT_ADDRESS, B_PAGE_SIZE, + B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + if (area < B_OK) + return area; + + error = sPhysicalPageMapper->InitPostArea(args); + if (error != B_OK) + return error; + + return B_OK; +} + + +status_t +X86PagingMethod32Bit::CreateTranslationMap(bool kernel, VMTranslationMap** _map) +{ + X86VMTranslationMap* map = new(std::nothrow) X86VMTranslationMap; + if (map == NULL) + return B_NO_MEMORY; + + status_t error = map->Init(kernel); + if (error != B_OK) { + delete map; + return error; + } + + *_map = map; + return B_OK; +} + + +status_t +X86PagingMethod32Bit::MapEarly(kernel_args* args, addr_t virtualAddress, + phys_addr_t physicalAddress, uint8 attributes, + phys_addr_t (*get_free_page)(kernel_args*)) +{ + // XXX horrible back door to map a page quickly regardless of translation + // map object, etc. used only during VM setup. + // uses a 'page hole' set up in the stage 2 bootloader. The page hole is + // created by pointing one of the pgdir entries back at itself, effectively + // mapping the contents of all of the 4MB of pagetables into a 4 MB region. + // It's only used here, and is later unmapped. + + // check to see if a page table exists for this range + int index = VADDR_TO_PDENT(virtualAddress); + if ((sPageHolePageDir[index] & X86_PDE_PRESENT) == 0) { + phys_addr_t pgtable; + page_directory_entry *e; + // we need to allocate a pgtable + pgtable = get_free_page(args); + // pgtable is in pages, convert to physical address + pgtable *= B_PAGE_SIZE; + + TRACE("X86PagingMethod32Bit::MapEarly(): asked for free page for " + "pgtable. %#" B_PRIxPHYSADDR "\n", pgtable); + + // put it in the pgdir + e = &sPageHolePageDir[index]; + x86_put_pgtable_in_pgdir(e, pgtable, attributes); + + // zero it out in it's new mapping + memset((unsigned int*)((addr_t)sPageHole + + (virtualAddress / B_PAGE_SIZE / 1024) * B_PAGE_SIZE), + 0, B_PAGE_SIZE); + } + + ASSERT_PRINT( + (sPageHole[virtualAddress / B_PAGE_SIZE] & X86_PTE_PRESENT) == 0, + "virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx32 + ", existing pte: %#" B_PRIx32, virtualAddress, sPageHolePageDir[index], + sPageHole[virtualAddress / B_PAGE_SIZE]); + + // now, fill in the pentry + put_page_table_entry_in_pgtable(sPageHole + virtualAddress / B_PAGE_SIZE, + physicalAddress, attributes, 0, IS_KERNEL_ADDRESS(virtualAddress)); + + return B_OK; +} + + +bool +X86PagingMethod32Bit::IsKernelPageAccessible(addr_t virtualAddress, + uint32 protection) +{ + // We only trust the kernel team's page directory. So switch to it first. + // Always set it to make sure the TLBs don't contain obsolete data. + uint32 physicalPageDirectory; + read_cr3(physicalPageDirectory); + write_cr3(sKernelPhysicalPageDirectory); + + // get the page directory entry for the address + page_directory_entry pageDirectoryEntry; + uint32 index = VADDR_TO_PDENT(virtualAddress); + + if (physicalPageDirectory == sKernelPhysicalPageDirectory) { + pageDirectoryEntry = sKernelVirtualPageDirectory[index]; + } else if (sPhysicalPageMapper != NULL) { + // map the original page directory and get the entry + void* handle; + addr_t virtualPageDirectory; + status_t error = sPhysicalPageMapper->GetPageDebug( + physicalPageDirectory, &virtualPageDirectory, &handle); + if (error == B_OK) { + pageDirectoryEntry + = ((page_directory_entry*)virtualPageDirectory)[index]; + sPhysicalPageMapper->PutPageDebug(virtualPageDirectory, + handle); + } else + pageDirectoryEntry = 0; + } else + pageDirectoryEntry = 0; + + // map the page table and get the entry + page_table_entry pageTableEntry; + index = VADDR_TO_PTENT(virtualAddress); + + if ((pageDirectoryEntry & X86_PDE_PRESENT) != 0 + && sPhysicalPageMapper != NULL) { + void* handle; + addr_t virtualPageTable; + status_t error = sPhysicalPageMapper->GetPageDebug( + pageDirectoryEntry & X86_PDE_ADDRESS_MASK, &virtualPageTable, + &handle); + if (error == B_OK) { + pageTableEntry = ((page_table_entry*)virtualPageTable)[index]; + sPhysicalPageMapper->PutPageDebug(virtualPageTable, handle); + } else + pageTableEntry = 0; + } else + pageTableEntry = 0; + + // switch back to the original page directory + if (physicalPageDirectory != sKernelPhysicalPageDirectory) + write_cr3(physicalPageDirectory); + + if ((pageTableEntry & X86_PTE_PRESENT) == 0) + return false; + + // present means kernel-readable, so check for writable + return (protection & B_KERNEL_WRITE_AREA) == 0 + || (pageTableEntry & X86_PTE_WRITABLE) != 0; +} diff --git a/src/system/kernel/arch/x86/X86PagingMethod32Bit.h b/src/system/kernel/arch/x86/X86PagingMethod32Bit.h new file mode 100644 index 0000000000..8bd7405f94 --- /dev/null +++ b/src/system/kernel/arch/x86/X86PagingMethod32Bit.h @@ -0,0 +1,46 @@ +/* + * Copyright 2010, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef KERNEL_ARCH_X86_X86_PAGING_METHOD_32_BIT_H +#define KERNEL_ARCH_X86_X86_PAGING_METHOD_32_BIT_H + + +#include "x86_paging.h" +#include "X86PagingMethod.h" + + +struct X86PagingStructures32Bit : X86PagingStructures { + X86PagingStructures32Bit(); + virtual ~X86PagingStructures32Bit(); + + virtual void Delete(); +}; + + +class X86PagingMethod32Bit : public X86PagingMethod { +public: + X86PagingMethod32Bit(); + virtual ~X86PagingMethod32Bit(); + + static X86PagingMethod* Create(); + + virtual status_t Init(kernel_args* args, + VMPhysicalPageMapper** _physicalPageMapper); + virtual status_t InitPostArea(kernel_args* args); + + virtual status_t CreateTranslationMap(bool kernel, + VMTranslationMap** _map); + + virtual status_t MapEarly(kernel_args* args, + addr_t virtualAddress, + phys_addr_t physicalAddress, + uint8 attributes, + phys_addr_t (*get_free_page)(kernel_args*)); + + virtual bool IsKernelPageAccessible(addr_t virtualAddress, + uint32 protection); +}; + + +#endif // KERNEL_ARCH_X86_X86_PAGING_METHOD_32_BIT_H diff --git a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp index 66e820a306..ab6cb66025 100644 --- a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp +++ b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp @@ -10,27 +10,7 @@ #include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "x86_paging.h" -#include "x86_physical_page_mapper.h" -#include "X86VMTranslationMap.h" +#include "X86PagingMethod32Bit.h" //#define TRACE_VM_TMAP @@ -41,52 +21,10 @@ #endif -static page_table_entry *sPageHole = NULL; -static page_directory_entry *sPageHolePageDir = NULL; -static uint32 sKernelPhysicalPageDirectory = 0; -static page_directory_entry *sKernelVirtualPageDirectory = NULL; - -static X86PhysicalPageMapper* sPhysicalPageMapper; -static TranslationMapPhysicalPageMapper* sKernelPhysicalPageMapper; +static X86PagingMethod* sPagingMethod; -// Accessor class to reuse the SinglyLinkedListLink of DeferredDeletable for -// X86PagingStructures. -struct PagingStructuresGetLink { -private: - typedef SinglyLinkedListLink Link; - -public: - inline Link* operator()(X86PagingStructures* element) const - { - return (Link*)element->GetSinglyLinkedListLink(); - } - - inline const Link* operator()( - const X86PagingStructures* element) const - { - return (const Link*)element->GetSinglyLinkedListLink(); - } - -}; - - -typedef SinglyLinkedList - PagingStructuresList; - - -static PagingStructuresList sPagingStructuresList; -static spinlock sPagingStructuresListLock; - -#define CHATTY_TMAP 0 - -#define FIRST_USER_PGDIR_ENT (VADDR_TO_PDENT(USER_BASE)) -#define NUM_USER_PGDIR_ENTS (VADDR_TO_PDENT(ROUNDUP(USER_SIZE, \ - B_PAGE_SIZE * 1024))) -#define FIRST_KERNEL_PGDIR_ENT (VADDR_TO_PDENT(KERNEL_BASE)) -#define NUM_KERNEL_PGDIR_ENTS (VADDR_TO_PDENT(KERNEL_SIZE)) -#define IS_KERNEL_MAP(map) (fPagingStructures->pgdir_phys \ - == sKernelPhysicalPageDirectory) +// #pragma mark - X86PagingStructures X86PagingStructures::X86PagingStructures() @@ -104,1202 +42,13 @@ X86PagingStructures::~X86PagingStructures() } -void -X86PagingStructures::Delete() -{ - // remove from global list - InterruptsSpinLocker locker(sPagingStructuresListLock); - sPagingStructuresList.Remove(this); - locker.Unlock(); - -#if 0 - // this sanity check can be enabled when corruption due to - // overwriting an active page directory is suspected - uint32 activePageDirectory; - read_cr3(activePageDirectory); - if (activePageDirectory == pgdir_phys) - panic("deleting a still active page directory\n"); -#endif - - if (are_interrupts_enabled()) - delete this; - else - deferred_delete(this); -} - - -// #pragma mark - - - -//! TODO: currently assumes this translation map is active -static status_t -early_query(addr_t va, phys_addr_t *_physicalAddress) -{ - if ((sPageHolePageDir[VADDR_TO_PDENT(va)] & X86_PDE_PRESENT) == 0) { - // no pagetable here - return B_ERROR; - } - - page_table_entry* pentry = sPageHole + va / B_PAGE_SIZE; - if ((*pentry & X86_PTE_PRESENT) == 0) { - // page mapping not valid - return B_ERROR; - } - - *_physicalAddress = *pentry & X86_PTE_ADDRESS_MASK; - return B_OK; -} - - -static inline uint32 -memory_type_to_pte_flags(uint32 memoryType) -{ - // ATM we only handle the uncacheable and write-through type explicitly. For - // all other types we rely on the MTRRs to be set up correctly. Since we set - // the default memory type to write-back and since the uncacheable type in - // the PTE overrides any MTRR attribute (though, as per the specs, that is - // not recommended for performance reasons), this reduces the work we - // actually *have* to do with the MTRRs to setting the remaining types - // (usually only write-combining for the frame buffer). - switch (memoryType) { - case B_MTR_UC: - return X86_PTE_CACHING_DISABLED | X86_PTE_WRITE_THROUGH; - - case B_MTR_WC: - // X86_PTE_WRITE_THROUGH would be closer, but the combination with - // MTRR WC is "implementation defined" for Pentium Pro/II. - return 0; - - case B_MTR_WT: - return X86_PTE_WRITE_THROUGH; - - case B_MTR_WP: - case B_MTR_WB: - default: - return 0; - } -} - - -static void -put_page_table_entry_in_pgtable(page_table_entry* entry, - phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType, - bool globalPage) -{ - page_table_entry page = (physicalAddress & X86_PTE_ADDRESS_MASK) - | X86_PTE_PRESENT | (globalPage ? X86_PTE_GLOBAL : 0) - | memory_type_to_pte_flags(memoryType); - - // if the page is user accessible, it's automatically - // accessible in kernel space, too (but with the same - // protection) - if ((attributes & B_USER_PROTECTION) != 0) { - page |= X86_PTE_USER; - if ((attributes & B_WRITE_AREA) != 0) - page |= X86_PTE_WRITABLE; - } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) - page |= X86_PTE_WRITABLE; - - // put it in the page table - *(volatile page_table_entry*)entry = page; -} - - -// #pragma mark - - - -uint32 -i386_translation_map_get_pgdir(VMTranslationMap* map) -{ - return static_cast(map)->PhysicalPageDir(); -} - - -void -x86_update_all_pgdirs(int index, page_directory_entry e) -{ - unsigned int state = disable_interrupts(); - - acquire_spinlock(&sPagingStructuresListLock); - - PagingStructuresList::Iterator it = sPagingStructuresList.GetIterator(); - while (X86PagingStructures* info = it.Next()) - info->pgdir_virt[index] = e; - - release_spinlock(&sPagingStructuresListLock); - restore_interrupts(state); -} - - -void -x86_put_pgtable_in_pgdir(page_directory_entry *entry, - phys_addr_t pgtablePhysical, uint32 attributes) -{ - *entry = (pgtablePhysical & X86_PDE_ADDRESS_MASK) - | X86_PDE_PRESENT - | X86_PDE_WRITABLE - | X86_PDE_USER; - // TODO: we ignore the attributes of the page table - for compatibility - // with BeOS we allow having user accessible areas in the kernel address - // space. This is currently being used by some drivers, mainly for the - // frame buffer. Our current real time data implementation makes use of - // this fact, too. - // We might want to get rid of this possibility one day, especially if - // we intend to port it to a platform that does not support this. -} - - -void -x86_early_prepare_page_tables(page_table_entry* pageTables, addr_t address, - size_t size) -{ - memset(pageTables, 0, B_PAGE_SIZE * (size / (B_PAGE_SIZE * 1024))); - - // put the array of pgtables directly into the kernel pagedir - // these will be wired and kept mapped into virtual space to be easy to get - // to - { - addr_t virtualTable = (addr_t)pageTables; - - for (size_t i = 0; i < (size / (B_PAGE_SIZE * 1024)); - i++, virtualTable += B_PAGE_SIZE) { - phys_addr_t physicalTable = 0; - early_query(virtualTable, &physicalTable); - page_directory_entry* entry = &sPageHolePageDir[ - (address / (B_PAGE_SIZE * 1024)) + i]; - x86_put_pgtable_in_pgdir(entry, physicalTable, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - } - } -} - - -// #pragma mark - VM ops - - -X86VMTranslationMap::X86VMTranslationMap() - : - fPagingStructures(NULL), - fPageMapper(NULL), - fInvalidPagesCount(0) -{ -} - - -X86VMTranslationMap::~X86VMTranslationMap() -{ - if (fPagingStructures == NULL) - return; - - if (fPageMapper != NULL) - fPageMapper->Delete(); - - if (fPagingStructures->pgdir_virt != NULL) { - // cycle through and free all of the user space pgtables - for (uint32 i = VADDR_TO_PDENT(USER_BASE); - i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) { - if ((fPagingStructures->pgdir_virt[i] & X86_PDE_PRESENT) != 0) { - addr_t address = fPagingStructures->pgdir_virt[i] - & X86_PDE_ADDRESS_MASK; - vm_page* page = vm_lookup_page(address / B_PAGE_SIZE); - if (!page) - panic("destroy_tmap: didn't find pgtable page\n"); - DEBUG_PAGE_ACCESS_START(page); - vm_page_set_state(page, PAGE_STATE_FREE); - } - } - } - - fPagingStructures->RemoveReference(); -} - - -status_t -X86VMTranslationMap::Init(bool kernel) -{ - TRACE("X86VMTranslationMap::Init()\n"); - - fPagingStructures = new(std::nothrow) X86PagingStructures; - if (fPagingStructures == NULL) - return B_NO_MEMORY; - - fPagingStructures->active_on_cpus = 0; - - if (!kernel) { - // user - // allocate a physical page mapper - status_t error = sPhysicalPageMapper - ->CreateTranslationMapPhysicalPageMapper(&fPageMapper); - if (error != B_OK) - return error; - - // allocate a pgdir - fPagingStructures->pgdir_virt = (page_directory_entry *)memalign( - B_PAGE_SIZE, B_PAGE_SIZE); - if (fPagingStructures->pgdir_virt == NULL) - return B_NO_MEMORY; - - phys_addr_t physicalPageDir; - vm_get_page_mapping(VMAddressSpace::KernelID(), - (addr_t)fPagingStructures->pgdir_virt, - &physicalPageDir); - fPagingStructures->pgdir_phys = physicalPageDir; - } else { - // kernel - // get the physical page mapper - fPageMapper = sKernelPhysicalPageMapper; - - // we already know the kernel pgdir mapping - fPagingStructures->pgdir_virt = sKernelVirtualPageDirectory; - fPagingStructures->pgdir_phys = sKernelPhysicalPageDirectory; - } - - // zero out the bottom portion of the new pgdir - memset(fPagingStructures->pgdir_virt + FIRST_USER_PGDIR_ENT, 0, - NUM_USER_PGDIR_ENTS * sizeof(page_directory_entry)); - - // insert this new map into the map list - { - int state = disable_interrupts(); - acquire_spinlock(&sPagingStructuresListLock); - - // copy the top portion of the pgdir from the current one - memcpy(fPagingStructures->pgdir_virt + FIRST_KERNEL_PGDIR_ENT, - sKernelVirtualPageDirectory + FIRST_KERNEL_PGDIR_ENT, - NUM_KERNEL_PGDIR_ENTS * sizeof(page_directory_entry)); - - sPagingStructuresList.Add(fPagingStructures); - - release_spinlock(&sPagingStructuresListLock); - restore_interrupts(state); - } - - return B_OK; -} - - -/*! Acquires the map's recursive lock, and resets the invalidate pages counter - in case it's the first locking recursion. -*/ -bool -X86VMTranslationMap::Lock() -{ - TRACE("%p->X86VMTranslationMap::Lock()\n", this); - - recursive_lock_lock(&fLock); - if (recursive_lock_get_recursion(&fLock) == 1) { - // we were the first one to grab the lock - TRACE("clearing invalidated page count\n"); - fInvalidPagesCount = 0; - } - - return true; -} - - -/*! Unlocks the map, and, if we'll actually losing the recursive lock, - flush all pending changes of this map (ie. flush TLB caches as - needed). -*/ -void -X86VMTranslationMap::Unlock() -{ - TRACE("%p->X86VMTranslationMap::Unlock()\n", this); - - if (recursive_lock_get_recursion(&fLock) == 1) { - // we're about to release it for the last time - X86VMTranslationMap::Flush(); - } - - recursive_lock_unlock(&fLock); -} - - -size_t -X86VMTranslationMap::MaxPagesNeededToMap(addr_t start, addr_t end) const -{ - // If start == 0, the actual base address is not yet known to the caller and - // we shall assume the worst case. - if (start == 0) { - // offset the range so it has the worst possible alignment - start = 1023 * B_PAGE_SIZE; - end += 1023 * B_PAGE_SIZE; - } - - return VADDR_TO_PDENT(end) + 1 - VADDR_TO_PDENT(start); -} - - -status_t -X86VMTranslationMap::Map(addr_t va, phys_addr_t pa, uint32 attributes, - uint32 memoryType, vm_page_reservation* reservation) -{ - TRACE("map_tmap: entry pa 0x%lx va 0x%lx\n", pa, va); - -/* - dprintf("pgdir at 0x%x\n", pgdir); - dprintf("index is %d\n", va / B_PAGE_SIZE / 1024); - dprintf("final at 0x%x\n", &pgdir[va / B_PAGE_SIZE / 1024]); - dprintf("value is 0x%x\n", *(int *)&pgdir[va / B_PAGE_SIZE / 1024]); - dprintf("present bit is %d\n", pgdir[va / B_PAGE_SIZE / 1024].present); - dprintf("addr is %d\n", pgdir[va / B_PAGE_SIZE / 1024].addr); -*/ - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - // check to see if a page table exists for this range - uint32 index = VADDR_TO_PDENT(va); - if ((pd[index] & X86_PDE_PRESENT) == 0) { - phys_addr_t pgtable; - vm_page *page; - - // we need to allocate a pgtable - page = vm_page_allocate_page(reservation, - PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); - - DEBUG_PAGE_ACCESS_END(page); - - pgtable = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE; - - TRACE("map_tmap: asked for free page for pgtable. 0x%lx\n", pgtable); - - // put it in the pgdir - x86_put_pgtable_in_pgdir(&pd[index], pgtable, attributes - | ((attributes & B_USER_PROTECTION) != 0 - ? B_WRITE_AREA : B_KERNEL_WRITE_AREA)); - - // update any other page directories, if it maps kernel space - if (index >= FIRST_KERNEL_PGDIR_ENT - && index < (FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS)) - x86_update_all_pgdirs(index, pd[index]); - - fMapCount++; - } - - // now, fill in the pentry - struct thread* thread = thread_get_current_thread(); - ThreadCPUPinner pinner(thread); - - page_table_entry* pt = fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - index = VADDR_TO_PTENT(va); - - ASSERT_PRINT((pt[index] & X86_PTE_PRESENT) == 0, - "virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx32, va, - pt[index]); - - put_page_table_entry_in_pgtable(&pt[index], pa, attributes, memoryType, - IS_KERNEL_MAP(map)); - - pinner.Unlock(); - - // Note: We don't need to invalidate the TLB for this address, as previously - // the entry was not present and the TLB doesn't cache those entries. - - fMapCount++; - - return 0; -} - - -status_t -X86VMTranslationMap::Unmap(addr_t start, addr_t end) -{ - page_directory_entry *pd = fPagingStructures->pgdir_virt; - - start = ROUNDDOWN(start, B_PAGE_SIZE); - end = ROUNDUP(end, B_PAGE_SIZE); - - TRACE("unmap_tmap: asked to free pages 0x%lx to 0x%lx\n", start, end); - -restart: - if (start >= end) - return B_OK; - - int index = VADDR_TO_PDENT(start); - if ((pd[index] & X86_PDE_PRESENT) == 0) { - // no pagetable here, move the start up to access the next page table - start = ROUNDUP(start + 1, B_PAGE_SIZE * 1024); - if (start == 0) - return B_OK; - goto restart; - } - - struct thread* thread = thread_get_current_thread(); - ThreadCPUPinner pinner(thread); - - page_table_entry* pt = fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - - for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end); - index++, start += B_PAGE_SIZE) { - if ((pt[index] & X86_PTE_PRESENT) == 0) { - // page mapping not valid - continue; - } - - TRACE("unmap_tmap: removing page 0x%lx\n", start); - - page_table_entry oldEntry = clear_page_table_entry_flags(&pt[index], - X86_PTE_PRESENT); - fMapCount--; - - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - // Note, that we only need to invalidate the address, if the - // accessed flags was set, since only then the entry could have been - // in any TLB. - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = start; - - fInvalidPagesCount++; - } - } - - pinner.Unlock(); - - goto restart; -} - - -/*! Caller must have locked the cache of the page to be unmapped. - This object shouldn't be locked. -*/ -status_t -X86VMTranslationMap::UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue) -{ - ASSERT(address % B_PAGE_SIZE == 0); - - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - TRACE("X86VMTranslationMap::UnmapPage(%#" B_PRIxADDR ")\n", address); - - RecursiveLocker locker(fLock); - - int index = VADDR_TO_PDENT(address); - if ((pd[index] & X86_PDE_PRESENT) == 0) - return B_ENTRY_NOT_FOUND; - - ThreadCPUPinner pinner(thread_get_current_thread()); - - page_table_entry* pt = fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - - index = VADDR_TO_PTENT(address); - page_table_entry oldEntry = clear_page_table_entry(&pt[index]); - - pinner.Unlock(); - - if ((oldEntry & X86_PTE_PRESENT) == 0) { - // page mapping not valid - return B_ENTRY_NOT_FOUND; - } - - fMapCount--; - - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - // Note, that we only need to invalidate the address, if the - // accessed flags was set, since only then the entry could have been - // in any TLB. - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = address; - - fInvalidPagesCount++; - - Flush(); - - // NOTE: Between clearing the page table entry and Flush() other - // processors (actually even this processor with another thread of the - // same team) could still access the page in question via their cached - // entry. We can obviously lose a modified flag in this case, with the - // effect that the page looks unmodified (and might thus be recycled), - // but is actually modified. - // In most cases this is harmless, but for vm_remove_all_page_mappings() - // this is actually a problem. - // Interestingly FreeBSD seems to ignore this problem as well - // (cf. pmap_remove_all()), unless I've missed something. - } - - if (area->cache_type == CACHE_TYPE_DEVICE) - return B_OK; - - // get the page - vm_page* page = vm_lookup_page( - (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); - ASSERT(page != NULL); - - // transfer the accessed/dirty flags to the page - if ((oldEntry & X86_PTE_ACCESSED) != 0) - page->accessed = true; - if ((oldEntry & X86_PTE_DIRTY) != 0) - page->modified = true; - - // remove the mapping object/decrement the wired_count of the page - vm_page_mapping* mapping = NULL; - if (area->wiring == B_NO_LOCK) { - vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); - while ((mapping = iterator.Next()) != NULL) { - if (mapping->area == area) { - area->mappings.Remove(mapping); - page->mappings.Remove(mapping); - break; - } - } - - ASSERT(mapping != NULL); - } else - page->wired_count--; - - locker.Unlock(); - - if (page->wired_count == 0 && page->mappings.IsEmpty()) { - atomic_add(&gMappedPagesCount, -1); - - if (updatePageQueue) { - if (page->Cache()->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - } - } - - if (mapping != NULL) { - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - object_cache_free(gPageMappingsObjectCache, mapping, - CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0)); - } - - return B_OK; -} - - -void -X86VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size, - bool updatePageQueue) -{ - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - addr_t start = base; - addr_t end = base + size; - - TRACE("X86VMTranslationMap::UnmapPages(%p, %#" B_PRIxADDR ", %#" - B_PRIxADDR ")\n", area, start, end); - - VMAreaMappings queue; - - RecursiveLocker locker(fLock); - - while (start < end) { - int index = VADDR_TO_PDENT(start); - if ((pd[index] & X86_PDE_PRESENT) == 0) { - // no page table here, move the start up to access the next page - // table - start = ROUNDUP(start + 1, B_PAGE_SIZE * 1024); - if (start == 0) - break; - continue; - } - - struct thread* thread = thread_get_current_thread(); - ThreadCPUPinner pinner(thread); - - page_table_entry* pt = fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - - for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end); - index++, start += B_PAGE_SIZE) { - page_table_entry oldEntry = clear_page_table_entry(&pt[index]); - if ((oldEntry & X86_PTE_PRESENT) == 0) - continue; - - fMapCount--; - - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - // Note, that we only need to invalidate the address, if the - // accessed flags was set, since only then the entry could have - // been in any TLB. - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = start; - - fInvalidPagesCount++; - } - - if (area->cache_type != CACHE_TYPE_DEVICE) { - // get the page - vm_page* page = vm_lookup_page( - (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); - ASSERT(page != NULL); - - DEBUG_PAGE_ACCESS_START(page); - - // transfer the accessed/dirty flags to the page - if ((oldEntry & X86_PTE_ACCESSED) != 0) - page->accessed = true; - if ((oldEntry & X86_PTE_DIRTY) != 0) - page->modified = true; - - // remove the mapping object/decrement the wired_count of the - // page - if (area->wiring == B_NO_LOCK) { - vm_page_mapping* mapping = NULL; - vm_page_mappings::Iterator iterator - = page->mappings.GetIterator(); - while ((mapping = iterator.Next()) != NULL) { - if (mapping->area == area) - break; - } - - ASSERT(mapping != NULL); - - area->mappings.Remove(mapping); - page->mappings.Remove(mapping); - queue.Add(mapping); - } else - page->wired_count--; - - if (page->wired_count == 0 && page->mappings.IsEmpty()) { - atomic_add(&gMappedPagesCount, -1); - - if (updatePageQueue) { - if (page->Cache()->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - } - } - - DEBUG_PAGE_ACCESS_END(page); - } - } - - Flush(); - // flush explicitly, since we directly use the lock - - pinner.Unlock(); - } - - // TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not - // really critical here, as in all cases this method is used, the unmapped - // area range is unmapped for good (resized/cut) and the pages will likely - // be freed. - - locker.Unlock(); - - // free removed mappings - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - while (vm_page_mapping* mapping = queue.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); -} - - -void -X86VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace, - bool ignoreTopCachePageFlags) -{ - if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - X86VMTranslationMap::UnmapPages(area, area->Base(), area->Size(), true); - return; - } - - bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; - - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - RecursiveLocker locker(fLock); - - VMAreaMappings mappings; - mappings.MoveFrom(&area->mappings); - - for (VMAreaMappings::Iterator it = mappings.GetIterator(); - vm_page_mapping* mapping = it.Next();) { - vm_page* page = mapping->page; - page->mappings.Remove(mapping); - - VMCache* cache = page->Cache(); - - bool pageFullyUnmapped = false; - if (page->wired_count == 0 && page->mappings.IsEmpty()) { - atomic_add(&gMappedPagesCount, -1); - pageFullyUnmapped = true; - } - - if (unmapPages || cache != area->cache) { - addr_t address = area->Base() - + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); - - int index = VADDR_TO_PDENT(address); - if ((pd[index] & X86_PDE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page dir entry", page, area, address); - continue; - } - - ThreadCPUPinner pinner(thread_get_current_thread()); - - page_table_entry* pt = fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - page_table_entry oldEntry = clear_page_table_entry( - &pt[VADDR_TO_PTENT(address)]); - - pinner.Unlock(); - - if ((oldEntry & X86_PTE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page table entry", page, area, address); - continue; - } - - // transfer the accessed/dirty flags to the page and invalidate - // the mapping, if necessary - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - page->accessed = true; - - if (!deletingAddressSpace) { - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = address; - - fInvalidPagesCount++; - } - } - - if ((oldEntry & X86_PTE_DIRTY) != 0) - page->modified = true; - - if (pageFullyUnmapped) { - DEBUG_PAGE_ACCESS_START(page); - - if (cache->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - - DEBUG_PAGE_ACCESS_END(page); - } - } - - fMapCount--; - } - - Flush(); - // flush explicitely, since we directly use the lock - - locker.Unlock(); - - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - while (vm_page_mapping* mapping = mappings.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); -} - - -status_t -X86VMTranslationMap::Query(addr_t va, phys_addr_t *_physical, uint32 *_flags) -{ - // default the flags to not present - *_flags = 0; - *_physical = 0; - - int index = VADDR_TO_PDENT(va); - page_directory_entry *pd = fPagingStructures->pgdir_virt; - if ((pd[index] & X86_PDE_PRESENT) == 0) { - // no pagetable here - return B_OK; - } - - struct thread* thread = thread_get_current_thread(); - ThreadCPUPinner pinner(thread); - - page_table_entry* pt = fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - page_table_entry entry = pt[VADDR_TO_PTENT(va)]; - - *_physical = entry & X86_PDE_ADDRESS_MASK; - - // read in the page state flags - if ((entry & X86_PTE_USER) != 0) { - *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0) - | B_READ_AREA; - } - - *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0) - | B_KERNEL_READ_AREA - | ((entry & X86_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0) - | ((entry & X86_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0) - | ((entry & X86_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0); - - pinner.Unlock(); - - TRACE("query_tmap: returning pa 0x%lx for va 0x%lx\n", *_physical, va); - - return B_OK; -} - - -status_t -X86VMTranslationMap::QueryInterrupt(addr_t va, phys_addr_t *_physical, - uint32 *_flags) -{ - *_flags = 0; - *_physical = 0; - - int index = VADDR_TO_PDENT(va); - page_directory_entry* pd = fPagingStructures->pgdir_virt; - if ((pd[index] & X86_PDE_PRESENT) == 0) { - // no pagetable here - return B_OK; - } - - // map page table entry - page_table_entry* pt = sPhysicalPageMapper->InterruptGetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - page_table_entry entry = pt[VADDR_TO_PTENT(va)]; - - *_physical = entry & X86_PDE_ADDRESS_MASK; - - // read in the page state flags - if ((entry & X86_PTE_USER) != 0) { - *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0) - | B_READ_AREA; - } - - *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0) - | B_KERNEL_READ_AREA - | ((entry & X86_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0) - | ((entry & X86_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0) - | ((entry & X86_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0); - - return B_OK; -} - - -addr_t -X86VMTranslationMap::MappedSize() const -{ - return fMapCount; -} - - -status_t -X86VMTranslationMap::Protect(addr_t start, addr_t end, uint32 attributes, - uint32 memoryType) -{ - page_directory_entry *pd = fPagingStructures->pgdir_virt; - - start = ROUNDDOWN(start, B_PAGE_SIZE); - - TRACE("protect_tmap: pages 0x%lx to 0x%lx, attributes %lx\n", start, end, - attributes); - - // compute protection flags - uint32 newProtectionFlags = 0; - if ((attributes & B_USER_PROTECTION) != 0) { - newProtectionFlags = X86_PTE_USER; - if ((attributes & B_WRITE_AREA) != 0) - newProtectionFlags |= X86_PTE_WRITABLE; - } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) - newProtectionFlags = X86_PTE_WRITABLE; - -restart: - if (start >= end) - return B_OK; - - int index = VADDR_TO_PDENT(start); - if ((pd[index] & X86_PDE_PRESENT) == 0) { - // no pagetable here, move the start up to access the next page table - start = ROUNDUP(start + 1, B_PAGE_SIZE * 1024); - if (start == 0) - return B_OK; - goto restart; - } - - struct thread* thread = thread_get_current_thread(); - ThreadCPUPinner pinner(thread); - - page_table_entry* pt = fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - - for (index = VADDR_TO_PTENT(start); index < 1024 && start < end; - index++, start += B_PAGE_SIZE) { - page_table_entry entry = pt[index]; - if ((entry & X86_PTE_PRESENT) == 0) { - // page mapping not valid - continue; - } - - TRACE("protect_tmap: protect page 0x%lx\n", start); - - // set the new protection flags -- we want to do that atomically, - // without changing the accessed or dirty flag - page_table_entry oldEntry; - while (true) { - oldEntry = test_and_set_page_table_entry(&pt[index], - (entry & ~(X86_PTE_PROTECTION_MASK | X86_PTE_MEMORY_TYPE_MASK)) - | newProtectionFlags | memory_type_to_pte_flags(memoryType), - entry); - if (oldEntry == entry) - break; - entry = oldEntry; - } - - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - // Note, that we only need to invalidate the address, if the - // accessed flag was set, since only then the entry could have been - // in any TLB. - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = start; - - fInvalidPagesCount++; - } - } - - pinner.Unlock(); - - goto restart; -} - - -status_t -X86VMTranslationMap::ClearFlags(addr_t va, uint32 flags) -{ - int index = VADDR_TO_PDENT(va); - page_directory_entry* pd = fPagingStructures->pgdir_virt; - if ((pd[index] & X86_PDE_PRESENT) == 0) { - // no pagetable here - return B_OK; - } - - uint32 flagsToClear = ((flags & PAGE_MODIFIED) ? X86_PTE_DIRTY : 0) - | ((flags & PAGE_ACCESSED) ? X86_PTE_ACCESSED : 0); - - struct thread* thread = thread_get_current_thread(); - ThreadCPUPinner pinner(thread); - - page_table_entry* pt = fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - index = VADDR_TO_PTENT(va); - - // clear out the flags we've been requested to clear - page_table_entry oldEntry - = clear_page_table_entry_flags(&pt[index], flagsToClear); - - pinner.Unlock(); - - if ((oldEntry & flagsToClear) != 0) { - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = va; - - fInvalidPagesCount++; - } - - return B_OK; -} - - -bool -X86VMTranslationMap::ClearAccessedAndModified(VMArea* area, addr_t address, - bool unmapIfUnaccessed, bool& _modified) -{ - ASSERT(address % B_PAGE_SIZE == 0); - - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - TRACE("X86VMTranslationMap::ClearAccessedAndModified(%#" B_PRIxADDR ")\n", - address); - - RecursiveLocker locker(fLock); - - int index = VADDR_TO_PDENT(address); - if ((pd[index] & X86_PDE_PRESENT) == 0) - return false; - - ThreadCPUPinner pinner(thread_get_current_thread()); - - page_table_entry* pt = fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - - index = VADDR_TO_PTENT(address); - - // perform the deed - page_table_entry oldEntry; - - if (unmapIfUnaccessed) { - while (true) { - oldEntry = pt[index]; - if ((oldEntry & X86_PTE_PRESENT) == 0) { - // page mapping not valid - return false; - } - - if (oldEntry & X86_PTE_ACCESSED) { - // page was accessed -- just clear the flags - oldEntry = clear_page_table_entry_flags(&pt[index], - X86_PTE_ACCESSED | X86_PTE_DIRTY); - break; - } - - // page hasn't been accessed -- unmap it - if (test_and_set_page_table_entry(&pt[index], 0, oldEntry) - == oldEntry) { - break; - } - - // something changed -- check again - } - } else { - oldEntry = clear_page_table_entry_flags(&pt[index], - X86_PTE_ACCESSED | X86_PTE_DIRTY); - } - - pinner.Unlock(); - - _modified = (oldEntry & X86_PTE_DIRTY) != 0; - - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - // Note, that we only need to invalidate the address, if the - // accessed flags was set, since only then the entry could have been - // in any TLB. - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = address; - - fInvalidPagesCount++; - - Flush(); - - return true; - } - - if (!unmapIfUnaccessed) - return false; - - // We have unmapped the address. Do the "high level" stuff. - - fMapCount--; - - if (area->cache_type == CACHE_TYPE_DEVICE) - return false; - - // get the page - vm_page* page = vm_lookup_page( - (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); - ASSERT(page != NULL); - - // remove the mapping object/decrement the wired_count of the page - vm_page_mapping* mapping = NULL; - if (area->wiring == B_NO_LOCK) { - vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); - while ((mapping = iterator.Next()) != NULL) { - if (mapping->area == area) { - area->mappings.Remove(mapping); - page->mappings.Remove(mapping); - break; - } - } - - ASSERT(mapping != NULL); - } else - page->wired_count--; - - locker.Unlock(); - - if (page->wired_count == 0 && page->mappings.IsEmpty()) - atomic_add(&gMappedPagesCount, -1); - - if (mapping != NULL) { - object_cache_free(gPageMappingsObjectCache, mapping, - CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); - // Since this is called by the page daemon, we never want to lock - // the kernel address space. - } - - return false; -} - - -void -X86VMTranslationMap::Flush() -{ - if (fInvalidPagesCount <= 0) - return; - - struct thread* thread = thread_get_current_thread(); - thread_pin_to_current_cpu(thread); - - if (fInvalidPagesCount > PAGE_INVALIDATE_CACHE_SIZE) { - // invalidate all pages - TRACE("flush_tmap: %d pages to invalidate, invalidate all\n", - fInvalidPagesCount); - - if (IS_KERNEL_MAP(map)) { - arch_cpu_global_TLB_invalidate(); - smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, - NULL, SMP_MSG_FLAG_SYNC); - } else { - cpu_status state = disable_interrupts(); - arch_cpu_user_TLB_invalidate(); - restore_interrupts(state); - - int cpu = smp_get_current_cpu(); - uint32 cpuMask = fPagingStructures->active_on_cpus - & ~((uint32)1 << cpu); - if (cpuMask != 0) { - smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES, - 0, 0, 0, NULL, SMP_MSG_FLAG_SYNC); - } - } - } else { - TRACE("flush_tmap: %d pages to invalidate, invalidate list\n", - fInvalidPagesCount); - - arch_cpu_invalidate_TLB_list(fInvalidPages, fInvalidPagesCount); - - if (IS_KERNEL_MAP(map)) { - smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, - (uint32)fInvalidPages, fInvalidPagesCount, 0, NULL, - SMP_MSG_FLAG_SYNC); - } else { - int cpu = smp_get_current_cpu(); - uint32 cpuMask = fPagingStructures->active_on_cpus - & ~((uint32)1 << cpu); - if (cpuMask != 0) { - smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST, - (uint32)fInvalidPages, fInvalidPagesCount, 0, NULL, - SMP_MSG_FLAG_SYNC); - } - } - } - fInvalidPagesCount = 0; - - thread_unpin_from_current_cpu(thread); -} - - // #pragma mark - VM API status_t arch_vm_translation_map_create_map(bool kernel, VMTranslationMap** _map) { - X86VMTranslationMap* map = new(std::nothrow) X86VMTranslationMap; - if (map == NULL) - return B_NO_MEMORY; - - status_t error = map->Init(kernel); - if (error != B_OK) { - delete map; - return error; - } - - *_map = map; - return B_OK; + return sPagingMethod->CreateTranslationMap(kernel, _map); } @@ -1309,25 +58,7 @@ arch_vm_translation_map_init(kernel_args *args, { TRACE("vm_translation_map_init: entry\n"); - // page hole set up in stage2 - sPageHole = (page_table_entry *)args->arch_args.page_hole; - // calculate where the pgdir would be - sPageHolePageDir = (page_directory_entry*) - (((addr_t)args->arch_args.page_hole) - + (B_PAGE_SIZE * 1024 - B_PAGE_SIZE)); - // clear out the bottom 2 GB, unmap everything - memset(sPageHolePageDir + FIRST_USER_PGDIR_ENT, 0, - sizeof(page_directory_entry) * NUM_USER_PGDIR_ENTS); - - sKernelPhysicalPageDirectory = args->arch_args.phys_pgdir; - sKernelVirtualPageDirectory = (page_directory_entry*) - args->arch_args.vir_pgdir; - #ifdef TRACE_VM_TMAP - TRACE("page hole: %p, page dir: %p\n", sPageHole, sPageHolePageDir); - TRACE("page dir: %p (physical: %#" B_PRIx32 ")\n", - sKernelVirtualPageDirectory, sKernelPhysicalPageDirectory); - TRACE("physical memory ranges:\n"); for (uint32 i = 0; i < args->num_physical_memory_ranges; i++) { phys_addr_t start = args->physical_memory_range[i].start; @@ -1352,24 +83,9 @@ arch_vm_translation_map_init(kernel_args *args, } #endif - B_INITIALIZE_SPINLOCK(&sPagingStructuresListLock); - new (&sPagingStructuresList) PagingStructuresList; + sPagingMethod = X86PagingMethod32Bit::Create(); - large_memory_physical_page_ops_init(args, sPhysicalPageMapper, - sKernelPhysicalPageMapper); - // TODO: Select the best page mapper! - - // enable global page feature if available - if (x86_check_feature(IA32_FEATURE_PGE, FEATURE_COMMON)) { - // this prevents kernel pages from being flushed from TLB on - // context-switch - x86_write_cr4(x86_read_cr4() | IA32_CR4_GLOBAL_PAGES); - } - - TRACE("vm_translation_map_init: done\n"); - - *_physicalPageMapper = sPhysicalPageMapper; - return B_OK; + return sPagingMethod->Init(args, _physicalPageMapper); } @@ -1383,81 +99,19 @@ arch_vm_translation_map_init_post_sem(kernel_args *args) status_t arch_vm_translation_map_init_post_area(kernel_args *args) { - // now that the vm is initialized, create a region that represents - // the page hole - void *temp; - status_t error; - area_id area; - TRACE("vm_translation_map_init_post_area: entry\n"); - // unmap the page hole hack we were using before - sKernelVirtualPageDirectory[1023] = 0; - sPageHolePageDir = NULL; - sPageHole = NULL; - - temp = (void *)sKernelVirtualPageDirectory; - area = create_area("kernel_pgdir", &temp, B_EXACT_ADDRESS, B_PAGE_SIZE, - B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - if (area < B_OK) - return area; - - error = sPhysicalPageMapper->InitPostArea(args); - if (error != B_OK) - return error; - - TRACE("vm_translation_map_init_post_area: done\n"); - return B_OK; + return sPagingMethod->InitPostArea(args); } -// XXX horrible back door to map a page quickly regardless of translation map -// object, etc. -// used only during VM setup. -// uses a 'page hole' set up in the stage 2 bootloader. The page hole is created -// by pointing one of the pgdir entries back at itself, effectively mapping the -// contents of all of the 4MB of pagetables into a 4 MB region. It's only used -// here, and is later unmapped. - status_t arch_vm_translation_map_early_map(kernel_args *args, addr_t va, phys_addr_t pa, uint8 attributes, phys_addr_t (*get_free_page)(kernel_args *)) { - int index; - TRACE("early_tmap: entry pa 0x%lx va 0x%lx\n", pa, va); - // check to see if a page table exists for this range - index = VADDR_TO_PDENT(va); - if ((sPageHolePageDir[index] & X86_PDE_PRESENT) == 0) { - phys_addr_t pgtable; - page_directory_entry *e; - // we need to allocate a pgtable - pgtable = get_free_page(args); - // pgtable is in pages, convert to physical address - pgtable *= B_PAGE_SIZE; - - TRACE("early_map: asked for free page for pgtable. 0x%lx\n", pgtable); - - // put it in the pgdir - e = &sPageHolePageDir[index]; - x86_put_pgtable_in_pgdir(e, pgtable, attributes); - - // zero it out in it's new mapping - memset((unsigned int*)((addr_t)sPageHole - + (va / B_PAGE_SIZE / 1024) * B_PAGE_SIZE), 0, B_PAGE_SIZE); - } - - ASSERT_PRINT((sPageHole[va / B_PAGE_SIZE] & X86_PTE_PRESENT) == 0, - "virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx32 - ", existing pte: %#" B_PRIx32, va, sPageHolePageDir[index], - sPageHole[va / B_PAGE_SIZE]); - - // now, fill in the pentry - put_page_table_entry_in_pgtable(sPageHole + va / B_PAGE_SIZE, pa, - attributes, 0, IS_KERNEL_ADDRESS(va)); - - return B_OK; + return sPagingMethod->MapEarly(args, va, pa, attributes, get_free_page); } @@ -1477,61 +131,5 @@ bool arch_vm_translation_map_is_kernel_page_accessible(addr_t virtualAddress, uint32 protection) { - // We only trust the kernel team's page directory. So switch to it first. - // Always set it to make sure the TLBs don't contain obsolete data. - uint32 physicalPageDirectory; - read_cr3(physicalPageDirectory); - write_cr3(sKernelPhysicalPageDirectory); - - // get the page directory entry for the address - page_directory_entry pageDirectoryEntry; - uint32 index = VADDR_TO_PDENT(virtualAddress); - - if (physicalPageDirectory == sKernelPhysicalPageDirectory) { - pageDirectoryEntry = sKernelVirtualPageDirectory[index]; - } else if (sPhysicalPageMapper != NULL) { - // map the original page directory and get the entry - void* handle; - addr_t virtualPageDirectory; - status_t error = sPhysicalPageMapper->GetPageDebug( - physicalPageDirectory, &virtualPageDirectory, &handle); - if (error == B_OK) { - pageDirectoryEntry - = ((page_directory_entry*)virtualPageDirectory)[index]; - sPhysicalPageMapper->PutPageDebug(virtualPageDirectory, - handle); - } else - pageDirectoryEntry = 0; - } else - pageDirectoryEntry = 0; - - // map the page table and get the entry - page_table_entry pageTableEntry; - index = VADDR_TO_PTENT(virtualAddress); - - if ((pageDirectoryEntry & X86_PDE_PRESENT) != 0 - && sPhysicalPageMapper != NULL) { - void* handle; - addr_t virtualPageTable; - status_t error = sPhysicalPageMapper->GetPageDebug( - pageDirectoryEntry & X86_PDE_ADDRESS_MASK, &virtualPageTable, - &handle); - if (error == B_OK) { - pageTableEntry = ((page_table_entry*)virtualPageTable)[index]; - sPhysicalPageMapper->PutPageDebug(virtualPageTable, handle); - } else - pageTableEntry = 0; - } else - pageTableEntry = 0; - - // switch back to the original page directory - if (physicalPageDirectory != sKernelPhysicalPageDirectory) - write_cr3(physicalPageDirectory); - - if ((pageTableEntry & X86_PTE_PRESENT) == 0) - return false; - - // present means kernel-readable, so check for writable - return (protection & B_KERNEL_WRITE_AREA) == 0 - || (pageTableEntry & X86_PTE_WRITABLE) != 0; + return sPagingMethod->IsKernelPageAccessible(virtualAddress, protection); } diff --git a/src/system/kernel/arch/x86/x86_paging.h b/src/system/kernel/arch/x86/x86_paging.h index 4fdf7d613e..10499702c9 100644 --- a/src/system/kernel/arch/x86/x86_paging.h +++ b/src/system/kernel/arch/x86/x86_paging.h @@ -74,7 +74,7 @@ struct X86PagingStructures : DeferredDeletable { inline void AddReference(); inline void RemoveReference(); - void Delete(); + virtual void Delete() = 0; };