From a25542e7ef4b7c96878d0a7778d63f03bdcb60da Mon Sep 17 00:00:00 2001 From: milek7 Date: Thu, 28 Apr 2022 21:46:32 +0200 Subject: [PATCH] arm64: MMU WIP. Change-Id: I19b2b9617fcb7bc047f9bea156801ec78786532e Reviewed-on: https://review.haiku-os.org/c/haiku/+/5261 Reviewed-by: Adrien Destugues Tested-by: Commit checker robot --- .../platform/efi/arch/arm64/arch_start.cpp | 160 +++-- src/system/kernel/arch/arm64/Jamfile | 3 + .../arch/arm64/PMAPPhysicalPageMapper.cpp | 71 +++ .../arch/arm64/PMAPPhysicalPageMapper.h | 51 ++ .../arch/arm64/VMSAv8TranslationMap.cpp | 597 ++++++++++++++++++ .../kernel/arch/arm64/VMSAv8TranslationMap.h | 101 +++ src/system/kernel/arch/arm64/arch_cpu.cpp | 9 + src/system/kernel/arch/arm64/arch_vm.cpp | 34 +- .../arch/arm64/arch_vm_translation_map.cpp | 161 ++++- 9 files changed, 1084 insertions(+), 103 deletions(-) create mode 100644 src/system/kernel/arch/arm64/PMAPPhysicalPageMapper.cpp create mode 100644 src/system/kernel/arch/arm64/PMAPPhysicalPageMapper.h create mode 100644 src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp create mode 100644 src/system/kernel/arch/arm64/VMSAv8TranslationMap.h diff --git a/src/system/boot/platform/efi/arch/arm64/arch_start.cpp b/src/system/boot/platform/efi/arch/arm64/arch_start.cpp index 7251b90b68..9715e969c8 100644 --- a/src/system/boot/platform/efi/arch/arm64/arch_start.cpp +++ b/src/system/boot/platform/efi/arch/arm64/arch_start.cpp @@ -13,19 +13,17 @@ #include "aarch64.h" -extern "C" void arch_enter_kernel(struct kernel_args *kernelArgs, - addr_t kernelEntry, addr_t kernelStackTop); +extern "C" void arch_enter_kernel( + struct kernel_args* kernelArgs, addr_t kernelEntry, addr_t kernelStackTop); extern void arch_mmu_dump_present_tables(); extern const char* granule_type_str(int tg); extern uint32_t arch_mmu_generate_post_efi_page_tables(size_t memory_map_size, - efi_memory_descriptor *memory_map, size_t descriptor_size, - uint32_t descriptor_version); + efi_memory_descriptor* memory_map, size_t descriptor_size, uint32_t descriptor_version); -extern void arch_mmu_post_efi_setup(size_t memory_map_size, - efi_memory_descriptor *memory_map, size_t descriptor_size, - uint32_t descriptor_version); +extern void arch_mmu_post_efi_setup(size_t memory_map_size, efi_memory_descriptor* memory_map, + size_t descriptor_size, uint32_t descriptor_version); extern void arch_mmu_setup_EL1(uint64 tcr); @@ -33,39 +31,39 @@ extern void arch_mmu_setup_EL1(uint64 tcr); static const char* memory_region_type_str(int type) { - switch (type) { - case EfiReservedMemoryType: - return "ReservedMemoryType"; - case EfiLoaderCode: - return "LoaderCode"; - case EfiLoaderData: - return "LoaderData"; - case EfiBootServicesCode: - return "BootServicesCode"; - case EfiBootServicesData: - return "BootServicesData"; - case EfiRuntimeServicesCode: - return "RuntimeServicesCode"; - case EfiRuntimeServicesData: - return "RuntimeServicesData"; - case EfiConventionalMemory: - return "ConventionalMemory"; - case EfiUnusableMemory: - return "UnusableMemory"; - case EfiACPIReclaimMemory: - return "ACPIReclaimMemory"; - case EfiACPIMemoryNVS: - return "ACPIMemoryNVS"; - case EfiMemoryMappedIO: - return "MMIO"; - case EfiMemoryMappedIOPortSpace: - return "MMIOPortSpace"; - case EfiPalCode: - return "PalCode"; - case EfiPersistentMemory: - return "PersistentMemory"; - default: - return "unknown"; + switch (type) { + case EfiReservedMemoryType: + return "ReservedMemoryType"; + case EfiLoaderCode: + return "LoaderCode"; + case EfiLoaderData: + return "LoaderData"; + case EfiBootServicesCode: + return "BootServicesCode"; + case EfiBootServicesData: + return "BootServicesData"; + case EfiRuntimeServicesCode: + return "RuntimeServicesCode"; + case EfiRuntimeServicesData: + return "RuntimeServicesData"; + case EfiConventionalMemory: + return "ConventionalMemory"; + case EfiUnusableMemory: + return "UnusableMemory"; + case EfiACPIReclaimMemory: + return "ACPIReclaimMemory"; + case EfiACPIMemoryNVS: + return "ACPIMemoryNVS"; + case EfiMemoryMappedIO: + return "MMIO"; + case EfiMemoryMappedIOPortSpace: + return "MMIOPortSpace"; + case EfiPalCode: + return "PalCode"; + case EfiPersistentMemory: + return "PersistentMemory"; + default: + return "unknown"; } } @@ -85,44 +83,42 @@ arch_start_kernel(addr_t kernelEntry) // First call is to determine the buffer size. size_t memory_map_size = 0; efi_memory_descriptor dummy; - efi_memory_descriptor *memory_map; + efi_memory_descriptor* memory_map; size_t map_key; size_t descriptor_size; uint32_t descriptor_version; - if (kBootServices->GetMemoryMap(&memory_map_size, &dummy, &map_key, - &descriptor_size, &descriptor_version) != EFI_BUFFER_TOO_SMALL) { + if (kBootServices->GetMemoryMap( + &memory_map_size, &dummy, &map_key, &descriptor_size, &descriptor_version) + != EFI_BUFFER_TOO_SMALL) { panic("Unable to determine size of system memory map"); } // Allocate a buffer twice as large as needed just in case it gets bigger // between calls to ExitBootServices. size_t actual_memory_map_size = memory_map_size * 2; - memory_map - = (efi_memory_descriptor *)kernel_args_malloc(actual_memory_map_size); + memory_map = (efi_memory_descriptor*) kernel_args_malloc(actual_memory_map_size); if (memory_map == NULL) panic("Unable to allocate memory map."); // Read (and print) the memory map. memory_map_size = actual_memory_map_size; - if (kBootServices->GetMemoryMap(&memory_map_size, memory_map, &map_key, - &descriptor_size, &descriptor_version) != EFI_SUCCESS) { + if (kBootServices->GetMemoryMap( + &memory_map_size, memory_map, &map_key, &descriptor_size, &descriptor_version) + != EFI_SUCCESS) { panic("Unable to fetch system memory map."); } - addr_t addr = (addr_t)memory_map; + addr_t addr = (addr_t) memory_map; efi_physical_addr loaderCode = 0LL; dprintf("System provided memory map:\n"); for (size_t i = 0; i < memory_map_size / descriptor_size; ++i) { - efi_memory_descriptor *entry - = (efi_memory_descriptor *)(addr + i * descriptor_size); - dprintf(" phys: 0x%0lx-0x%0lx, virt: 0x%0lx-0x%0lx, size = 0x%0lx, type: %s (%#x), attr: %#lx\n", - entry->PhysicalStart, - entry->PhysicalStart + entry->NumberOfPages * B_PAGE_SIZE, - entry->VirtualStart, - entry->VirtualStart + entry->NumberOfPages * B_PAGE_SIZE, - entry->NumberOfPages * B_PAGE_SIZE, - memory_region_type_str(entry->Type), entry->Type, + efi_memory_descriptor* entry = (efi_memory_descriptor*) (addr + i * descriptor_size); + dprintf(" phys: 0x%0lx-0x%0lx, virt: 0x%0lx-0x%0lx, size = 0x%0lx, type: %s (%#x), attr: " + "%#lx\n", + entry->PhysicalStart, entry->PhysicalStart + entry->NumberOfPages * B_PAGE_SIZE, + entry->VirtualStart, entry->VirtualStart + entry->NumberOfPages * B_PAGE_SIZE, + entry->NumberOfPages * B_PAGE_SIZE, memory_region_type_str(entry->Type), entry->Type, entry->Attribute); if (entry->Type == EfiLoaderCode) loaderCode = entry->PhysicalStart; @@ -131,36 +127,32 @@ arch_start_kernel(addr_t kernelEntry) // offset for properly align symbols dprintf("Efi loader symbols offset: 0x%0lx:\n", loaderCode); -/* -* "The AArch64 exception model is made up of a number of exception levels -* (EL0 - EL3), with EL0 and EL1 having a secure and a non-secure -* counterpart. EL2 is the hypervisor level and exists only in non-secure -* mode. EL3 is the highest priority level and exists only in secure mode." -* -* "2.3 UEFI System Environment and Configuration -* The resident UEFI boot-time environment shall use the highest non-secure -* privilege level available. The exact meaning of this is architecture -* dependent, as detailed below." + /* + * "The AArch64 exception model is made up of a number of exception levels + * (EL0 - EL3), with EL0 and EL1 having a secure and a non-secure + * counterpart. EL2 is the hypervisor level and exists only in non-secure + * mode. EL3 is the highest priority level and exists only in secure mode." + * + * "2.3 UEFI System Environment and Configuration + * The resident UEFI boot-time environment shall use the highest non-secure + * privilege level available. The exact meaning of this is architecture + * dependent, as detailed below." -* "2.3.1 AArch64 Exception Levels -* On AArch64 UEFI shall execute as 64-bit code at either EL1 or EL2, -* depending on whether or not virtualization is available at OS load time." -*/ + * "2.3.1 AArch64 Exception Levels + * On AArch64 UEFI shall execute as 64-bit code at either EL1 or EL2, + * depending on whether or not virtualization is available at OS load time." + */ uint64 el = arch_exception_level(); dprintf("Current Exception Level EL%1lx\n", el); dprintf("TTBR0: %" B_PRIx64 " TTBRx: %" B_PRIx64 " SCTLR: %" B_PRIx64 " TCR: %" B_PRIx64 "\n", - arch_mmu_base_register(), - arch_mmu_base_register(true), - _arch_mmu_get_sctlr(), + arch_mmu_base_register(), arch_mmu_base_register(true), _arch_mmu_get_sctlr(), _arch_mmu_get_tcr()); if (arch_mmu_enabled()) { - dprintf("MMU Enabled, Granularity %s, bits %d\n", - granule_type_str(arch_mmu_user_granule()), + dprintf("MMU Enabled, Granularity %s, bits %d\n", granule_type_str(arch_mmu_user_granule()), arch_mmu_user_address_bits()); - dprintf("Kernel entry accessibility W: %x R: %x\n", - arch_mmu_write_access(kernelEntry), + dprintf("Kernel entry accessibility W: %x R: %x\n", arch_mmu_write_access(kernelEntry), arch_mmu_read_access(kernelEntry)); arch_mmu_dump_present_tables(); @@ -199,14 +191,15 @@ arch_start_kernel(addr_t kernelEntry) } memory_map_size = actual_memory_map_size; - if (kBootServices->GetMemoryMap(&memory_map_size, memory_map, &map_key, - &descriptor_size, &descriptor_version) != EFI_SUCCESS) { + if (kBootServices->GetMemoryMap( + &memory_map_size, memory_map, &map_key, &descriptor_size, &descriptor_version) + != EFI_SUCCESS) { panic("Unable to fetch system memory map."); } } // Update EFI, generate final kernel physical memory map, etc. - // arch_mmu_post_efi_setup(memory_map_size, memory_map, descriptor_size, descriptor_version); + arch_mmu_post_efi_setup(memory_map_size, memory_map, descriptor_size, descriptor_version); switch (el) { case 1: @@ -224,9 +217,10 @@ arch_start_kernel(addr_t kernelEntry) arch_cache_enable(); - //smp_boot_other_cpus(final_pml4, kernelEntry, (addr_t)&gKernelArgs); + // smp_boot_other_cpus(final_pml4, kernelEntry, (addr_t)&gKernelArgs); - if (arch_mmu_read_access(kernelEntry) && arch_mmu_read_access(gKernelArgs.cpu_kstack[0].start)) { + if (arch_mmu_read_access(kernelEntry) + && arch_mmu_read_access(gKernelArgs.cpu_kstack[0].start)) { // Enter the kernel! arch_enter_kernel(&gKernelArgs, kernelEntry, gKernelArgs.cpu_kstack[0].start + gKernelArgs.cpu_kstack[0].size); diff --git a/src/system/kernel/arch/arm64/Jamfile b/src/system/kernel/arch/arm64/Jamfile index e9d97b6832..ec6546bdea 100644 --- a/src/system/kernel/arch/arm64/Jamfile +++ b/src/system/kernel/arch/arm64/Jamfile @@ -24,6 +24,9 @@ KernelMergeObject kernel_arch_arm64.o : arch_platform.cpp arch_asm.S + VMSAv8TranslationMap.cpp + PMAPPhysicalPageMapper.cpp + # Serial UART and drivers debug_uart.cpp debug_uart_8250.cpp diff --git a/src/system/kernel/arch/arm64/PMAPPhysicalPageMapper.cpp b/src/system/kernel/arch/arm64/PMAPPhysicalPageMapper.cpp new file mode 100644 index 0000000000..efddfdba21 --- /dev/null +++ b/src/system/kernel/arch/arm64/PMAPPhysicalPageMapper.cpp @@ -0,0 +1,71 @@ +/* + * Copyright 2022 Haiku, Inc. All Rights Reserved. + * Distributed under the terms of the MIT License. + */ +#include "PMAPPhysicalPageMapper.h" + + +status_t +PMAPPhysicalPageMapper::GetPage( + phys_addr_t physicalAddress, addr_t* _virtualAddress, void** _handle) +{ + ASSERT(physicalAddress < KERNEL_PMAP_SIZE); + + *_virtualAddress = KERNEL_PMAP_BASE + physicalAddress; + *_handle = NULL; + + return B_OK; +} + + +status_t +PMAPPhysicalPageMapper::PutPage(addr_t virtualAddress, void* handle) +{ + return B_OK; +} + + +status_t +PMAPPhysicalPageMapper::MemsetPhysical(phys_addr_t address, int value, phys_size_t length) +{ + ASSERT(address < KERNEL_PMAP_SIZE); + memset(reinterpret_cast(KERNEL_PMAP_BASE + address), value, length); + + return B_OK; +} + + +status_t +PMAPPhysicalPageMapper::MemcpyFromPhysical(void* to, phys_addr_t from, size_t length, bool user) +{ + if (user) + panic("MemcpyFromPhysical user not impl"); + + ASSERT(from < KERNEL_PMAP_SIZE); + memcpy(to, reinterpret_cast(KERNEL_PMAP_BASE + from), length); + + return B_OK; +} + + +status_t +PMAPPhysicalPageMapper::MemcpyToPhysical(phys_addr_t to, const void* from, size_t length, bool user) +{ + if (user) + panic("MemcpyToPhysical user not impl"); + + ASSERT(to < KERNEL_PMAP_SIZE); + memcpy(reinterpret_cast(KERNEL_PMAP_BASE + to), from, length); + + return B_OK; +} + + +void +PMAPPhysicalPageMapper::MemcpyPhysicalPage(phys_addr_t to, phys_addr_t from) +{ + ASSERT(to < KERNEL_PMAP_SIZE); + ASSERT(from < KERNEL_PMAP_SIZE); + memcpy(reinterpret_cast(KERNEL_PMAP_BASE + to), + reinterpret_cast(KERNEL_PMAP_BASE + from), B_PAGE_SIZE); +} diff --git a/src/system/kernel/arch/arm64/PMAPPhysicalPageMapper.h b/src/system/kernel/arch/arm64/PMAPPhysicalPageMapper.h new file mode 100644 index 0000000000..ce41051fda --- /dev/null +++ b/src/system/kernel/arch/arm64/PMAPPhysicalPageMapper.h @@ -0,0 +1,51 @@ +/* + * Copyright 2022 Haiku, Inc. All Rights Reserved. + * Distributed under the terms of the MIT License. + */ +#ifndef PMAP_PHYSICAL_PAGE_MAPPER_H +#define PMAP_PHYSICAL_PAGE_MAPPER_H + +#include +#include + + +struct PMAPPhysicalPageMapper : public VMPhysicalPageMapper +{ + virtual status_t GetPage(phys_addr_t physicalAddress, + addr_t* _virtualAddress, + void** _handle); + virtual status_t PutPage(addr_t virtualAddress, + void* handle); + + virtual status_t GetPageCurrentCPU( + phys_addr_t physicalAddress, + addr_t* _virtualAddress, + void** _handle) + { + return GetPage(physicalAddress, _virtualAddress, _handle); + } + virtual status_t PutPageCurrentCPU(addr_t virtualAddress, + void* _handle) { return PutPage(virtualAddress, _handle); } + + virtual status_t GetPageDebug(phys_addr_t physicalAddress, + addr_t* _virtualAddress, + void** _handle) { return GetPage(physicalAddress, + _virtualAddress, _handle); } + virtual status_t PutPageDebug(addr_t virtualAddress, + void* _handle) + { + return PutPage(virtualAddress, _handle); + } + + virtual status_t MemsetPhysical(phys_addr_t address, int value, + phys_size_t length); + virtual status_t MemcpyFromPhysical(void* to, phys_addr_t from, + size_t length, bool user); + virtual status_t MemcpyToPhysical(phys_addr_t to, + const void* from, size_t length, + bool user); + virtual void MemcpyPhysicalPage(phys_addr_t to, + phys_addr_t from); +}; + +#endif diff --git a/src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp b/src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp new file mode 100644 index 0000000000..f850829d4a --- /dev/null +++ b/src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp @@ -0,0 +1,597 @@ +/* + * Copyright 2022 Haiku, Inc. All Rights Reserved. + * Distributed under the terms of the MIT License. + */ +#include "VMSAv8TranslationMap.h" + +#include +#include +#include +#include + + +static constexpr uint64_t kPteAddrMask = (((1UL << 36) - 1) << 12); +static constexpr uint64_t kPteAttrMask = ~(kPteAddrMask | 0x3); + +static constexpr uint64_t kAttrSWDBM = (1UL << 55); +static constexpr uint64_t kAttrUXN = (1UL << 54); +static constexpr uint64_t kAttrPXN = (1UL << 53); +static constexpr uint64_t kAttrDBM = (1UL << 51); +static constexpr uint64_t kAttrNG = (1UL << 11); +static constexpr uint64_t kAttrAF = (1UL << 10); +static constexpr uint64_t kAttrSH1 = (1UL << 9); +static constexpr uint64_t kAttrSH0 = (1UL << 8); +static constexpr uint64_t kAttrAP2 = (1UL << 7); +static constexpr uint64_t kAttrAP1 = (1UL << 6); + +uint32_t VMSAv8TranslationMap::fHwFeature; +uint64_t VMSAv8TranslationMap::fMair; + + +VMSAv8TranslationMap::VMSAv8TranslationMap( + bool kernel, phys_addr_t pageTable, int pageBits, int vaBits, int minBlockLevel) + : + fIsKernel(kernel), + fPageTable(pageTable), + fPageBits(pageBits), + fVaBits(vaBits), + fMinBlockLevel(minBlockLevel) +{ + dprintf("VMSAv8TranslationMap\n"); + + fInitialLevel = CalcStartLevel(fVaBits, fPageBits); +} + + +VMSAv8TranslationMap::~VMSAv8TranslationMap() +{ + dprintf("~VMSAv8TranslationMap\n"); + + // FreeTable(fPageTable, fInitialLevel); +} + + +int +VMSAv8TranslationMap::CalcStartLevel(int vaBits, int pageBits) +{ + int level = 4; + + int bitsLeft = vaBits - pageBits; + while (bitsLeft > 0) { + int tableBits = pageBits - 3; + bitsLeft -= tableBits; + level--; + } + + ASSERT(level >= 0); + + return level; +} + + +bool +VMSAv8TranslationMap::Lock() +{ + recursive_lock_lock(&fLock); + return true; +} + + +void +VMSAv8TranslationMap::Unlock() +{ + if (recursive_lock_get_recursion(&fLock) == 1) { + // we're about to release it for the last time + Flush(); + } + recursive_lock_unlock(&fLock); +} + + +addr_t +VMSAv8TranslationMap::MappedSize() const +{ + panic("VMSAv8TranslationMap::MappedSize not implemented"); + return 0; +} + + +size_t +VMSAv8TranslationMap::MaxPagesNeededToMap(addr_t start, addr_t end) const +{ + size_t result = 0; + size_t size = end - start + 1; + + for (int i = fInitialLevel; i < 3; i++) { + int tableBits = fPageBits - 3; + int shift = tableBits * (3 - i) + fPageBits; + uint64_t entrySize = 1UL << shift; + + result += size / entrySize + 2; + } + + return result; +} + + +uint64_t* +VMSAv8TranslationMap::TableFromPa(phys_addr_t pa) +{ + return reinterpret_cast(KERNEL_PMAP_BASE + pa); +} + + +uint64_t +VMSAv8TranslationMap::MakeBlock(phys_addr_t pa, int level, uint64_t attr) +{ + ASSERT(level >= fMinBlockLevel && level < 4); + + return pa | attr | (level == 3 ? 0x3 : 0x1); +} + + +void +VMSAv8TranslationMap::FreeTable(phys_addr_t ptPa, int level) +{ + ASSERT(level < 3); + + if (level + 1 < 3) { + int tableBits = fPageBits - 3; + uint64_t tableSize = 1UL << tableBits; + + uint64_t* pt = TableFromPa(ptPa); + for (uint64_t i = 0; i < tableSize; i++) { + uint64_t pte = pt[i]; + if ((pte & 0x3) == 0x3) { + FreeTable(pte & kPteAddrMask, level + 1); + } + } + } + + vm_page* page = vm_lookup_page(ptPa >> fPageBits); + vm_page_set_state(page, PAGE_STATE_FREE); +} + + +phys_addr_t +VMSAv8TranslationMap::MakeTable( + phys_addr_t ptPa, int level, int index, vm_page_reservation* reservation) +{ + if (level == 3) + return 0; + + uint64_t* pte = &TableFromPa(ptPa)[index]; + vm_page* page = NULL; + +retry: + uint64_t oldPte = atomic_get64((int64*) pte); + + int type = oldPte & 0x3; + if (type == 0x3) { + return oldPte & kPteAddrMask; + } else if (reservation != NULL) { + if (page == NULL) + page = vm_page_allocate_page(reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); + phys_addr_t newTablePa = page->physical_page_number << fPageBits; + + if (type == 0x1) { + // If we're replacing existing block mapping convert it to pagetable + int tableBits = fPageBits - 3; + int shift = tableBits * (3 - (level + 1)) + fPageBits; + uint64_t entrySize = 1UL << shift; + uint64_t tableSize = 1UL << tableBits; + + uint64_t* newTable = TableFromPa(newTablePa); + uint64_t addr = oldPte & kPteAddrMask; + uint64_t attr = oldPte & kPteAttrMask; + + for (uint64_t i = 0; i < tableSize; i++) { + newTable[i] = MakeBlock(addr + i * entrySize, level + 1, attr); + } + } + + asm("dsb ish"); + + // FIXME: this is not enough on real hardware with SMP + if ((uint64_t) atomic_test_and_set64((int64*) pte, newTablePa | 0x3, oldPte) != oldPte) + goto retry; + + return newTablePa; + } + + return 0; +} + + +void +VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr_t pa, size_t size, + VMSAv8TranslationMap::VMAction action, uint64_t attr, vm_page_reservation* reservation) +{ + ASSERT(level < 4); + ASSERT(ptPa != 0); + ASSERT(reservation != NULL || action != VMAction::MAP); + + int tableBits = fPageBits - 3; + uint64_t tableMask = (1UL << tableBits) - 1; + + int shift = tableBits * (3 - level) + fPageBits; + uint64_t entrySize = 1UL << shift; + + uint64_t entryMask = entrySize - 1; + uint64_t nextVa = va; + uint64_t end = va + size; + int index; + + // Handle misaligned header that straddles entry boundary in next-level table + if ((va & entryMask) != 0) { + uint64_t aligned = (va & ~entryMask) + entrySize; + if (end > aligned) { + index = (va >> shift) & tableMask; + phys_addr_t table = MakeTable(ptPa, level, index, reservation); + MapRange(table, level + 1, va, pa, aligned - va, action, attr, reservation); + nextVa = aligned; + } + } + + // Handle fully aligned and appropriately sized chunks + while (nextVa + entrySize <= end) { + phys_addr_t targetPa = pa + (nextVa - va); + index = (nextVa >> shift) & tableMask; + + bool blockAllowed = false; + if (action == VMAction::MAP) + blockAllowed = (level >= fMinBlockLevel && (targetPa & entryMask) == 0); + if (action == VMAction::SET_ATTR || action == VMAction::CLEAR_FLAGS) + blockAllowed = (MakeTable(ptPa, level, index, NULL) == 0); + if (action == VMAction::UNMAP) + blockAllowed = true; + + if (blockAllowed) { + // Everything is aligned, we can make block mapping there + uint64_t* pte = &TableFromPa(ptPa)[index]; + + retry: + uint64_t oldPte = atomic_get64((int64*) pte); + + if (action == VMAction::MAP || (oldPte & 0x1) != 0) { + uint64_t newPte = 0; + if (action == VMAction::MAP) { + newPte = MakeBlock(targetPa, level, attr); + } else if (action == VMAction::SET_ATTR) { + newPte = MakeBlock(oldPte & kPteAddrMask, level, MoveAttrFlags(attr, oldPte)); + } else if (action == VMAction::CLEAR_FLAGS) { + newPte = MakeBlock(oldPte & kPteAddrMask, level, ClearAttrFlags(oldPte, attr)); + } else if (action == VMAction::UNMAP) { + newPte = 0; + tmp_pte = oldPte; + } + + // FIXME: this might not be enough on real hardware with SMP for some cases + if ((uint64_t) atomic_test_and_set64((int64*) pte, newPte, oldPte) != oldPte) + goto retry; + + if (level < 3 && (oldPte & 0x3) == 0x3) { + // If we're replacing existing pagetable clean it up + FreeTable(oldPte & kPteAddrMask, level); + } + } + } else { + // Otherwise handle mapping in next-level table + phys_addr_t table = MakeTable(ptPa, level, index, reservation); + MapRange(table, level + 1, nextVa, targetPa, entrySize, action, attr, reservation); + } + nextVa += entrySize; + } + + // Handle misaligned tail area (or entirety of small area) in next-level table + if (nextVa < end) { + index = (nextVa >> shift) & tableMask; + phys_addr_t table = MakeTable(ptPa, level, index, reservation); + MapRange( + table, level + 1, nextVa, pa + (nextVa - va), end - nextVa, action, attr, reservation); + } +} + + +uint8_t +VMSAv8TranslationMap::MairIndex(uint8_t type) +{ + for (int i = 0; i < 8; i++) + if (((fMair >> (i * 8)) & 0xff) == type) + return i; + + panic("MAIR entry not found"); + return 0; +} + + +uint64_t +VMSAv8TranslationMap::ClearAttrFlags(uint64_t attr, uint32 flags) +{ + attr &= kPteAttrMask; + + if ((flags & PAGE_ACCESSED) != 0) + attr &= ~kAttrAF; + + if ((flags & PAGE_MODIFIED) != 0 && (attr & kAttrSWDBM) != 0) + attr |= kAttrAP2; + + return attr; +} + + +uint64_t +VMSAv8TranslationMap::MoveAttrFlags(uint64_t newAttr, uint64_t oldAttr) +{ + if ((oldAttr & kAttrAF) != 0) + newAttr |= kAttrAF; + if (((newAttr & oldAttr) & kAttrSWDBM) != 0 && (oldAttr & kAttrAP2) == 0) + newAttr &= ~kAttrAP2; + + return newAttr; +} + + +uint64_t +VMSAv8TranslationMap::GetMemoryAttr(uint32 attributes, uint32 memoryType, bool isKernel) +{ + uint64_t attr = 0; + + if (!isKernel) + attr |= kAttrNG; + + if ((attributes & B_EXECUTE_AREA) == 0) + attr |= kAttrUXN; + if ((attributes & B_KERNEL_EXECUTE_AREA) == 0) + attr |= kAttrPXN; + + if ((attributes & B_READ_AREA) == 0) { + attr |= kAttrAP2; + if ((attributes & B_KERNEL_WRITE_AREA) != 0) + attr |= kAttrSWDBM; + } else { + attr |= kAttrAP2 | kAttrAP1; + if ((attributes & B_WRITE_AREA) != 0) + attr |= kAttrSWDBM; + } + + if ((fHwFeature & HW_DIRTY) != 0 && (attr & kAttrSWDBM)) + attr |= kAttrDBM; + + attr |= kAttrSH1 | kAttrSH0; + + attr |= MairIndex(MAIR_NORMAL_WB) << 2; + + return attr; +} + + +status_t +VMSAv8TranslationMap::Map(addr_t va, phys_addr_t pa, uint32 attributes, uint32 memoryType, + vm_page_reservation* reservation) +{ + ThreadCPUPinner pinner(thread_get_current_thread()); + + uint64_t pageMask = (1UL << fPageBits) - 1; + uint64_t vaMask = (1UL << fVaBits) - 1; + + ASSERT((va & pageMask) == 0); + ASSERT((pa & pageMask) == 0); + ASSERT(ValidateVa(va)); + + uint64_t attr = GetMemoryAttr(attributes, memoryType, fIsKernel); + + if (!fPageTable) { + vm_page* page = vm_page_allocate_page(reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); + fPageTable = page->physical_page_number << fPageBits; + } + + MapRange( + fPageTable, fInitialLevel, va & vaMask, pa, B_PAGE_SIZE, VMAction::MAP, attr, reservation); + + return B_OK; +} + + +status_t +VMSAv8TranslationMap::Unmap(addr_t start, addr_t end) +{ + ThreadCPUPinner pinner(thread_get_current_thread()); + + size_t size = end - start + 1; + + uint64_t pageMask = (1UL << fPageBits) - 1; + uint64_t vaMask = (1UL << fVaBits) - 1; + + ASSERT((start & pageMask) == 0); + ASSERT((size & pageMask) == 0); + ASSERT(ValidateVa(start)); + + MapRange(fPageTable, fInitialLevel, start & vaMask, 0, size, VMAction::UNMAP, 0, NULL); + + return B_OK; +} + + +status_t +VMSAv8TranslationMap::UnmapPage(VMArea* area, addr_t address, bool updatePageQueue) +{ + + ThreadCPUPinner pinner(thread_get_current_thread()); + RecursiveLocker locker(fLock); + + // TODO: replace this kludge + + phys_addr_t pa; + uint64_t pte; + if (!WalkTable(fPageTable, fInitialLevel, address, &pa, &pte)) + return B_ENTRY_NOT_FOUND; + + uint64_t vaMask = (1UL << fVaBits) - 1; + MapRange(fPageTable, fInitialLevel, address & vaMask, 0, B_PAGE_SIZE, VMAction::UNMAP, 0, NULL); + + pinner.Unlock(); + locker.Detach(); + PageUnmapped(area, pa >> fPageBits, (tmp_pte & kAttrAF) != 0, (tmp_pte & kAttrAP2) == 0, + updatePageQueue); + + return B_OK; +} + + +bool +VMSAv8TranslationMap::WalkTable( + phys_addr_t ptPa, int level, addr_t va, phys_addr_t* pa, uint64_t* rpte) +{ + int tableBits = fPageBits - 3; + uint64_t tableMask = (1UL << tableBits) - 1; + + int shift = tableBits * (3 - level) + fPageBits; + uint64_t entrySize = 1UL << shift; + uint64_t entryMask = entrySize - 1; + + int index = (va >> shift) & tableMask; + + uint64_t pte = TableFromPa(ptPa)[index]; + int type = pte & 0x3; + + if ((type & 0x1) == 0) + return false; + + uint64_t addr = pte & kPteAddrMask; + if (level < 3) { + if (type == 0x3) { + return WalkTable(addr, level + 1, va, pa, rpte); + } else { + *pa = addr | (va & entryMask); + *rpte = pte; + } + } else { + ASSERT(type == 0x3); + *pa = addr; + *rpte = pte; + } + + return true; +} + + +bool +VMSAv8TranslationMap::ValidateVa(addr_t va) +{ + uint64_t vaMask = (1UL << fVaBits) - 1; + bool kernelAddr = (va & (1UL << 63)) != 0; + if (kernelAddr != fIsKernel) + return false; + if ((va & ~vaMask) != (fIsKernel ? ~vaMask : 0)) + return false; + return true; +} + + +status_t +VMSAv8TranslationMap::Query(addr_t va, phys_addr_t* pa, uint32* flags) +{ + ThreadCPUPinner pinner(thread_get_current_thread()); + + ASSERT(ValidateVa(va)); + + uint64_t pte = 0; + bool ret = WalkTable(fPageTable, fInitialLevel, va, pa, &pte); + + uint32 result = 0; + + if (ret) { + result |= PAGE_PRESENT; + + if ((pte & kAttrAF) != 0) + result |= PAGE_ACCESSED; + if ((pte & kAttrAP2) == 0) + result |= PAGE_MODIFIED; + + if ((pte & kAttrUXN) == 0) + result |= B_EXECUTE_AREA; + if ((pte & kAttrPXN) == 0) + result |= B_KERNEL_EXECUTE_AREA; + + result |= B_KERNEL_READ_AREA; + + if ((pte & kAttrAP1) != 0) + result |= B_READ_AREA; + + if ((pte & kAttrAP2) == 0 || (pte & kAttrSWDBM) != 0) { + result |= B_KERNEL_WRITE_AREA; + + if ((pte & kAttrAP1) != 0) + result |= B_WRITE_AREA; + } + } + + *flags = result; + return B_OK; +} + + +status_t +VMSAv8TranslationMap::QueryInterrupt( + addr_t virtualAddress, phys_addr_t* _physicalAddress, uint32* _flags) +{ + return Query(virtualAddress, _physicalAddress, _flags); +} + + +status_t +VMSAv8TranslationMap::Protect(addr_t start, addr_t end, uint32 attributes, uint32 memoryType) +{ + ThreadCPUPinner pinner(thread_get_current_thread()); + + size_t size = end - start + 1; + + uint64_t pageMask = (1UL << fPageBits) - 1; + uint64_t vaMask = (1UL << fVaBits) - 1; + + ASSERT((start & pageMask) == 0); + ASSERT((size & pageMask) == 0); + ASSERT(ValidateVa(start)); + + uint64_t attr = GetMemoryAttr(attributes, memoryType, fIsKernel); + MapRange(fPageTable, fInitialLevel, start & vaMask, 0, size, VMAction::SET_ATTR, attr, NULL); + + return B_OK; +} + + +status_t +VMSAv8TranslationMap::ClearFlags(addr_t va, uint32 flags) +{ + ThreadCPUPinner pinner(thread_get_current_thread()); + + uint64_t pageMask = (1UL << fPageBits) - 1; + uint64_t vaMask = (1UL << fVaBits) - 1; + + ASSERT((va & pageMask) == 0); + ASSERT(ValidateVa(va)); + + MapRange( + fPageTable, fInitialLevel, va & vaMask, 0, B_PAGE_SIZE, VMAction::CLEAR_FLAGS, flags, NULL); + + return B_OK; +} + + +bool +VMSAv8TranslationMap::ClearAccessedAndModified( + VMArea* area, addr_t address, bool unmapIfUnaccessed, bool& _modified) +{ + panic("VMSAv8TranslationMap::ClearAccessedAndModified not implemented\n"); + return B_OK; +} + + +void +VMSAv8TranslationMap::Flush() +{ + ThreadCPUPinner pinner(thread_get_current_thread()); + + arch_cpu_global_TLB_invalidate(); +} diff --git a/src/system/kernel/arch/arm64/VMSAv8TranslationMap.h b/src/system/kernel/arch/arm64/VMSAv8TranslationMap.h new file mode 100644 index 0000000000..b9672e05d1 --- /dev/null +++ b/src/system/kernel/arch/arm64/VMSAv8TranslationMap.h @@ -0,0 +1,101 @@ +/* + * Copyright 2022 Haiku, Inc. All Rights Reserved. + * Distributed under the terms of the MIT License. + */ +#ifndef VMSA_V8_TRANSLATION_MAP_H +#define VMSA_V8_TRANSLATION_MAP_H + + +#include +#include + + +struct VMSAv8TranslationMap : public VMTranslationMap { +public: + VMSAv8TranslationMap( + bool kernel, phys_addr_t pageTable, int pageBits, int vaBits, int minBlockLevel); + ~VMSAv8TranslationMap(); + + virtual bool Lock(); + virtual void Unlock(); + + virtual addr_t MappedSize() const; + virtual size_t MaxPagesNeededToMap(addr_t start, + addr_t end) const; + + virtual status_t Map(addr_t virtualAddress, + phys_addr_t physicalAddress, + uint32 attributes, uint32 memoryType, + vm_page_reservation* reservation); + virtual status_t Unmap(addr_t start, addr_t end); + + virtual status_t UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue); +/* + virtual void UnmapPages(VMArea* area, addr_t base, + size_t size, bool updatePageQueue); + virtual void UnmapArea(VMArea* area, + bool deletingAddressSpace, + bool ignoreTopCachePageFlags); +*/ + + virtual status_t Query(addr_t virtualAddress, + phys_addr_t* _physicalAddress, + uint32* _flags); + virtual status_t QueryInterrupt(addr_t virtualAddress, + phys_addr_t* _physicalAddress, + uint32* _flags); + + virtual status_t Protect(addr_t base, addr_t top, + uint32 attributes, uint32 memoryType); + + virtual status_t ClearFlags(addr_t virtualAddress, + uint32 flags); + + virtual bool ClearAccessedAndModified( + VMArea* area, addr_t address, + bool unmapIfUnaccessed, + bool& _modified); + + virtual void Flush(); + + enum HWFeature { + HW_ACCESS = 0x1, + HW_DIRTY = 0x2 + }; + + static uint32_t fHwFeature; + static uint64_t fMair; + + static uint64_t GetMemoryAttr(uint32 attributes, uint32 memoryType, bool isKernel); + static int CalcStartLevel(int vaBits, int pageBits); + +private: + bool fIsKernel; + phys_addr_t fPageTable; + int fPageBits; + int fVaBits; + int fMinBlockLevel; + + int fInitialLevel; + + enum class VMAction { MAP, SET_ATTR, CLEAR_FLAGS, UNMAP }; + + uint64_t tmp_pte; // todo: remove kludge + +private: + static uint8_t MairIndex(uint8_t type); + uint64_t ClearAttrFlags(uint64_t attr, uint32 flags); + uint64_t MoveAttrFlags(uint64_t newAttr, uint64_t oldAttr); + bool ValidateVa(addr_t va); + uint64_t* TableFromPa(phys_addr_t pa); + uint64_t MakeBlock(phys_addr_t pa, int level, uint64_t attr); + void FreeTable(phys_addr_t ptPa, int level); + phys_addr_t MakeTable(phys_addr_t ptPa, int level, int index, vm_page_reservation* reservation); + void MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr_t pa, size_t size, + VMAction action, uint64_t attr, vm_page_reservation* reservation); + bool WalkTable(phys_addr_t ptPa, int level, addr_t va, phys_addr_t* pa, uint64_t* attr); +}; + + +#endif diff --git a/src/system/kernel/arch/arm64/arch_cpu.cpp b/src/system/kernel/arch/arm64/arch_cpu.cpp index 4ea96e9f95..c09c8dd7e1 100644 --- a/src/system/kernel/arch/arm64/arch_cpu.cpp +++ b/src/system/kernel/arch/arm64/arch_cpu.cpp @@ -76,22 +76,31 @@ arch_cpu_memory_write_barrier(void) void arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) { + arch_cpu_global_TLB_invalidate(); } void arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages) { + arch_cpu_global_TLB_invalidate(); } void arch_cpu_global_TLB_invalidate(void) { + asm( + "dsb ishst\n" + "tlbi vmalle1\n" + "dsb ish\n" + "isb\n" + ); } void arch_cpu_user_TLB_invalidate(void) { + arch_cpu_global_TLB_invalidate(); } diff --git a/src/system/kernel/arch/arm64/arch_vm.cpp b/src/system/kernel/arch/arm64/arch_vm.cpp index 6252151da6..82ca4a150e 100644 --- a/src/system/kernel/arch/arm64/arch_vm.cpp +++ b/src/system/kernel/arch/arm64/arch_vm.cpp @@ -4,70 +4,82 @@ */ #include -#include #include +#include +#include #include #include -#include status_t -arch_vm_init(kernel_args *args) +arch_vm_init(kernel_args* args) { + dprintf("arch_vm_init\n"); return B_OK; } status_t -arch_vm_init2(kernel_args *args) +arch_vm_init2(kernel_args* args) { + dprintf("arch_vm_init2\n"); return B_OK; } status_t -arch_vm_init_post_area(kernel_args *args) +arch_vm_init_post_area(kernel_args* args) { + dprintf("arch_vm_init_post_area\n"); return B_OK; } status_t -arch_vm_init_end(kernel_args *args) +arch_vm_init_end(kernel_args* args) { + dprintf("arch_vm_init_end\n"); return B_OK; } status_t -arch_vm_init_post_modules(kernel_args *args) +arch_vm_init_post_modules(kernel_args* args) { + dprintf("arch_vm_init_post_modules\n"); return B_OK; } void -arch_vm_aspace_swap(struct VMAddressSpace *from, struct VMAddressSpace *to) +arch_vm_aspace_swap(struct VMAddressSpace* from, struct VMAddressSpace* to) { + dprintf("arch_vm_aspace_swap\n"); } bool arch_vm_supports_protection(uint32 protection) { - return false; + // User-RO/Kernel-RW is not possible + if ((protection & B_READ_AREA) != 0 && (protection & B_WRITE_AREA) == 0 + && (protection & B_KERNEL_WRITE_AREA) != 0) { + return false; + } + + return true; } void -arch_vm_unset_memory_type(VMArea *area) +arch_vm_unset_memory_type(VMArea* area) { } status_t -arch_vm_set_memory_type(VMArea *area, phys_addr_t physicalBase, uint32 type) +arch_vm_set_memory_type(VMArea* area, phys_addr_t physicalBase, uint32 type) { return B_OK; } diff --git a/src/system/kernel/arch/arm64/arch_vm_translation_map.cpp b/src/system/kernel/arch/arm64/arch_vm_translation_map.cpp index 5d4735b1be..1cd3b83d24 100644 --- a/src/system/kernel/arch/arm64/arch_vm_translation_map.cpp +++ b/src/system/kernel/arch/arm64/arch_vm_translation_map.cpp @@ -2,50 +2,193 @@ * Copyright 2019 Haiku, Inc. All Rights Reserved. * Distributed under the terms of the MIT License. */ + + #include #include +#include +#include + +#include "PMAPPhysicalPageMapper.h" +#include "VMSAv8TranslationMap.h" + +static char sPhysicalPageMapperData[sizeof(PMAPPhysicalPageMapper)]; status_t arch_vm_translation_map_create_map(bool kernel, VMTranslationMap** _map) { + phys_addr_t pt = 0; + if (kernel) { + pt = READ_SPECIALREG(TTBR1_EL1); + } else { + panic("arch_vm_translation_map_create_map user not implemented"); + } + + *_map = new (std::nothrow) VMSAv8TranslationMap(kernel, pt, 12, 48, 1); + + if (*_map == NULL) + return B_NO_MEMORY; + return B_OK; } status_t -arch_vm_translation_map_init(kernel_args *args, - VMPhysicalPageMapper** _physicalPageMapper) +arch_vm_translation_map_init(kernel_args* args, VMPhysicalPageMapper** _physicalPageMapper) { + dprintf("arch_vm_translation_map_init\n"); + + // nuke TTBR0 mapping, we use identity mapping in kernel space at KERNEL_PMAP_BASE + memset((void*) READ_SPECIALREG(TTBR0_EL1), 0, B_PAGE_SIZE); + + uint64_t tcr = READ_SPECIALREG(TCR_EL1); + uint32_t t0sz = tcr & 0x1f; + uint32_t t1sz = (tcr >> 16) & 0x1f; + uint32_t tg0 = (tcr >> 14) & 0x3; + uint32_t tg1 = (tcr >> 30) & 0x3; + uint64_t ttbr0 = READ_SPECIALREG(TTBR0_EL1); + uint64_t ttbr1 = READ_SPECIALREG(TTBR1_EL1); + uint64_t mair = READ_SPECIALREG(MAIR_EL1); + uint64_t mmfr1 = READ_SPECIALREG(ID_AA64MMFR1_EL1); + uint64_t sctlr = READ_SPECIALREG(SCTLR_EL1); + + uint64_t hafdbs = ID_AA64MMFR1_HAFDBS(mmfr1); + if (hafdbs == ID_AA64MMFR1_HAFDBS_AF) { + VMSAv8TranslationMap::fHwFeature = VMSAv8TranslationMap::HW_ACCESS; + tcr |= (1UL << 39); + } + if (hafdbs == ID_AA64MMFR1_HAFDBS_AF_DBS) { + VMSAv8TranslationMap::fHwFeature + = VMSAv8TranslationMap::HW_ACCESS | VMSAv8TranslationMap::HW_DIRTY; + tcr |= (1UL << 40) | (1UL << 39); + } + + VMSAv8TranslationMap::fMair = mair; + + WRITE_SPECIALREG(TCR_EL1, tcr); + + dprintf("vm config: MMFR1: %lx, TCR: %lx\nTTBR0: %lx, TTBR1: %lx\nT0SZ: %u, T1SZ: %u, TG0: %u, " + "TG1: %u, MAIR: %lx, SCTLR: %lx\n", + mmfr1, tcr, ttbr0, ttbr1, t0sz, t1sz, tg0, tg1, mair, sctlr); + + *_physicalPageMapper = new (&sPhysicalPageMapperData) PMAPPhysicalPageMapper(); + return B_OK; } status_t -arch_vm_translation_map_init_post_sem(kernel_args *args) +arch_vm_translation_map_init_post_sem(kernel_args* args) { + dprintf("arch_vm_translation_map_init_post_sem\n"); return B_OK; } status_t -arch_vm_translation_map_init_post_area(kernel_args *args) +arch_vm_translation_map_init_post_area(kernel_args* args) { + dprintf("arch_vm_translation_map_init_post_area\n"); + + // Create an area covering the physical map area. + void* address = (void*) KERNEL_PMAP_BASE; + area_id area = vm_create_null_area(VMAddressSpace::KernelID(), "physical map area", &address, + B_EXACT_ADDRESS, KERNEL_PMAP_SIZE, 0); + return B_OK; } +// TODO: reuse some bits from VMSAv8TranslationMap + +static constexpr uint64_t kPteAddrMask = (((1UL << 36) - 1) << 12); +static constexpr uint64_t kPteAttrMask = ~(kPteAddrMask | 0x3); + +static uint64_t page_bits = 12; +static uint64_t tsz = 16; + + +static uint64_t* +TableFromPa(phys_addr_t pa) +{ + return reinterpret_cast(KERNEL_PMAP_BASE + pa); +} + + +static void +map_page_early(phys_addr_t ptPa, int level, addr_t va, phys_addr_t pa, + phys_addr_t (*get_free_page)(kernel_args*), kernel_args* args) +{ + int tableBits = page_bits - 3; + uint64_t tableMask = (1UL << tableBits) - 1; + + int shift = tableBits * (3 - level) + page_bits; + + int index = (va >> shift) & tableMask; + uint64_t* pte = &TableFromPa(ptPa)[index]; + + if (level == 3) { + atomic_set64((int64*) pte, pa | 0x3); + asm("dsb ish"); + } else { + uint64_t pteVal = atomic_get64((int64*) pte); + int type = pteVal & 0x3; + + phys_addr_t table; + if (type == 0x3) { + table = pteVal & kPteAddrMask; + } else { + table = get_free_page(args) << page_bits; + dprintf("early: pulling page %lx\n", table); + uint64_t* newTableVa = TableFromPa(table); + + if (type == 0x1) { + int shift = tableBits * (3 - (level + 1)) + page_bits; + int entrySize = 1UL << shift; + + for (int i = 0; i < (1 << tableBits); i++) + newTableVa[i] = pteVal + i * entrySize; + } else { + memset(newTableVa, 0, 1 << page_bits); + } + + asm("dsb ish"); + + atomic_set64((int64*) pte, table | 0x3); + } + + map_page_early(table, level + 1, va, pa, get_free_page, args); + } +} + status_t -arch_vm_translation_map_early_map(kernel_args *args, addr_t va, phys_addr_t pa, - uint8 attributes, phys_addr_t (*get_free_page)(kernel_args *)) +arch_vm_translation_map_early_map(kernel_args* args, addr_t va, phys_addr_t pa, uint8 attributes, + phys_addr_t (*get_free_page)(kernel_args*)) { + int va_bits = 64 - tsz; + uint64_t va_mask = (1UL << va_bits) - 1; + ASSERT((va & ~va_mask) == ~va_mask); + + phys_addr_t ptPa = READ_SPECIALREG(TTBR1_EL1); + int level = VMSAv8TranslationMap::CalcStartLevel(va_bits, page_bits); + va &= va_mask; + pa |= VMSAv8TranslationMap::GetMemoryAttr(attributes, 0, true); + + map_page_early(ptPa, level, va, pa, get_free_page, args); + return B_OK; } bool -arch_vm_translation_map_is_kernel_page_accessible(addr_t virtualAddress, - uint32 protection) +arch_vm_translation_map_is_kernel_page_accessible(addr_t va, uint32 protection) { - return false; + if (protection & B_KERNEL_WRITE_AREA) { + asm("at s1e1w, %0" : : "r"((uint64_t) va)); + return (READ_SPECIALREG(PAR_EL1) & PAR_F) == 0; + } else { + asm("at s1e1r, %0" : : "r"((uint64_t) va)); + return (READ_SPECIALREG(PAR_EL1) & PAR_F) == 0; + } }