diff --git a/src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp b/src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp index 7e44b768e7..be19285a36 100644 --- a/src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp +++ b/src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp @@ -4,6 +4,7 @@ */ #include "VMSAv8TranslationMap.h" +#include #include #include #include @@ -12,10 +13,12 @@ static constexpr uint64_t kPteAddrMask = (((1UL << 36) - 1) << 12); static constexpr uint64_t kPteAttrMask = ~(kPteAddrMask | 0x3); +static constexpr uint64_t kPteTLBCompatMask = (kPteAddrMask | (0x3 << 2) | (0x3 << 8)); +static constexpr uint64_t kPteValidMask = 0x1; static constexpr uint64_t kPteTypeMask = 0x3; static constexpr uint64_t kPteTypeL012Table = 0x3; -static constexpr uint64_t kPteTypeL012Block = 0x1; +static constexpr uint64_t kPteTypeL12Block = 0x1; static constexpr uint64_t kPteTypeL3Page = 0x3; static constexpr uint64_t kAttrSWDBM = (1UL << 55); @@ -295,8 +298,8 @@ VMSAv8TranslationMap::FreeTable(phys_addr_t ptPa, uint64_t va, int level, // Returns the physical address of the new table, or the address of the existing // one if the PTE is already filled. phys_addr_t -VMSAv8TranslationMap::MakeTable( - phys_addr_t ptPa, int level, int index, vm_page_reservation* reservation) +VMSAv8TranslationMap::GetOrMakeTable(phys_addr_t ptPa, int level, int index, + vm_page_reservation* reservation) { ASSERT(level < 3); @@ -315,7 +318,7 @@ VMSAv8TranslationMap::MakeTable( // We only create mappings at the final level so we don't need to handle // splitting block mappings - ASSERT(type != kPteTypeL012Block); + ASSERT(type != kPteTypeL12Block); // Ensure that writes to page being attached have completed asm("dsb ishst"); @@ -340,6 +343,71 @@ VMSAv8TranslationMap::MakeTable( } +void +VMSAv8TranslationMap::FlushVAFromTLBByASID(addr_t va) +{ + SpinLocker locker(sAsidLock); + if (fASID != 0) { + asm("tlbi vae1is, %0" ::"r"(((va >> 12) & kTLBIMask) | (uint64_t(fASID) << 48))); + asm("dsb ish"); // Wait for TLB flush to complete + } +} + + +void +VMSAv8TranslationMap::PerformPteBreakBeforeMake(uint64_t* ptePtr, addr_t va) +{ + atomic_set64((int64*)ptePtr, 0); + asm("dsb ishst"); // Ensure PTE write completed + FlushVAFromTLBByASID(va); +} + + +template +void +VMSAv8TranslationMap::ProcessRange(phys_addr_t ptPa, int level, addr_t va, size_t size, + vm_page_reservation* reservation, UpdatePte&& updatePte) +{ + ASSERT(level < 4); + ASSERT(ptPa != 0); + + int tableBits = fPageBits - 3; + uint64_t tableMask = (1UL << tableBits) - 1; + + int shift = tableBits * (3 - level) + fPageBits; + uint64_t entrySize = 1UL << shift; + uint64_t entryMask = entrySize - 1; + + uint64_t alignedDownVa = va & ~entryMask; + uint64_t alignedUpEnd = (va + size + (entrySize - 1)) & ~entryMask; + if (level == 3) + ASSERT(alignedDownVa == va); + + for (uint64_t effectiveVa = alignedDownVa; effectiveVa < alignedUpEnd; + effectiveVa += entrySize) { + int index = (effectiveVa >> shift) & tableMask; + uint64_t* ptePtr = TableFromPa(ptPa) + index; + + if (level == 3) { + updatePte(ptePtr, effectiveVa); + } else { + phys_addr_t subTable = GetOrMakeTable(ptPa, level, index, reservation); + + // When reservation is null, we can't create a new subtable. This can be intentional, + // for example when called from Unmap(). + if (subTable == 0) + continue; + + uint64_t subVa = std::max(effectiveVa, va); + size_t subSize = std::min(size_t(entrySize - (subVa & entryMask)), size); + ProcessRange(subTable, level + 1, subVa, subSize, reservation, updatePte); + + size -= subSize; + } + } +} + + void VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr_t pa, size_t size, VMSAv8TranslationMap::VMAction action, uint64_t attr, vm_page_reservation* reservation) @@ -364,7 +432,7 @@ VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr uint64_t aligned = (va & ~entryMask) + entrySize; if (end > aligned) { index = (va >> shift) & tableMask; - phys_addr_t table = MakeTable(ptPa, level, index, reservation); + phys_addr_t table = GetOrMakeTable(ptPa, level, index, reservation); MapRange(table, level + 1, va, pa, aligned - va, action, attr, reservation); nextVa = aligned; } @@ -379,7 +447,7 @@ VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr if (action == VMAction::MAP) blockAllowed = (level >= fMinBlockLevel && (targetPa & entryMask) == 0); if (action == VMAction::SET_ATTR || action == VMAction::CLEAR_FLAGS) - blockAllowed = (MakeTable(ptPa, level, index, NULL) == 0); + blockAllowed = (GetOrMakeTable(ptPa, level, index, NULL) == 0); if (action == VMAction::UNMAP) blockAllowed = true; @@ -415,7 +483,7 @@ VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr } } else { // Otherwise handle mapping in next-level table - phys_addr_t table = MakeTable(ptPa, level, index, reservation); + phys_addr_t table = GetOrMakeTable(ptPa, level, index, reservation); MapRange(table, level + 1, nextVa, targetPa, entrySize, action, attr, reservation); } nextVa += entrySize; @@ -424,7 +492,7 @@ VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr // Handle misaligned tail area (or entirety of small area) in next-level table if (nextVa < end) { index = (nextVa >> shift) & tableMask; - phys_addr_t table = MakeTable(ptPa, level, index, reservation); + phys_addr_t table = GetOrMakeTable(ptPa, level, index, reservation); MapRange( table, level + 1, nextVa, pa + (nextVa - va), end - nextVa, action, attr, reservation); } @@ -541,13 +609,32 @@ VMSAv8TranslationMap::Map(addr_t va, phys_addr_t pa, uint32 attributes, uint32 m uint64_t attr = GetMemoryAttr(attributes, memoryType, fIsKernel); - if (!fPageTable) { + // During first mapping we need to allocate root table + if (fPageTable == 0) { vm_page* page = vm_page_allocate_page(reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); + DEBUG_PAGE_ACCESS_END(page); fPageTable = page->physical_page_number << fPageBits; } - MapRange( - fPageTable, fInitialLevel, va & vaMask, pa, B_PAGE_SIZE, VMAction::MAP, attr, reservation); + ProcessRange(fPageTable, 0, va & vaMask, B_PAGE_SIZE, reservation, + [=](uint64_t* ptePtr, uint64_t effectiveVa) { + phys_addr_t effectivePa = effectiveVa - (va & vaMask); + uint64_t oldPte = atomic_get64((int64*)ptePtr); + uint64_t newPte = effectivePa | attr | kPteTypeL3Page; + + if (newPte == oldPte) + return; + + if ((newPte & kPteValidMask) != 0 && (oldPte & kPteValidMask) != 0) { + // ARM64 requires "break-before-make". We must set the PTE to an invalid + // entry and flush the TLB as appropriate before we can write the new PTE. + PerformPteBreakBeforeMake(ptePtr, effectiveVa); + } + + // Install the new PTE + atomic_set64((int64*)ptePtr, newPte); + asm("dsb ishst"); // Ensure PTE write completed + }); return B_OK; } @@ -567,7 +654,17 @@ VMSAv8TranslationMap::Unmap(addr_t start, addr_t end) ASSERT((size & pageMask) == 0); ASSERT(ValidateVa(start)); - MapRange(fPageTable, fInitialLevel, start & vaMask, 0, size, VMAction::UNMAP, 0, NULL); + if (fPageTable == 0) + return B_OK; + + ProcessRange(fPageTable, 0, start & vaMask, size, nullptr, + [=](uint64_t* ptePtr, uint64_t effectiveVa) { + uint64_t oldPte = atomic_and64((int64_t*)ptePtr, ~kPteValidMask); + if ((oldPte & kPteValidMask) != 0) { + asm("dsb ishst"); // Ensure PTE write completed + FlushVAFromTLBByASID(effectiveVa); + } + }); return B_OK; } diff --git a/src/system/kernel/arch/arm64/VMSAv8TranslationMap.h b/src/system/kernel/arch/arm64/VMSAv8TranslationMap.h index bdc2750a5d..35e8937fda 100644 --- a/src/system/kernel/arch/arm64/VMSAv8TranslationMap.h +++ b/src/system/kernel/arch/arm64/VMSAv8TranslationMap.h @@ -102,9 +102,14 @@ private: uint64_t MakeBlock(phys_addr_t pa, int level, uint64_t attr); template void FreeTable(phys_addr_t ptPa, uint64_t va, int level, EntryRemoved &&entryRemoved); - phys_addr_t MakeTable(phys_addr_t ptPa, int level, int index, vm_page_reservation* reservation); + phys_addr_t GetOrMakeTable(phys_addr_t ptPa, int level, int index, vm_page_reservation* reservation); void MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr_t pa, size_t size, VMAction action, uint64_t attr, vm_page_reservation* reservation); + template + void ProcessRange(phys_addr_t ptPa, int level, addr_t va, size_t size, + vm_page_reservation* reservation, UpdatePte &&updatePte); + void PerformPteBreakBeforeMake(uint64_t* ptePtr, addr_t va); + void FlushVAFromTLBByASID(addr_t va); bool WalkTable(phys_addr_t ptPa, int level, addr_t va, phys_addr_t* pa, uint64_t* attr); };