i2p/haiku-beta6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

arm64: Reimplement ProcessRange and use it to implement Map().

Browse Source 
* Fully implement break-before-make PTE updating, fixing the
  race condition that necessitates it.
* Incorporate align-down logic into the normal flow of the table
  traversal.
* Drop support for huge pages and for table freeing for now. They
  can be reimplemented once we have a fully functional baseline.
* Further adoption of the new ProcessRange coming in subsequent
  patches.

Change-Id: I852c63aaeab48b847b4bb99b328dfe071d27bb29
Reviewed-on: https://review.haiku-os.org/c/haiku/+/8139
Reviewed-by: Fredrik Holmqvist <[email protected]>
This commit is contained in:
Owen Anderson
2024-09-01 22:48:50 +00:00
committed by waddlesplash
parent aa4f1de9ad
commit baf574c96a
2 changed files with 115 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/system/kernel/arch/arm64/VMSAv8TranslationMap.cpp
+109 -12
View File
@@ -4,6 +4,7 @@
*/
#include "VMSAv8TranslationMap.h"
#include <algorithm>
#include <util/AutoLock.h>
#include <util/ThreadAutoLock.h>
#include <vm/vm_page.h>
@@ -12,10 +13,12 @@
static constexpr uint64_t kPteAddrMask = (((1UL << 36) - 1) << 12);
static constexpr uint64_t kPteAttrMask = ~(kPteAddrMask | 0x3);
static constexpr uint64_t kPteTLBCompatMask = (kPteAddrMask | (0x3 << 2) | (0x3 << 8));
static constexpr uint64_t kPteValidMask = 0x1;
static constexpr uint64_t kPteTypeMask = 0x3;
static constexpr uint64_t kPteTypeL012Table = 0x3;
static constexpr uint64_t kPteTypeL012Block = 0x1;
static constexpr uint64_t kPteTypeL12Block = 0x1;
static constexpr uint64_t kPteTypeL3Page = 0x3;
static constexpr uint64_t kAttrSWDBM = (1UL << 55);
@@ -295,8 +298,8 @@ VMSAv8TranslationMap::FreeTable(phys_addr_t ptPa, uint64_t va, int level,
// Returns the physical address of the new table, or the address of the existing
// one if the PTE is already filled.
phys_addr_t
VMSAv8TranslationMap::MakeTable(
phys_addr_t ptPa, int level, int index, vm_page_reservation* reservation)
VMSAv8TranslationMap::GetOrMakeTable(phys_addr_t ptPa, int level, int index,
vm_page_reservation* reservation)
{
ASSERT(level < 3);
@@ -315,7 +318,7 @@ VMSAv8TranslationMap::MakeTable(
// We only create mappings at the final level so we don't need to handle
// splitting block mappings
ASSERT(type != kPteTypeL012Block);
ASSERT(type != kPteTypeL12Block);
// Ensure that writes to page being attached have completed
asm("dsb ishst");
@@ -340,6 +343,71 @@ VMSAv8TranslationMap::MakeTable(
}
void
VMSAv8TranslationMap::FlushVAFromTLBByASID(addr_t va)
{
SpinLocker locker(sAsidLock);
if (fASID != 0) {
asm("tlbi vae1is, %0" ::"r"(((va >> 12) & kTLBIMask) | (uint64_t(fASID) << 48)));
asm("dsb ish"); // Wait for TLB flush to complete
}
}
void
VMSAv8TranslationMap::PerformPteBreakBeforeMake(uint64_t* ptePtr, addr_t va)
{
atomic_set64((int64*)ptePtr, 0);
asm("dsb ishst"); // Ensure PTE write completed
FlushVAFromTLBByASID(va);
}
template<typename UpdatePte>
void
VMSAv8TranslationMap::ProcessRange(phys_addr_t ptPa, int level, addr_t va, size_t size,
vm_page_reservation* reservation, UpdatePte&& updatePte)
{
ASSERT(level < 4);
ASSERT(ptPa != 0);
int tableBits = fPageBits - 3;
uint64_t tableMask = (1UL << tableBits) - 1;
int shift = tableBits * (3 - level) + fPageBits;
uint64_t entrySize = 1UL << shift;
uint64_t entryMask = entrySize - 1;
uint64_t alignedDownVa = va & ~entryMask;
uint64_t alignedUpEnd = (va + size + (entrySize - 1)) & ~entryMask;
if (level == 3)
ASSERT(alignedDownVa == va);
for (uint64_t effectiveVa = alignedDownVa; effectiveVa < alignedUpEnd;
effectiveVa += entrySize) {
int index = (effectiveVa >> shift) & tableMask;
uint64_t* ptePtr = TableFromPa(ptPa) + index;
if (level == 3) {
updatePte(ptePtr, effectiveVa);
} else {
phys_addr_t subTable = GetOrMakeTable(ptPa, level, index, reservation);
// When reservation is null, we can't create a new subtable. This can be intentional,
// for example when called from Unmap().
if (subTable == 0)
continue;
uint64_t subVa = std::max(effectiveVa, va);
size_t subSize = std::min(size_t(entrySize - (subVa & entryMask)), size);
ProcessRange(subTable, level + 1, subVa, subSize, reservation, updatePte);
size -= subSize;
}
}
}
void
VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr_t pa, size_t size,
VMSAv8TranslationMap::VMAction action, uint64_t attr, vm_page_reservation* reservation)
@@ -364,7 +432,7 @@ VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr
uint64_t aligned = (va & ~entryMask) + entrySize;
if (end > aligned) {
index = (va >> shift) & tableMask;
phys_addr_t table = MakeTable(ptPa, level, index, reservation);
phys_addr_t table = GetOrMakeTable(ptPa, level, index, reservation);
MapRange(table, level + 1, va, pa, aligned - va, action, attr, reservation);
nextVa = aligned;
}
@@ -379,7 +447,7 @@ VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr
if (action == VMAction::MAP)
blockAllowed = (level >= fMinBlockLevel && (targetPa & entryMask) == 0);
if (action == VMAction::SET_ATTR || action == VMAction::CLEAR_FLAGS)
blockAllowed = (MakeTable(ptPa, level, index, NULL) == 0);
blockAllowed = (GetOrMakeTable(ptPa, level, index, NULL) == 0);
if (action == VMAction::UNMAP)
blockAllowed = true;
@@ -415,7 +483,7 @@ VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr
}
} else {
// Otherwise handle mapping in next-level table
phys_addr_t table = MakeTable(ptPa, level, index, reservation);
phys_addr_t table = GetOrMakeTable(ptPa, level, index, reservation);
MapRange(table, level + 1, nextVa, targetPa, entrySize, action, attr, reservation);
}
nextVa += entrySize;
@@ -424,7 +492,7 @@ VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr
// Handle misaligned tail area (or entirety of small area) in next-level table
if (nextVa < end) {
index = (nextVa >> shift) & tableMask;
phys_addr_t table = MakeTable(ptPa, level, index, reservation);
phys_addr_t table = GetOrMakeTable(ptPa, level, index, reservation);
MapRange(
table, level + 1, nextVa, pa + (nextVa - va), end - nextVa, action, attr, reservation);
}
@@ -541,13 +609,32 @@ VMSAv8TranslationMap::Map(addr_t va, phys_addr_t pa, uint32 attributes, uint32 m
uint64_t attr = GetMemoryAttr(attributes, memoryType, fIsKernel);
if (!fPageTable) {
// During first mapping we need to allocate root table
if (fPageTable == 0) {
vm_page* page = vm_page_allocate_page(reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
DEBUG_PAGE_ACCESS_END(page);
fPageTable = page->physical_page_number << fPageBits;
}
MapRange(
fPageTable, fInitialLevel, va & vaMask, pa, B_PAGE_SIZE, VMAction::MAP, attr, reservation);
ProcessRange(fPageTable, 0, va & vaMask, B_PAGE_SIZE, reservation,
[=](uint64_t* ptePtr, uint64_t effectiveVa) {
phys_addr_t effectivePa = effectiveVa - (va & vaMask);
uint64_t oldPte = atomic_get64((int64*)ptePtr);
uint64_t newPte = effectivePa | attr | kPteTypeL3Page;
if (newPte == oldPte)
return;
if ((newPte & kPteValidMask) != 0 && (oldPte & kPteValidMask) != 0) {
// ARM64 requires "break-before-make". We must set the PTE to an invalid
// entry and flush the TLB as appropriate before we can write the new PTE.
PerformPteBreakBeforeMake(ptePtr, effectiveVa);
}
// Install the new PTE
atomic_set64((int64*)ptePtr, newPte);
asm("dsb ishst"); // Ensure PTE write completed
});
return B_OK;
}
@@ -567,7 +654,17 @@ VMSAv8TranslationMap::Unmap(addr_t start, addr_t end)
ASSERT((size & pageMask) == 0);
ASSERT(ValidateVa(start));
MapRange(fPageTable, fInitialLevel, start & vaMask, 0, size, VMAction::UNMAP, 0, NULL);
if (fPageTable == 0)
return B_OK;
ProcessRange(fPageTable, 0, start & vaMask, size, nullptr,
[=](uint64_t* ptePtr, uint64_t effectiveVa) {
uint64_t oldPte = atomic_and64((int64_t*)ptePtr, ~kPteValidMask);
if ((oldPte & kPteValidMask) != 0) {
asm("dsb ishst"); // Ensure PTE write completed
FlushVAFromTLBByASID(effectiveVa);
}
});
return B_OK;
}
src/system/kernel/arch/arm64/VMSAv8TranslationMap.h
+6 -1
View File
@@ -102,9 +102,14 @@ private:
uint64_t MakeBlock(phys_addr_t pa, int level, uint64_t attr);
template<typename EntryRemoved>
void FreeTable(phys_addr_t ptPa, uint64_t va, int level, EntryRemoved &&entryRemoved);
phys_addr_t MakeTable(phys_addr_t ptPa, int level, int index, vm_page_reservation* reservation);
phys_addr_t GetOrMakeTable(phys_addr_t ptPa, int level, int index, vm_page_reservation* reservation);
void MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr_t pa, size_t size,
VMAction action, uint64_t attr, vm_page_reservation* reservation);
template<typename UpdatePte>
void ProcessRange(phys_addr_t ptPa, int level, addr_t va, size_t size,
vm_page_reservation* reservation, UpdatePte &&updatePte);
void PerformPteBreakBeforeMake(uint64_t* ptePtr, addr_t va);
void FlushVAFromTLBByASID(addr_t va);
bool WalkTable(phys_addr_t ptPa, int level, addr_t va, phys_addr_t* pa, uint64_t* attr);
};