i2p/haiku-beta6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvmm: Implement necessary functions and classes for Haiku.

Browse Source 
Intel VMX only.

Co-authored-by: Augustin Cavalier <[email protected]>
This commit is contained in:
Augustin Cavalier
2026-06-03 23:04:56 -04:00
parent 8b7bc49ce0
commit 099de13932
31 changed files with 2346 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
headers/libs/nvmm/nvmm.h
+4
View File
@@ -38,6 +38,10 @@
#elif defined(__DragonFly__)
#include <dev/virtual/nvmm/nvmm.h>
#include <dev/virtual/nvmm/nvmm_ioctl.h>
#elif defined(__HAIKU__)
#include <driver/nvmm.h>
#include <driver/nvmm_ioctl.h>
#include <driver/nvmm_bitops.h>
#else
#error "Unsupported OS."
#endif
src/add-ons/kernel/drivers/Jamfile
+1
View File
@@ -13,6 +13,7 @@ SubInclude HAIKU_TOP src add-ons kernel drivers joystick ;
SubInclude HAIKU_TOP src add-ons kernel drivers graphics ;
SubInclude HAIKU_TOP src add-ons kernel drivers midi ;
SubInclude HAIKU_TOP src add-ons kernel drivers misc ;
SubInclude HAIKU_TOP src add-ons kernel drivers nvmm ;
SubInclude HAIKU_TOP src add-ons kernel drivers network ;
SubInclude HAIKU_TOP src add-ons kernel drivers ports ;
SubInclude HAIKU_TOP src add-ons kernel drivers power ;
src/add-ons/kernel/drivers/nvmm/Jamfile
+27
View File
@@ -0,0 +1,27 @@
SubDir HAIKU_TOP src add-ons kernel drivers nvmm ;
UseHeaders [ FDirName $(HAIKU_TOP) headers compatibility bsd ] : true ;
UseHeaders [ FDirName $(SUBDIR) include ] : true ;
UsePrivateKernelHeaders ;
UsePrivateHeaders kernel ;
SEARCH_SOURCE += [ FDirName $(SUBDIR) x86 ] ;
DEFINES += _KERNEL=1 ;
# TranslationMaps
UseHeaders [ FDirName $(HAIKU_TOP) src system kernel arch x86 ] : true ;
# VMUserAddressSpace
UseHeaders [ FDirName $(HAIKU_TOP) src system kernel ] : true ;
KernelAddon nvmm :
nvmm.c
nvmm_x86.c
nvmm_x86_vmx.c
nvmm_x86_vmxfunc.S
nvmm_x86_svm.c
nvmm_haiku.cpp
VMVirtualAddressSpace.cpp
;
src/add-ons/kernel/drivers/nvmm/VMVirtualAddressSpace.cpp
+15
View File
@@ -0,0 +1,15 @@
/*
* Copyright 2026, Haiku, Inc. All rights reserved.
* Distributed under the terms of the MIT License.
*/
#include "VMVirtualAddressSpace.h"
VMVirtualAddressSpace::VMVirtualAddressSpace(VMTranslationMap* map,
addr_t base, size_t size)
:
VMUserAddressSpace(-1, base, size)
{
fTranslationMap = map;
}
src/add-ons/kernel/drivers/nvmm/VMVirtualAddressSpace.h
+19
View File
@@ -0,0 +1,19 @@
/*
* Copyright 2026, Haiku, Inc. All rights reserved.
* Distributed under the terms of the MIT License.
*/
#ifndef VM_VIRTUAL_ADDRESS_SPACE_H
#define VM_VIRTUAL_ADDRESS_SPACE_H
#include <vm/VMUserAddressSpace.h>
struct VMVirtualAddressSpace final : VMUserAddressSpace {
public:
VMVirtualAddressSpace(VMTranslationMap* map,
addr_t base, size_t size);
};
#endif /* VM_VIRTUAL_ADDRESS_SPACE_H */
src/add-ons/kernel/drivers/nvmm/nvmm.c
+56 -2
View File
@@ -26,11 +26,13 @@
* SUCH DAMAGE.
*/
#if defined(__NetBSD__) || defined(__DragonFly__)
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/mman.h>
#endif
#include "nvmm.h"
#include "nvmm_internal.h"
@@ -51,7 +53,6 @@ const struct nvmm_impl *nvmm_impl __read_mostly = NULL;
struct nvmm_owner nvmm_root_owner;
/* -------------------------------------------------------------------------- */
static int
nvmm_machine_alloc(struct nvmm_machine **ret)
{
@@ -68,7 +69,11 @@ nvmm_machine_alloc(struct nvmm_machine **ret)
}
mach->present = true;
#if defined(__HAIKU__)
time(&mach->time);
#else
mach->time = time_second;
#endif
*ret = mach;
os_atomic_inc_uint(&nmachines);
return 0;
@@ -151,7 +156,9 @@ nvmm_vcpu_alloc(struct nvmm_machine *mach, nvmm_cpuid_t cpuid,
static void
nvmm_vcpu_free(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
{
#if !defined(__HAIKU__)
OS_ASSERT(os_mtx_owned(&vcpu->lock));
#endif
vcpu->present = false;
if (vcpu->comm != NULL) {
os_vmobj_unmap(os_kernel_map, (vaddr_t)vcpu->comm,
@@ -381,12 +388,21 @@ nvmm_vcpu_create(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_create *args)
if (error)
goto out;
#if defined(__HAIKU__)
/* Map the comm page on the kernel side, as wired. */
error = os_vmobj_map(os_kernel_map, (vaddr_t *)&vcpu->comm,
NVMM_COMM_PAGE_SIZE, mach->commvmobj,
args->cpuid * NVMM_COMM_PAGE_SIZE, true /* wired */,
false /* !fixed */, true /* shared */, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
#else
/* Map the comm page on the kernel side, as wired. */
error = os_vmobj_map(os_kernel_map, (vaddr_t *)&vcpu->comm,
NVMM_COMM_PAGE_SIZE, mach->commvmobj,
args->cpuid * NVMM_COMM_PAGE_SIZE, true /* wired */,
false /* !fixed */, true /* shared */, PROT_READ | PROT_WRITE,
PROT_READ | PROT_WRITE);
#endif
if (error) {
nvmm_vcpu_free(mach, vcpu);
nvmm_vcpu_put(vcpu);
@@ -395,12 +411,21 @@ nvmm_vcpu_create(struct nvmm_owner *owner, struct nvmm_ioc_vcpu_create *args)
memset(vcpu->comm, 0, NVMM_COMM_PAGE_SIZE);
#if defined(__HAIKU__)
os_vmmap_t *os_curproc_map = os_get_curproc_map();
#endif
/* Map the comm page on the user side, as pageable. */
error = os_vmobj_map(os_curproc_map, (vaddr_t *)&args->comm,
NVMM_COMM_PAGE_SIZE, mach->commvmobj,
args->cpuid * NVMM_COMM_PAGE_SIZE, false /* !wired */,
false /* !fixed */, true /* shared */, PROT_READ | PROT_WRITE,
PROT_READ | PROT_WRITE);
#if defined(__HAIKU__)
os_free_curproc_map(os_curproc_map);
#endif
if (error) {
nvmm_vcpu_free(mach, vcpu);
nvmm_vcpu_put(vcpu);
@@ -567,7 +592,7 @@ static int
nvmm_do_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu,
struct nvmm_vcpu_exit *exit)
{
struct vmspace *vm = mach->vm;
os_vmspace_t *vm = mach->vm;
int ret;
while (1) {
@@ -722,10 +747,17 @@ nvmm_hmapping_free(struct nvmm_machine *mach, uintptr_t hva, size_t size)
continue;
}
#if defined(__HAIKU__)
os_vmmap_t *os_curproc_map = os_get_curproc_map();
#endif
os_vmobj_unmap(os_curproc_map, hmapping->hva,
hmapping->hva + hmapping->size, false);
os_vmobj_rel(hmapping->vmobj);
#if defined(__HAIKU__)
os_free_curproc_map(os_curproc_map);
#endif
hmapping->vmobj = NULL;
hmapping->present = false;
@@ -762,11 +794,19 @@ nvmm_hva_map(struct nvmm_owner *owner, struct nvmm_ioc_hva_map *args)
hmapping->vmobj = os_vmobj_create(hmapping->size);
uva = hmapping->hva;
#if defined(__HAIKU__)
os_vmmap_t *os_curproc_map = os_get_curproc_map();
#endif
/* Map the vmobj into the user address space, as pageable. */
error = os_vmobj_map(os_curproc_map, &uva, hmapping->size,
hmapping->vmobj, 0, false /* !wired */, true /* fixed */,
true /* shared */, PROT_READ | PROT_WRITE, PROT_READ | PROT_WRITE);
#if defined(__HAIKU__)
os_free_curproc_map(os_curproc_map);
#endif
out:
nvmm_machine_put(mach);
return error;
@@ -845,10 +885,18 @@ nvmm_gpa_map(struct nvmm_owner *owner, struct nvmm_ioc_gpa_map *args)
goto out;
}
#if defined(__HAIKU__)
os_vmmap_t *vm_map = os_vmspace_get_vmmap(mach->vm);
/* Map the vmobj into the machine address space, as pageable. */
error = os_vmobj_map(vm_map, &gpa, args->size, vmobj, off,
false /* !wired */, true /* fixed */, false /* !shared */,
args->prot, PROT_READ | PROT_WRITE | PROT_EXEC);
#else
/* Map the vmobj into the machine address space, as pageable. */
error = os_vmobj_map(&mach->vm->vm_map, &gpa, args->size, vmobj, off,
false /* !wired */, true /* fixed */, false /* !shared */,
args->prot, PROT_READ | PROT_WRITE | PROT_EXEC);
#endif
out:
nvmm_machine_put(mach);
@@ -891,8 +939,14 @@ nvmm_gpa_unmap(struct nvmm_owner *owner, struct nvmm_ioc_gpa_unmap *args)
goto out;
}
#if defined(__HAIKU__)
os_vmmap_t *vm_map = os_vmspace_get_vmmap(mach->vm);
/* Unmap the memory from the machine. */
os_vmobj_unmap(vm_map, gpa, gpa + args->size, false);
#else
/* Unmap the memory from the machine. */
os_vmobj_unmap(&mach->vm->vm_map, gpa, gpa + args->size, false);
#endif
out:
nvmm_machine_put(mach);
src/add-ons/kernel/drivers/nvmm/nvmm.h
+12 -1
View File
@@ -36,6 +36,11 @@
#include <stdbool.h>
#endif
#if defined(__HAIKU__)
#include <stdint.h>
#include <stdbool.h>
#endif
typedef uint64_t gpaddr_t;
typedef uint64_t gvaddr_t;
@@ -45,13 +50,19 @@ typedef uint32_t nvmm_cpuid_t;
#undef CTASSERT
#define CTASSERT(x) NVMM_CTASSERT(x, __LINE__)
#define NVMM_CTASSERT(x, y) NVMM__CTASSERT(x, y)
#define NVMM__CTASSERT(x, y) typedef char __assert ## y[(x) ? 1 : -1] __unused
#if defined(__cplusplus)
#define NVMM__CTASSERT(x, y) static_assert(x, #x)
#else
#define NVMM__CTASSERT(x, y) _Static_assert(x, #x)
#endif
#if defined(__x86_64__)
#if defined(__NetBSD__)
#include <dev/nvmm/x86/nvmm_x86.h>
#elif defined(__DragonFly__)
#include <dev/virtual/nvmm/x86/nvmm_x86.h>
#elif defined(__HAIKU__)
#include "x86/nvmm_x86.h"
#endif
#endif /* __x86_64__ */
src/add-ons/kernel/drivers/nvmm/nvmm_bitops.h
+22
View File
@@ -0,0 +1,22 @@
/*
* Copyright 2024 Daniel Martin, [email protected]
* All rights reserved. Distributed under the terms of the MIT License.
*/
#ifndef NVMM_BITOPS
#define NVMM_BITOPS
// bitops macros taken from sys/cdefs.h (NetBSD)
#define NBBY 8 // (Number of Bits in a BYte)
#define __BIT(__n) \
(((uintmax_t)(__n) >= NBBY * sizeof(uintmax_t)) ? 0 : \
((uintmax_t)1 << (uintmax_t)((__n) & (NBBY * sizeof(uintmax_t) - 1))))
#define __BITS(__m, __n) \
((__BIT(max_c((__m), (__n)) + 1) - 1) ^ (__BIT(min_c((__m), (__n))) - 1))
#define __LOWEST_SET_BIT(__mask) ((((__mask) - 1) & (__mask)) ^ (__mask))
#define __SHIFTOUT(__x, __mask) (((__x) & (__mask)) / __LOWEST_SET_BIT(__mask))
#define __SHIFTIN(__x, __mask) ((__x) * __LOWEST_SET_BIT(__mask))
#endif /* NVMM_BITOPS */
src/add-ons/kernel/drivers/nvmm/nvmm_haiku.cpp
+696
View File
@@ -0,0 +1,696 @@
/*
* Copyright 2024 Daniel Martin, [email protected]
* Copyright 2026, Haiku, Inc. All rights reserved.
* Distributed under the terms of the MIT License.
*/
#include <Drivers.h>
#include <arch/x86/arch_cpu.h>
extern "C" {
#include "nvmm.h"
#include "nvmm_internal.h"
#include "nvmm_os.h"
#include "x86/nvmm_x86.h"
}
#include <drivers/KernelExport.h>
#include <OS.h>
#include <StackOrHeapArray.h>
#include <arch/x86/arch_system_info.h>
#include <kernel/heap.h>
#include <kernel/smp.h>
#include <kernel/thread.h>
#include <sys/ioccom.h>
#include <vm/VMCache.h>
#include <vm/VMAddressSpace.h>
#include <vm/vm_page.h>
#include "VMVirtualAddressSpace.h"
#include <paging/nested/X86VMTranslationMapEPT.h>
extern "C" void
x86_get_cpuid2(uint32_t eax, uint32_t ecx, cpuid_desc_t *descriptors)
{
cpuid_info info;
if (get_current_cpuid(&info, eax, ecx) != B_OK) {
*descriptors = {};
return;
}
descriptors->eax = info.regs.eax;
descriptors->ebx = info.regs.ebx;
descriptors->ecx = info.regs.ecx;
descriptors->edx = info.regs.edx;
}
extern "C" void
x86_get_cpuid(uint32_t eax, cpuid_desc_t *descriptors)
{
x86_get_cpuid2(eax, 0, descriptors);
}
extern "C" int
haiku_get_xsave_mask()
{
if (x86_check_feature(IA32_FEATURE_EXT_XSAVE, FEATURE_EXT))
return IA32_XCR0_X87 | IA32_XCR0_SSE;
return 0;
}
extern "C" int32
haiku_smp_get_current_cpu()
{
return smp_get_current_cpu();
}
extern "C" int32
haiku_smp_get_num_cpus()
{
return smp_get_num_cpus();
}
extern "C" os_cpu_t*
haiku_get_cpu_struct(uint32 cpu_number)
{
return &gCPU[cpu_number];
}
extern "C" int
os_cpu_number(os_cpu_t *cpu)
{
return cpu->cpu_num;
}
extern "C" thread_id
haiku_get_current_thread_id()
{
return thread_get_current_thread_id();
}
extern "C" void
os_ipi_unicast(os_cpu_t *cpu, void (*func)(void *, int), void *arg)
{
OS_ASSERT(os_cpu_number(cpu) >= 0 && os_cpu_number(cpu) < haiku_smp_get_num_cpus());
call_single_cpu_sync((uint32)os_cpu_number(cpu), func, arg);
}
extern "C" int
haiku_thread_bind()
{
thread_pin_to_current_cpu(thread_get_current_thread());
return 0;
}
extern "C" void
haiku_thread_unbind()
{
thread_unpin_from_current_cpu(thread_get_current_thread());
}
extern "C" status_t
os_mtx_lock(os_mtx_t *lock)
{
if (!are_interrupts_enabled())
while (mutex_trylock(lock) != B_OK);
else
return mutex_lock(lock);
return B_OK;
}
extern "C" void
x86_curthread_restore_dbregs(uint64_t *drs)
{
// TODO: If necessary, restore kernel debug registers from arch_team_debug_info
}
// this should be called with preemption disabled
// otherwise we might return a GDT not of the
// current CPU..
extern "C" void*
os_curcpu_gdt()
{
struct gdtr {
uint16 limit;
uint64 base;
} _PACKED;
struct gdtr gdtr;
__asm __volatile("sgdt %0" : "=m" (gdtr));
return (void*)gdtr.base;
}
extern "C" uint64
os_curcpu_idt()
{
struct idtr {
uint16 limit;
uint64 base;
} _PACKED;
struct idtr idtr;
__asm __volatile("sidt %0" : "=m" (idtr));
return idtr.base;
}
extern "C" void*
os_curcpu_tss()
{
return &gCPU[os_curcpu_number()].arch.tss;
}
extern "C" uint16
os_curcpu_tss_sel()
{
uint16 selector;
__asm __volatile("str %0" : "=m" (selector));
return selector;
}
// #pragma mark - os_* APIs
// aka os_vmobj_t
struct haiku_vmobj {
VMCache *cache;
int32 ref_count;
};
// aka os_vmspace_t
struct haiku_vmspace {
VMAddressSpace *address_space;
struct pmap pmap;
};
// aka os_cpuset_t
struct haiku_cpuset {
CPUSet set;
};
os_vmmap_t *os_kernel_map;
cpu_status *interrupt_status;
extern "C" void *
os_pagemem_zalloc(size_t size)
{
void *ptr;
size_t alloc_size = roundup(size, PAGE_SIZE);
area_id area = create_area("os_pagemem_zalloc_area", &ptr, B_ANY_KERNEL_ADDRESS,
alloc_size, B_FULL_LOCK, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
if (area < 0)
return NULL;
memset(ptr, 0, alloc_size);
return ptr;
}
extern "C" void
os_pagemem_free(void *ptr, size_t size __unused)
{
delete_area(area_for(ptr));
}
extern "C" int
os_contigpa_zalloc(paddr_t *pa, vaddr_t *va, size_t npages)
{
area_id area = create_area("os_contigpa_zalloc_area", (void **)va, B_ANY_KERNEL_ADDRESS,
npages * PAGE_SIZE, B_CONTIGUOUS, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
if (area < 0)
return area;
memset((void *)*va, 0, npages * PAGE_SIZE);
physical_entry entry;
status_t status = get_memory_map((void *)*va, 1, &entry, 1);
if (status < B_OK) {
delete_area(area);
return status;
}
*pa = entry.address;
return 0;
}
extern "C" void
os_contigpa_free(paddr_t pa __unused, vaddr_t va, size_t npages __unused)
{
delete_area(area_for((void *)va));
}
extern "C" os_vmspace_t *
os_vmspace_create(vaddr_t vmin, vaddr_t vmax)
{
if (vmax < vmin)
return NULL;
os_vmspace_t *ret = (os_vmspace_t *)os_mem_alloc(sizeof(os_vmspace_t));
if (ret == NULL)
return NULL;
X86VMTranslationMapEPT *translationMap = new(std::nothrow) X86VMTranslationMapEPT;
status_t status = translationMap->Init();
if (status != B_OK) {
delete translationMap;
os_mem_free(ret, sizeof(os_vmspace_t));
return NULL;
}
size_t size = vmax - vmin + 1;
ret->address_space = new(std::nothrow) VMVirtualAddressSpace(translationMap, vmin, size);
if (ret->address_space == NULL) {
delete translationMap;
os_mem_free(ret, sizeof(os_vmspace_t));
return NULL;
}
ret->pmap.pm_invgen = 0;
return ret;
}
extern "C" void
os_vmspace_destroy(os_vmspace_t *vm)
{
vm->address_space->Put();
os_mem_free(vm, sizeof(os_vmspace_t));
}
extern "C" int
os_vmspace_fault(os_vmspace_t *vm, vaddr_t va, vm_prot_t prot)
{
// if va isn't present on any area vm_soft_fault() will return
// error and print to syslog, which can heavily affect performance
vm->address_space->ReadLock();
if (!vm->address_space->LookupArea(va)) {
vm->address_space->ReadUnlock();
return 1;
}
vm->address_space->ReadUnlock();
status_t status = vm_soft_fault(vm->address_space, va,
prot & PROT_WRITE, prot & PROT_EXEC, true, NULL);
return status != B_OK;
}
extern "C" struct pmap*
os_vmspace_pmap(os_vmspace_t *vm)
{
return &vm->pmap;
}
extern "C" paddr_t
os_vmspace_pdirpa(os_vmspace_t *vm)
{
X86VMTranslationMap *map
= (X86VMTranslationMap*)vm->address_space->TranslationMap();
return map->PagingStructures()->pgdir_phys;
}
extern "C" os_vmmap_t *
os_vmspace_get_vmmap(os_vmspace_t *vm)
{
return vm;
}
extern "C" os_vmobj_t *
os_vmobj_create(voff_t size)
{
os_vmobj_t *ret = (os_vmobj_t *)os_mem_alloc(sizeof(os_vmobj_t));
if (ret == NULL)
return NULL;
int32 numPages = size / PAGE_SIZE;
if (size % PAGE_SIZE != 0)
numPages++;
// TODO: We can't enable swapping unless A/D bits are supported.
status_t status = VMCacheFactory::CreateAnonymousCache(ret->cache, false,
numPages, 0, false, 0);
if (status != B_OK) {
os_mem_free(ret, sizeof(os_vmobj_t));
return NULL;
}
ret->ref_count = 0;
ret->cache->temporary = 1;
ret->cache->virtual_base = 0;
ret->cache->virtual_end = size;
return ret;
}
extern "C" void
os_vmobj_ref(os_vmobj_t *vmobj)
{
atomic_add(&vmobj->ref_count, 1);
}
extern "C" void
os_vmobj_rel(os_vmobj_t *vmobj)
{
int32 previous = atomic_add(&vmobj->ref_count, -1);
if (previous != 0)
return;
while (true) {
vmobj->cache->Lock();
VMArea *area = vmobj->cache->areas.First();
if (area == NULL) {
vmobj->cache->Unlock();
break;
}
vaddr_t start = area->Base();
VMAddressSpace *address_space = area->address_space;
address_space->Get();
vmobj->cache->Unlock();
address_space->WriteLock();
if (address_space->LookupArea(start) != area
|| area->cache != vmobj->cache) {
// Something changed; restart.
address_space->WriteUnlock();
address_space->Put();
continue;
}
vm_unmap_address_range(address_space, start, area->Size(), true);
address_space->WriteUnlock();
address_space->Put();
}
vmobj->cache->ReleaseRef();
os_mem_free(vmobj, sizeof(os_vmobj_t));
}
//! shared indicates whether the mapping is inherit on fork calls or not
extern "C" int
os_vmobj_map(os_vmmap_t *map, vaddr_t *addr, vsize_t size, os_vmobj_t *vmobj,
voff_t offset, bool wired, bool fixed, bool shared __unused, int prot, int maxprot)
{
if (!vmobj->cache->Lock())
return B_ERROR;
status_t status = map->address_space->WriteLock();
if (status != B_OK)
return status;
uint32 wiring = wired ? B_FULL_LOCK : B_NO_LOCK;
uint32 flags = fixed ? CREATE_AREA_UNMAP_ADDRESS_RANGE : 0;
bool kernel = false;
if (map->address_space == VMAddressSpace::Kernel())
kernel = true;
virtual_address_restrictions addressRestrictions = {
.address = (void *)*addr,
.address_specification = fixed ? B_EXACT_ADDRESS : B_ANY_ADDRESS,
.alignment = B_PAGE_SIZE,
};
VMArea *area;
status = vm_map_cache(map->address_space, vmobj->cache,
vmobj->cache->virtual_base + offset, "nvmm_vmobj_area",
size, wiring, prot, maxprot, REGION_NO_PRIVATE_MAP, flags,
&addressRestrictions, kernel, &area, (void **)addr);
// RefCount() must always be number of areas + 1
if (status == B_OK)
vmobj->cache->AcquireRefLocked();
map->address_space->WriteUnlock();
vmobj->cache->Unlock();
return status;
}
//! the range [start, end-1] will be unmapped
extern "C" void
os_vmobj_unmap(os_vmmap_t *map, vaddr_t start, vaddr_t end,
bool wired __unused)
{
map->address_space->WriteLock();
vm_unmap_address_range(map->address_space, start, end - start - 1, true);
map->address_space->WriteUnlock();
}
extern "C" os_vmmap_t *
os_get_curproc_map()
{
os_vmmap_t *ret = (os_vmmap_t *)os_mem_alloc(sizeof(os_vmmap_t));
if (ret == NULL)
return NULL;
ret->address_space = VMAddressSpace::GetCurrent();
return ret;
}
extern "C" void
os_free_curproc_map(os_vmmap_t *map)
{
if (map->address_space != NULL)
map->address_space->Put();
os_mem_free(map, sizeof(os_vmmap_t));
}
extern "C" status_t
os_cpuset_init(os_cpuset_t **cpuset)
{
*cpuset = new os_cpuset_t;
if (*cpuset == NULL)
return B_NO_MEMORY;
return B_OK;
}
extern "C" void
os_cpuset_destroy(os_cpuset_t *cpuset)
{
free(cpuset);
}
extern "C" bool
os_cpuset_isset(os_cpuset_t *cpuset, int32 cpu)
{
return cpuset->set.GetBitAtomic(cpu);
}
extern "C" void
os_cpuset_clear(os_cpuset_t *cpuset, int32 cpu)
{
cpuset->set.ClearBitAtomic(cpu);
}
extern "C" void
os_cpuset_setrunning(os_cpuset_t *cpuset)
{
cpuset->set.SetAll();
}
// #pragma mark - driver API
int32 api_version = B_CUR_DRIVER_API_VERSION;
static const char *sDevices[] = { "misc/nvmm", NULL };
static status_t
nvmm_open_hook(const char *name, uint32 flags, void **cookie)
{
if (!(flags & O_CLOEXEC))
return B_BAD_VALUE;
struct nvmm_owner *owner;
if ((flags & O_ACCMODE) == O_WRONLY)
owner = &nvmm_root_owner;
else {
owner = (struct nvmm_owner *)os_mem_alloc(sizeof(*owner));
if (owner == NULL)
return B_NO_MEMORY;
owner->pid = getpid();
}
*cookie = owner;
return B_OK;
}
static status_t
nvmm_close_hook(void *cookie)
{
if (cookie == NULL)
return B_NO_INIT;
struct nvmm_owner *owner = (struct nvmm_owner *)cookie;
nvmm_kill_machines(owner);
return B_OK;
}
static status_t
nvmm_free_hook(void* cookie)
{
if (cookie == NULL)
return B_NO_INIT;
if (cookie != &nvmm_root_owner)
os_mem_free(cookie, sizeof(struct nvmm_owner));
return B_OK;
}
static status_t
nvmm_control_hook(void *cookie, uint32 op, void *data, size_t len)
{
len = IOCPARM_LEN(op);
BStackOrHeapArray<char, 128> kernel_data(len);
status_t status = user_memcpy(kernel_data, data, len);
if (status < 0)
return status;
status_t ioctl_status = nvmm_ioctl((struct nvmm_owner *)cookie, op, kernel_data);
status = user_memcpy(data, kernel_data, len);
if (status < 0)
return status;
return ioctl_status;
}
static device_hooks sHooks = {
.open = nvmm_open_hook,
.close = nvmm_close_hook,
.free = nvmm_free_hook,
.control = nvmm_control_hook,
};
status_t
init_hardware(void)
{
if (nvmm_ident() == NULL) {
TRACE_ALWAYS("nvmm: cpu not supported\n");
return B_ERROR;
}
return B_OK;
}
const char**
publish_devices(void)
{
TRACE_ALWAYS("nvmm: publish_devices\n");
return sDevices;
}
device_hooks*
find_device(const char* name)
{
TRACE_ALWAYS("nvmm: find_device\n");
return &sHooks;
}
status_t
init_driver(void)
{
status_t status;
int32 n_cpus;
if (nvmm_init())
return B_ERROR;
os_kernel_map = (os_vmmap_t *)malloc(sizeof(os_vmmap_t));
if (os_kernel_map == NULL) {
status = B_NO_MEMORY;
goto err1;
}
os_kernel_map->address_space = VMAddressSpace::Kernel();
n_cpus = smp_get_num_cpus();
if (n_cpus <= 0) {
status = B_BAD_VALUE;
goto err2;
}
interrupt_status = (cpu_status *)malloc(n_cpus * sizeof(cpu_status));
if (interrupt_status == NULL) {
status = B_NO_MEMORY;
goto err2;
}
TRACE_ALWAYS("nvmm: init_driver OK\n");
return B_OK;
err2:
free(os_kernel_map);
err1:
nvmm_fini();
return status;
}
void
uninit_driver(void)
{
TRACE_ALWAYS("nvmm: uninit_driver\n");
nvmm_fini();
free(os_kernel_map);
free(interrupt_status);
}
src/add-ons/kernel/drivers/nvmm/nvmm_internal.h
+3 -1
View File
@@ -43,6 +43,8 @@
#define NVMM_MAX_RAM (128ULL * (1 << 30))
#elif defined(__DragonFly__)
#define NVMM_MAX_RAM (127ULL * 1024ULL * (1 << 30))
#elif defined(__HAIKU__)
#define NVMM_MAX_RAM (128ULL * (1 << 30))
#else
#error "OS dependency for NVMM_MAX_RAM required"
#endif
@@ -88,7 +90,7 @@ struct nvmm_machine {
os_vmobj_t *commvmobj;
/* Kernel */
struct vmspace *vm;
os_vmspace_t *vm;
gpaddr_t gpa_begin;
gpaddr_t gpa_end;
src/add-ons/kernel/drivers/nvmm/nvmm_ioctl.h
+2
View File
@@ -35,6 +35,8 @@
#include <dev/nvmm/nvmm.h>
#elif defined(__DragonFly__)
#include <dev/virtual/nvmm/nvmm.h>
#elif defined(__HAIKU__)
#include "nvmm.h"
#else
#error "Unsupported OS."
#endif
src/add-ons/kernel/drivers/nvmm/nvmm_os.h
+169 -3
View File
@@ -56,15 +56,26 @@
#include <vm/pmap.h> /* pmap_ept_transform, pmap_npt_transform */
#include <machine/cpu.h> /* hvm_break_wanted */
#include <machine/cpufunc.h> /* ffsl, ffs, etc. */
#elif defined(__HAIKU__)
#include <sys/cdefs.h>
#include <Drivers.h>
#include <interrupts.h>
#include <machine/specialreg.h>
#include <kernel/lock.h>
#include <SupportDefs.h>
#include <stdlib.h>
#include <string.h>
#endif
/* Types. */
#if defined(__NetBSD__)
typedef struct vm_map os_vmmap_t;
typedef struct vmspace os_vmspace_t;
typedef struct uvm_object os_vmobj_t;
typedef krwlock_t os_rwl_t;
typedef kmutex_t os_mtx_t;
#elif defined(__DragonFly__)
typedef struct vm_map os_vmmap_t;
typedef struct vmspace os_vmspace_t;
typedef struct vm_object os_vmobj_t;
typedef struct lock os_rwl_t;
@@ -74,18 +85,39 @@ typedef vm_offset_t vaddr_t;
typedef vm_offset_t voff_t;
typedef vm_size_t vsize_t;
typedef vm_paddr_t paddr_t;
#elif defined(__HAIKU__)
typedef struct haiku_vmspace os_vmspace_t;
typedef struct haiku_vmspace os_vmmap_t;
typedef struct haiku_vmobj os_vmobj_t;
typedef phys_addr_t paddr_t;
typedef addr_t vaddr_t;
typedef off_t voff_t;
typedef size_t vsize_t;
typedef uint32 vm_prot_t;
typedef rw_lock os_rwl_t;
typedef mutex os_mtx_t;
#endif
/* Attributes. */
#if defined(__DragonFly__)
#define __cacheline_aligned __cachealign
#define __diagused __debugvar
#elif defined(__HAIKU__)
#define __cacheline_aligned
#define __read_mostly
#endif
/* Macros. */
#if defined(__DragonFly__)
#define __arraycount(__x) (sizeof(__x) / sizeof(__x[0]))
#define __insn_barrier() __asm __volatile("":::"memory")
#elif defined (__HAIKU__)
#define __arraycount B_COUNT_OF
// roundup() taken from headers/private/firewire/fwglue.h
#define roundup(x, y) ((((x)+((y)-1))/(y))*(y)) /* to any y */
#define __diagused
#define __insn_barrier() __asm __volatile("":::"memory")
#define PAGE_SIZE B_PAGE_SIZE
#endif
/* Bitops. */
@@ -99,12 +131,18 @@ typedef vm_paddr_t paddr_t;
#undef __BITS
#define __BITS(__m, __n) __BITS64(__m, __n)
#endif /* __x86_64__ */
#elif defined(__HAIKU__)
#include "nvmm_bitops.h"
#endif
/* Maps. */
#if defined(__NetBSD__) || defined(__DragonFly__)
#define os_kernel_map kernel_map
#define os_curproc_map &curproc->p_vmspace->vm_map
#elif defined(__HAIKU__)
extern os_vmmap_t *os_kernel_map;
os_vmmap_t* os_get_curproc_map();
void os_free_curproc_map(os_vmmap_t *map);
#endif
/* R/W locks. */
@@ -122,6 +160,14 @@ typedef vm_paddr_t paddr_t;
#define os_rwl_wlock(lock) lockmgr(lock, LK_EXCLUSIVE);
#define os_rwl_unlock(lock) lockmgr(lock, LK_RELEASE)
#define os_rwl_wheld(lock) (lockstatus(lock, curthread) == LK_EXCLUSIVE)
#elif defined(__HAIKU__)
#define os_rwl_init(lock) rw_lock_init(lock, NULL)
#define os_rwl_destroy(lock) rw_lock_destroy(lock)
#define os_rwl_rlock(lock) rw_lock_read_lock(lock)
#define os_rwl_wlock(lock) rw_lock_write_lock(lock)
#define os_rwl_wheld(lock) ((lock)->holder == haiku_get_current_thread_id())
#define os_rwl_unlock(lock) (os_rwl_wheld(lock) \
? rw_lock_write_unlock(lock) : rw_lock_read_unlock(lock))
#endif
/* Mutexes. */
@@ -137,6 +183,11 @@ typedef vm_paddr_t paddr_t;
#define os_mtx_lock(lock) lockmgr(lock, LK_EXCLUSIVE)
#define os_mtx_unlock(lock) lockmgr(lock, LK_RELEASE)
#define os_mtx_owned(lock) (lockstatus(lock, curthread) == LK_EXCLUSIVE)
#elif defined(__HAIKU__)
#define os_mtx_init(lock) mutex_init(lock, NULL)
#define os_mtx_destroy(lock) mutex_destroy(lock)
status_t os_mtx_lock(os_mtx_t *lock);
#define os_mtx_unlock(lock) mutex_unlock(lock)
#endif
/* Malloc. */
@@ -151,6 +202,10 @@ MALLOC_DECLARE(M_NVMM);
#define os_mem_alloc(size) kmalloc(size, M_NVMM, M_WAITOK)
#define os_mem_zalloc(size) kmalloc(size, M_NVMM, M_WAITOK | M_ZERO)
#define os_mem_free(ptr, size) kfree(ptr, M_NVMM)
#elif defined(__HAIKU__)
#define os_mem_alloc(size) malloc(size)
#define os_mem_zalloc(size) calloc(1, size)
#define os_mem_free(ptr, size) free(ptr)
#endif
/* Printf. */
@@ -158,6 +213,9 @@ MALLOC_DECLARE(M_NVMM);
#define os_printf printf
#elif defined(__DragonFly__)
#define os_printf kprintf
#elif defined(__HAIKU__)
#define TRACE_ALWAYS(a...) dprintf(a)
#define os_printf TRACE_ALWAYS
#endif
/* Atomics. */
@@ -173,6 +231,11 @@ MALLOC_DECLARE(M_NVMM);
#define os_atomic_dec_uint(x) atomic_subtract_int(x, 1)
#define os_atomic_load_uint(x) atomic_load_acq_int(x)
#define os_atomic_inc_64(x) atomic_add_64(x, 1)
#elif defined(__HAIKU__)
#define os_atomic_inc_uint(x) atomic_add((int32*)x, 1)
#define os_atomic_dec_uint(x) atomic_add((int32*)x, -1)
#define os_atomic_load_uint(x) atomic_get((int32*)x)
#define os_atomic_inc_64(x) atomic_add64((int64*)x, 1)
#endif
/* Pmap. */
@@ -184,6 +247,16 @@ extern bool pmap_ept_has_ad;
#elif defined(__DragonFly__)
#define os_vmspace_pmap(vm) vmspace_pmap(vm)
#define os_vmspace_pdirpa(vm) (vtophys(vmspace_pmap(vm)->pm_pml4))
#elif defined(__HAIKU__)
// contains some fields
// present at struct pmap from DragonFlyBSD
// see sys/platform/pc64/include/pmap.h
struct pmap {
uint64_t pm_invgen;
};
struct pmap *os_vmspace_pmap(os_vmspace_t *vm);
paddr_t os_vmspace_pdirpa(os_vmspace_t *vm);
os_vmmap_t *os_vmspace_get_vmmap(os_vmspace_t *vm);
#endif
/* CPU. */
@@ -213,6 +286,30 @@ typedef struct globaldata os_cpu_t;
#define os_curcpu_tss() &mycpu->gd_prvspace->common_tss
#define os_curcpu_gdt() mdcpu->gd_gdt
#define os_curcpu_idt() r_idt_arr[mycpuid].rd_base
#elif defined(__HAIKU__)
/*
* On NetBSD/DragonFlyBSD each CPU has a unique structure. NVMM
* uses the unique pointers to those structures as a way to check
* whether two CPUs are the same or not. In our case we'll use
* the cpu_ent structure that is declared on cpu.h which can't be
* included from C. We declare it here as an incomplete type.
*/
struct cpu_ent;
typedef struct cpu_ent os_cpu_t;
os_cpu_t* haiku_get_cpu_struct(uint32 cpu_number);
#define OS_CPU_FOREACH(cpu) \
uint32 curcpu = 0; \
uint32 _ncpus = haiku_smp_get_num_cpus(); \
for (cpu = haiku_get_cpu_struct(0); curcpu < _ncpus; \
cpu = haiku_get_cpu_struct(++curcpu))
int os_cpu_number(os_cpu_t *cpu);
#define os_curcpu() haiku_get_cpu_struct(haiku_smp_get_current_cpu())
#define os_curcpu_number() haiku_smp_get_current_cpu()
uint16 os_curcpu_tss_sel();
void *os_curcpu_tss();
void *os_curcpu_gdt();
uint64 os_curcpu_idt();
#endif
/* Cpusets. */
@@ -234,6 +331,13 @@ typedef cpumask_t os_cpuset_t;
#define os_cpuset_isset(s, c) CPUMASK_TESTBIT(*(s), c)
#define os_cpuset_clear(s, c) ATOMIC_CPUMASK_NANDBIT(*(s), c)
#define os_cpuset_setrunning(s) ATOMIC_CPUMASK_ORMASK(*(s), smp_active_mask)
#elif defined(__HAIKU__)
typedef struct haiku_cpuset os_cpuset_t;
status_t os_cpuset_init(os_cpuset_t **cpuset);
void os_cpuset_destroy(os_cpuset_t *cpuset);
bool os_cpuset_isset(os_cpuset_t *cpuset, int32 cpu);
void os_cpuset_clear(os_cpuset_t *cpuset, int32 cpu);
void os_cpuset_setrunning(os_cpuset_t *cpuset);
#endif
/* Preemption. */
@@ -256,6 +360,11 @@ typedef cpumask_t os_cpuset_t;
#define os_preempt_disable() crit_enter()
#define os_preempt_enable() crit_exit()
#define os_preempt_disabled() (curthread->td_critcount != 0)
#elif defined(__HAIKU__)
extern cpu_status *interrupt_status;
#define os_preempt_disable() interrupt_status[haiku_smp_get_current_cpu()] = disable_interrupts()
#define os_preempt_enable() restore_interrupts(interrupt_status[haiku_smp_get_current_cpu()])
#define os_preempt_disabled() !interrupts_enabled()
#endif
/* Asserts. */
@@ -263,13 +372,24 @@ typedef cpumask_t os_cpuset_t;
#define OS_ASSERT KASSERT
#elif defined(__DragonFly__)
#define OS_ASSERT KKASSERT
#elif defined(__HAIKU__)
#define OS_ASSERT ASSERT
#endif
/* Misc. */
#if defined(__DragonFly__)
#if defined(__DragonFly__) || defined(__HAIKU__)
#define uimin(a, b) ((u_int)a < (u_int)b ? (u_int)a : (u_int)b)
#endif
/* Memory protection */
#if defined(__HAIKU__)
#define PROT_READ B_READ_AREA
#define PROT_WRITE B_WRITE_AREA
#define PROT_EXEC B_EXECUTE_AREA
#define copyin(from, to, size) user_memcpy(to, from, size)
#define copyout(from, to, size) user_memcpy(to, from, size)
#endif
/* -------------------------------------------------------------------------- */
os_vmspace_t * os_vmspace_create(vaddr_t, vaddr_t);
@@ -280,15 +400,17 @@ os_vmobj_t * os_vmobj_create(voff_t);
void os_vmobj_ref(os_vmobj_t *);
void os_vmobj_rel(os_vmobj_t *);
int os_vmobj_map(struct vm_map *, vaddr_t *, vsize_t, os_vmobj_t *,
int os_vmobj_map(os_vmmap_t *, vaddr_t *, vsize_t, os_vmobj_t *,
voff_t, bool, bool, bool, int, int);
void os_vmobj_unmap(struct vm_map *map, vaddr_t, vaddr_t, bool);
void os_vmobj_unmap(os_vmmap_t *map, vaddr_t, vaddr_t, bool);
void * os_pagemem_zalloc(size_t);
void os_pagemem_free(void *, size_t);
#if defined(__DragonFly__) || defined(__NetBSD__)
paddr_t os_pa_zalloc(void);
void os_pa_free(paddr_t);
#endif
int os_contigpa_zalloc(paddr_t *, vaddr_t *, size_t);
void os_contigpa_free(paddr_t, vaddr_t, size_t);
@@ -312,9 +434,34 @@ os_return_needed(void)
return true;
}
return false;
#elif defined(__HAIKU__)
// FIXME.
return false;
#endif
}
// Haiku auxiliary functions
#if defined(__HAIKU__)
int haiku_get_xsave_mask();
int32 haiku_smp_get_current_cpu();
int32 haiku_smp_get_num_cpus();
thread_id haiku_get_current_thread_id();
static __inline int
fls(int mask)
{
return (mask == 0 ? 0 :
8 * sizeof(mask) - __builtin_clz((u_int)mask));
}
int flsll(long long mask);
// based on ilog2() from DragonFlyBSD
// available at /sys/dev/drm/include/linux/log2.h
#define ilog2(n) (sizeof(n) <= 4) ? \
fls((uint32)(n)) - 1 : flsll((uint64)(n)) - 1
#endif
/* -------------------------------------------------------------------------- */
/* IPIs. */
@@ -380,6 +527,25 @@ os_ipi_broadcast(void (*func)(void *), void *arg)
#define curlwp_bind() ((int)0)
#define curlwp_bindx(bound) /* nothing */
#elif defined(__HAIKU__)
#include <drivers/KernelExport.h>
#define OS_IPI_FUNC(func) void func(void *arg, int unused)
void os_ipi_unicast(os_cpu_t *cpu, void (*func)(void *, int), void *arg);
static inline void
os_ipi_broadcast(void (*func)(void *, int), void *arg)
{
call_all_cpus_sync(func, arg);
}
int haiku_thread_bind();
void haiku_thread_unbind();
#define curlwp_bind() haiku_thread_bind()
#define curlwp_bindx(bound) haiku_thread_unbind()
#endif /* __NetBSD__ */
#endif /* _NVMM_OS_H_ */
src/add-ons/kernel/drivers/nvmm/x86/nvmm_x86.h
+60 -2
View File
@@ -37,6 +37,22 @@
/* -------------------------------------------------------------------------- */
#if defined(__HAIKU__) && defined(_KERNEL)
#define _ASSEMBLER
#include <arch/x86/descriptors.h>
#undef _ASSEMBLER
#define GSEL(s,r) (((s) << 3) | r)
#define GCODE_SEL KERNEL_CODE_SEGMENT
#define GDATA_SEL KERNEL_DATA_SEGMENT
#define GUDATA_SEL USER_DATA_SEGMENT
#define SEL_KPL DPL_KERNEL
#define SEL_UPL DPL_USER
#define PSL_I 0x00000200 /* interrupt enable bit */
#define PSL_RF 0x00010000 /* resume flag bit */
#endif
#ifndef ASM_NVMM
struct nvmm_x86_exit_memory {
@@ -221,6 +237,7 @@ struct nvmm_cap_md {
#ifndef ASM_NVMM
#if defined(__NetBSD__) || defined(__DragonFly__)
#include <sys/types.h>
#include <sys/bitops.h>
#if defined(__DragonFly__)
@@ -229,8 +246,9 @@ struct nvmm_cap_md {
#define __BIT(__n) __BIT64(__n)
#undef __BITS
#define __BITS(__m, __n) __BITS64(__m, __n)
#endif /* __DragonFly__ */
#endif /* __x86_64__ */
#endif
#endif /* defined(__NetBSD__) || defined(__DragonFly__) */
/* Segment state. */
struct nvmm_x64_state_seg {
@@ -657,7 +675,7 @@ struct nvmm_vcpu_conf_tpr {
* Register defines. We mainly rely on the already-existing OS definitions.
*/
#if defined(__DragonFly__)
#if defined(__DragonFly__) || defined(__HAIKU__)
#define XCR0_X87 CPU_XFEATURE_X87 /* 0x00000001 */
#define XCR0_SSE CPU_XFEATURE_SSE /* 0x00000002 */
@@ -732,6 +750,8 @@ uint32_t nvmm_x86_xsave_size(uint64_t);
#elif defined(__DragonFly__)
#include <machine/cpufunc.h>
#include <machine/npx.h>
#elif defined(__HAIKU__)
#include <machine/cpufunc.h>
#endif
/* CPUID. */
@@ -745,6 +765,9 @@ typedef struct {
#elif defined(__DragonFly__)
#define x86_get_cpuid(l, d) do_cpuid(l, (uint32_t *)d)
#define x86_get_cpuid2(l, c, d) cpuid_count(l, c, (uint32_t *)d)
#elif defined(__HAIKU__)
void x86_get_cpuid(uint32_t eax, cpuid_desc_t *descriptors);
void x86_get_cpuid2(uint32_t eax, uint32_t ecx, cpuid_desc_t *descriptors);
#endif
/* Control registers. */
@@ -764,6 +787,14 @@ typedef struct {
#define x86_set_cr0(v) load_cr0(v)
#define x86_set_cr2(v) load_cr2(v)
#define x86_set_cr4(v) load_cr4(v)
#elif defined(__HAIKU__)
#define x86_get_cr0() rcr0()
#define x86_get_cr2() rcr2()
#define x86_get_cr3() rcr3()
#define x86_get_cr4() rcr4()
#define x86_set_cr0(v) load_cr0(v)
#define x86_set_cr2(v) load_cr2(v)
#define x86_set_cr4(v) load_cr4(v)
#endif
/* Debug registers. */
@@ -835,6 +866,26 @@ x86_curthread_restore_dbregs(uint64_t *drs)
#define x86_set_dr3(v) load_dr3(v)
#define x86_set_dr6(v) load_dr6(v)
#define x86_set_dr7(v) load_dr7(v)
#elif defined(__HAIKU__)
#define x86_get_dr0() rdr0()
#define x86_get_dr1() rdr1()
#define x86_get_dr2() rdr2()
#define x86_get_dr3() rdr3()
#define x86_get_dr6() rdr6()
#define x86_get_dr7() rdr7()
#define x86_set_dr0(v) load_dr0(v)
#define x86_set_dr1(v) load_dr1(v)
#define x86_set_dr2(v) load_dr2(v)
#define x86_set_dr3(v) load_dr3(v)
#define x86_set_dr6(v) load_dr6(v)
#define x86_set_dr7(v) load_dr7(v)
static inline void
x86_curthread_save_dbregs(uint64_t *drs)
{
/* not needed */
}
void x86_curthread_restore_dbregs(uint64_t *drs);
#endif
/* FPU. */
@@ -856,6 +907,10 @@ x86_curthread_restore_dbregs(uint64_t *drs)
__asm volatile("clts" ::: "memory"); \
fpurstor((union savefpu *)(a), m); \
})
#elif defined(__HAIKU__)
// Haiku allows floating point in kernel, we don't need to save and restore FPU
#define x86_save_fpu(a, m)
#define x86_restore_fpu(a, m)
#endif
/* XCRs. */
@@ -891,6 +946,9 @@ x86_set_xcr(uint32_t xcr, uint64_t val)
#if defined(__DragonFly__)
#define x86_xsave_features npx_xcr0_mask
#define x86_fpu_mxcsr_mask npx_mxcsr_mask
#elif defined(__HAIKU__)
#define x86_xsave_features haiku_get_xsave_mask()
#define x86_fpu_mxcsr_mask 0xFFBF /* default */
#endif
#endif /* _KERNEL */
src/add-ons/kernel/drivers/nvmm/x86/nvmm_x86_svm.c
+58 -2
View File
@@ -26,15 +26,20 @@
* SUCH DAMAGE.
*/
#if defined(__HAIKU__)
#include "../nvmm_os.h"
#else
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/mman.h>
#endif
#include "../nvmm.h"
#include "../nvmm_internal.h"
#include "nvmm_x86.h"
#if 0
void svm_vmrun(paddr_t, uint64_t *);
static inline void
@@ -48,6 +53,7 @@ svm_stgi(void)
{
__asm volatile ("stgi" ::: "memory");
}
#endif
#define MSR_VM_HSAVE_PA 0xC0010117
@@ -492,9 +498,16 @@ static struct {
struct svm_hsave {
paddr_t pa;
#if defined(__HAIKU__)
vaddr_t va;
#endif
};
#if defined(__HAIKU__)
static struct svm_hsave *hsave;
#else
static struct svm_hsave hsave[OS_MAXCPUS];
#endif
static uint8_t *svm_asidmap __read_mostly;
static uint32_t svm_maxasid __read_mostly;
@@ -529,6 +542,7 @@ static uint64_t svm_xcr0_mask __read_mostly;
/* -------------------------------------------------------------------------- */
#if 0
struct svm_machdata {
volatile uint64_t mach_htlb_gen;
};
@@ -819,11 +833,13 @@ svm_inkernel_advance(struct vmcb *vmcb)
vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW;
}
#endif
#define SVM_CPUID_MAX_BASIC 0xD
#define SVM_CPUID_MAX_HYPERVISOR 0x40000000
#define SVM_CPUID_MAX_EXTENDED 0x8000001F
static uint32_t svm_cpuid_max_basic __read_mostly;
static uint32_t svm_cpuid_max_extended __read_mostly;
#if 0
static void
svm_inkernel_exec_cpuid(struct svm_cpudata *cpudata, uint32_t eax, uint32_t ecx)
@@ -1361,8 +1377,10 @@ svm_exit_invalid(struct nvmm_vcpu_exit *exit, uint64_t code)
exit->reason = NVMM_VCPU_EXIT_INVALID;
}
#endif // 0
/* -------------------------------------------------------------------------- */
#if 0
static void
svm_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu)
{
@@ -1454,9 +1472,11 @@ svm_vcpu_guest_misc_leave(struct nvmm_cpu *vcpu)
wrmsr(MSR_FSBASE, cpudata->hstate.fsbase);
wrmsr(MSR_KERNELGSBASE, cpudata->hstate.kernelgsbase);
}
#endif
/* -------------------------------------------------------------------------- */
#if 0
static inline void
svm_gtlb_catchup(struct nvmm_cpu *vcpu, int hcpu)
{
@@ -1721,9 +1741,11 @@ svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu,
return error;
}
#endif
/* -------------------------------------------------------------------------- */
#if 0
#define SVM_MSRBM_READ __BIT(0)
#define SVM_MSRBM_WRITE __BIT(1)
@@ -2099,9 +2121,11 @@ svm_vcpu_state_commit(struct nvmm_cpu *vcpu)
vcpu->comm->state_commit = 0;
svm_vcpu_setstate(vcpu);
}
#endif // 0
/* -------------------------------------------------------------------------- */
#if 0
static void
svm_asid_alloc(struct nvmm_cpu *vcpu)
{
@@ -2349,9 +2373,11 @@ svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
os_pagemem_free(cpudata, sizeof(*cpudata));
}
#endif
/* -------------------------------------------------------------------------- */
#if 0
static int
svm_vcpu_configure_cpuid(struct svm_cpudata *cpudata, void *data)
{
@@ -2419,9 +2445,11 @@ svm_vcpu_configure(struct nvmm_cpu *vcpu, uint64_t op, void *data)
return EINVAL;
}
}
#endif
/* -------------------------------------------------------------------------- */
#if 0
#ifdef __NetBSD__
static void
svm_tlb_flush(struct pmap *pm)
@@ -2472,6 +2500,7 @@ svm_machine_configure(struct nvmm_machine *mach, uint64_t op, void *data)
{
panic("%s: impossible", __func__);
}
#endif // 0
/* -------------------------------------------------------------------------- */
@@ -2622,9 +2651,23 @@ svm_init(void)
svm_global_hstate.cstar = rdmsr(MSR_CSTAR);
svm_global_hstate.sfmask = rdmsr(MSR_SFMASK);
#if defined(__HAIKU__)
hsave = os_mem_zalloc(sizeof(struct svm_hsave) * haiku_smp_get_num_cpus());
#else
memset(hsave, 0, sizeof(hsave));
#endif
OS_CPU_FOREACH(cpu) {
#if defined(__HAIKU__)
int32 cpu_index = os_cpu_number(cpu);
paddr_t *pa = &hsave[cpu_index].pa;
vaddr_t *va = &hsave[cpu_index].va;
int error = os_contigpa_zalloc(pa, va, 1);
if (error) {
panic("%s: out of memory", __func__);
}
#else
hsave[os_cpu_number(cpu)].pa = os_pa_zalloc();
#endif
}
os_ipi_broadcast(svm_change_cpu, (void *)true);
@@ -2648,12 +2691,25 @@ svm_fini(void)
os_ipi_broadcast(svm_change_cpu, (void *)false);
#if defined(__HAIKU__)
size_t n_cpus = haiku_smp_get_num_cpus();
for (i = 0; i < n_cpus; i++) {
#else
for (i = 0; i < OS_MAXCPUS; i++) {
#endif
if (hsave[i].pa != 0)
#if defined(__HAIKU__)
os_contigpa_free(hsave[i].pa, hsave[i].va, 1);
#else
os_pa_free(hsave[i].pa);
#endif
}
svm_fini_asid();
#if defined(__HAIKU__)
// second argument is ignored
os_mem_free(hsave, 0);
#endif
}
static void
@@ -2672,7 +2728,7 @@ const struct nvmm_impl nvmm_x86_svm = {
.ident = svm_ident,
.init = svm_init,
.fini = svm_fini,
.capability = svm_capability,
.capability = svm_capability/*,
.mach_conf_max = NVMM_X86_MACH_NCONF,
.mach_conf_sizes = NULL,
.vcpu_conf_max = NVMM_X86_VCPU_NCONF,
@@ -2687,5 +2743,5 @@ const struct nvmm_impl nvmm_x86_svm = {
.vcpu_setstate = svm_vcpu_setstate,
.vcpu_getstate = svm_vcpu_getstate,
.vcpu_inject = svm_vcpu_inject,
.vcpu_run = svm_vcpu_run
.vcpu_run = svm_vcpu_run*/
};
src/add-ons/kernel/drivers/nvmm/x86/nvmm_x86_vmx.c
+51 -1
View File
@@ -26,15 +26,20 @@
* SUCH DAMAGE.
*/
#if defined(__HAIKU__)
#include "../nvmm_os.h"
#else
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/mman.h>
#endif
#include "../nvmm.h"
#include "../nvmm_internal.h"
#include "nvmm_x86.h"
int vmx_vmlaunch(uint64_t *gprs);
int vmx_vmresume(uint64_t *gprs);
void vmx_resume_rip(void);
@@ -175,13 +180,30 @@ vmx_vmclear(paddr_t *pa)
static inline void
vmx_cli(void)
{
#if defined(__HAIKU__)
/*
* vmx_cli() and vmx_sti() are only called from vmx_vcpu_run(). They
* are called after vmx_vmcs_enter() (which disables preemption) and
* before vmx_vmcs_leave() (which enables preemption). In Haiku we
* need to disable interrupts to disable preemption, so if we leave
* this code unmodified the call to vmx_sti() would enable preemption
* before vmx_vmcs_leave() is called, when it should be still disabled.
*/
OS_ASSERT(!interrupts_enabled());
__asm volatile("" ::: "memory");
#else
__asm volatile ("cli" ::: "memory");
#endif
}
static inline void
vmx_sti(void)
{
#if defined(__HAIKU__)
__asm volatile("" ::: "memory");
#else
__asm volatile ("sti" ::: "memory");
#endif
}
#define MSR_IA32_FEATURE_CONTROL 0x003A
@@ -630,7 +652,11 @@ struct vmxoncpu {
paddr_t pa;
};
#if defined(__HAIKU__)
static struct vmxoncpu *vmxoncpu;
#else
static struct vmxoncpu vmxoncpu[OS_MAXCPUS];
#endif
struct vmcs {
uint32_t ident;
@@ -2056,6 +2082,12 @@ vmx_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu)
* FPU or not. Need to use npxpush()/npxpop() to handle this.
*/
npxpush(&cpudata->hstate.hmctx);
#elif defined(__HAIKU__)
/*
* Haiku allows floating point on kernel and it handles save and restore
* FPU state on context switches (see commit 396b742). The only thing
* we need to save and restore manually is the XCR0 register.
*/
#endif
x86_restore_fpu(&cpudata->gxsave, vmx_xcr0_mask);
@@ -2183,7 +2215,7 @@ vmx_htlb_flush(struct nvmm_machine *mach, struct vmx_cpudata *cpudata)
struct ept_desc ept_desc;
uint64_t machgen;
#if defined(__NetBSD__)
#if defined(__NetBSD__) || defined(__HAIKU__)
machgen = ((struct vmx_machdata *)mach->machdata)->mach_htlb_gen;
#elif defined(__DragonFly__)
clear_xinvltlb();
@@ -2441,6 +2473,11 @@ vmx_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu,
if (exit->reason != NVMM_VCPU_EXIT_NONE) {
break;
}
#if defined(__HAIKU__)
// FIXME: ugly hack
break;
#endif
}
cpudata->vmcs_launched = launched;
@@ -3586,7 +3623,11 @@ vmx_init(void)
vmx_global_hstate.cstar = rdmsr(MSR_CSTAR);
vmx_global_hstate.sfmask = rdmsr(MSR_SFMASK);
#if defined(__HAIKU__)
vmxoncpu = os_mem_zalloc(sizeof(struct vmxoncpu) * haiku_smp_get_num_cpus());
#else
memset(vmxoncpu, 0, sizeof(vmxoncpu));
#endif
revision = vmx_get_revision();
OS_CPU_FOREACH(cpu) {
@@ -3622,12 +3663,21 @@ vmx_fini(void)
os_ipi_broadcast(vmx_change_cpu, (void *)false);
#if defined(__HAIKU__)
size_t n_cpus = haiku_smp_get_num_cpus();
for (i = 0; i < n_cpus; i++) {
#else
for (i = 0; i < OS_MAXCPUS; i++) {
#endif
if (vmxoncpu[i].pa != 0)
os_contigpa_free(vmxoncpu[i].pa, vmxoncpu[i].va, 1);
}
vmx_fini_asid();
#if defined(__HAIKU__)
// second argument is ignored
os_mem_free(vmxoncpu, 0);
#endif
}
static void
src/add-ons/kernel/drivers/nvmm/x86/nvmm_x86_vmxfunc.S
+13
View File
@@ -38,6 +38,9 @@
#define _C_LABEL(x) x
#include <machine/asmacros.h>
#include "assym.s"
#elif defined(__HAIKU__)
#define _C_LABEL(x) x
#include <machine/asmacros.h>
#endif
#define ASM_NVMM
@@ -204,12 +207,22 @@ ENTRY(vmx_resume_rip)
END(vmx_resume_rip)
ENTRY(vmx_insn_failvalid)
// this is a (hopefully) temporary workaround
#ifdef __HAIKU__
movq 0,%rdi
#else
movq $.Lvmx_validstr,%rdi
#endif
call _C_LABEL(panic)
END(vmx_insn_failvalid)
ENTRY(vmx_insn_failinvalid)
// this is a (hopefully) temporary workaround
#ifdef __HAIKU__
movq 0,%rdi
#else
movq $.Lvmx_invalidstr,%rdi
#endif
call _C_LABEL(panic)
END(vmx_insn_failinvalid)
src/libs/Jamfile
+1
View File
@@ -12,6 +12,7 @@ SubInclude HAIKU_TOP src libs libsolv ;
SubInclude HAIKU_TOP src libs libtelnet ;
SubInclude HAIKU_TOP src libs linprog ;
SubInclude HAIKU_TOP src libs mapm ;
SubInclude HAIKU_TOP src libs nvmm ;
SubInclude HAIKU_TOP src libs posix_error_mapper ;
SubInclude HAIKU_TOP src libs print ;
SubInclude HAIKU_TOP src libs stdc++ ;
src/libs/nvmm/Jamfile
+10
View File
@@ -0,0 +1,10 @@
SubDir HAIKU_TOP src libs nvmm ;
UseHeaders [ FDirName $(HAIKU_TOP) headers compatibility bsd ] : true ;
UseHeaders [ FDirName $(HAIKU_TOP) headers libs nvmm ] : true ;
UseHeaders [ FDirName $(SUBDIR) ] : true ;
UseHeaders [ FDirName $(SUBDIR) driver include ] : true ;
SharedLibrary [ MultiArchDefaultGristFiles libnvmm.so ] :
libnvmm.c
;
src/libs/nvmm/driver
Symlink
+1
View File
@@ -0,0 +1 @@
../../add-ons/kernel/drivers/nvmm/
src/libs/nvmm/libnvmm.c
+20
View File
@@ -38,6 +38,10 @@
#include <fcntl.h>
#include <errno.h>
#if defined(__HAIKU__)
#include <machine/specialreg.h>
#endif
#include "nvmm.h"
static struct nvmm_capability __capability;
@@ -160,7 +164,11 @@ nvmm_init(void)
{
if (nvmm_fd != -1)
return 0;
#if defined(__HAIKU__)
nvmm_fd = open("/dev/misc/nvmm", O_RDONLY | O_CLOEXEC);
#else
nvmm_fd = open("/dev/nvmm", O_RDONLY | O_CLOEXEC);
#endif
if (nvmm_fd == -1)
return -1;
if (nvmm_capability(&__capability) == -1) {
@@ -171,7 +179,11 @@ nvmm_init(void)
if (__capability.version != NVMM_KERN_VERSION) {
close(nvmm_fd);
nvmm_fd = -1;
#if defined(__HAIKU__)
errno = EPERM;
#else
errno = EPROGMISMATCH;
#endif
return -1;
}
@@ -183,7 +195,11 @@ nvmm_root_init(void)
{
if (nvmm_fd != -1)
return 0;
#if defined(__HAIKU__)
nvmm_fd = open("/dev/misc/nvmm", O_WRONLY | O_CLOEXEC);
#else
nvmm_fd = open("/dev/nvmm", O_WRONLY | O_CLOEXEC);
#endif
if (nvmm_fd == -1)
return -1;
if (nvmm_capability(&__capability) == -1) {
@@ -194,7 +210,11 @@ nvmm_root_init(void)
if (__capability.version != NVMM_KERN_VERSION) {
close(nvmm_fd);
nvmm_fd = -1;
#if defined(__HAIKU__)
errno = EPERM;
#else
errno = EPROGMISMATCH;
#endif
return -1;
}
src/system/kernel/arch/x86/Jamfile
+5
View File
@@ -21,6 +21,7 @@ local archSpecificSources ;
if $(TARGET_ARCH) = x86_64 {
SEARCH_SOURCE += [ FDirName $(SUBDIR) 64 ] ;
SEARCH_SOURCE += [ FDirName $(SUBDIR) paging 64bit ] ;
SEARCH_SOURCE += [ FDirName $(SUBDIR) paging nested ] ;
archSpecificSources =
arch.S
@@ -40,6 +41,10 @@ if $(TARGET_ARCH) = x86_64 {
X86PagingMethod64Bit.cpp
X86PagingStructures64Bit.cpp
X86VMTranslationMap64Bit.cpp
# paging/nested
X86PagingMethodEPT.cpp
X86VMTranslationMapEPT.cpp
;
} else {
SEARCH_SOURCE += [ FDirName $(SUBDIR) 32 ] ;
src/system/kernel/arch/x86/paging/nested/X86PagingMethodEPT.cpp
+215
View File
@@ -0,0 +1,215 @@
/*
* Copyright 2024, Daniel Martin, dalmemail@gmail.com
* Copyright 2012, Alex Smith, alex@alex-smith.me.uk.
* Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
* Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
* Distributed under the terms of the MIT License.
*
* Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
* Distributed under the terms of the NewOS License.
*/
#include "paging/nested/X86PagingMethodEPT.h"
#include <stdlib.h>
#include <string.h>
#include <util/AutoLock.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include "paging/x86_physical_page_mapper.h"
#include "paging/x86_physical_page_mapper_mapped.h"
//#define TRACE_EPT_PAGING_METHOD
#ifdef TRACE_EPT_PAGING_METHOD
# define TRACE(x...) dprintf(x)
#else
# define TRACE(x...) ;
#endif
// #pragma mark - X86PagingMethodEPT
X86PagingMethodEPT::X86PagingMethodEPT()
:
fPhysicalPageMapper(NULL)
{
}
X86PagingMethodEPT::~X86PagingMethodEPT()
{
}
/*! Traverses down the paging structure hierarchy to find the page directory
for a virtual address, allocating new tables if required.
*/
/*static*/ uint64*
X86PagingMethodEPT::PageDirectoryForAddress(uint64* virtualPMLTop,
addr_t virtualAddress, bool isKernel, bool allocateTables,
vm_page_reservation* reservation,
TranslationMapPhysicalPageMapper* pageMapper, int32& mapCount)
{
uint64* virtualPML4 = virtualPMLTop;
// Get the PDPT.
uint64* pml4e = &virtualPML4[VADDR_TO_PML4E(virtualAddress)];
if ((*pml4e & EPT_PML4E_PRESENT) == 0) {
if (!allocateTables)
return NULL;
// Allocate a new PDPT.
vm_page* page = vm_page_allocate_page(reservation,
PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
DEBUG_PAGE_ACCESS_END(page);
phys_addr_t physicalPDPT
= (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;
TRACE("X86PagingMethodEPT::PageTableForAddress(): creating PDPT "
"for va %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n", virtualAddress,
physicalPDPT);
SetTableEntry(pml4e, (physicalPDPT & EPT_PML4E_ADDRESS_MASK)
| EPT_PML4E_PRESENT
| EPT_PML4E_WRITABLE
| EPT_PML4E_EXECUTABLE);
mapCount++;
}
uint64* virtualPDPT = (uint64*)pageMapper->GetPageTableAt(
*pml4e & EPT_PML4E_ADDRESS_MASK);
// Get the page directory.
uint64* pdpte = &virtualPDPT[VADDR_TO_PDPTE(virtualAddress)];
if ((*pdpte & EPT_PDPTE_PRESENT) == 0) {
if (!allocateTables)
return NULL;
// Allocate a new page directory.
vm_page* page = vm_page_allocate_page(reservation,
PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
DEBUG_PAGE_ACCESS_END(page);
phys_addr_t physicalPageDir
= (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;
TRACE("X86PagingMethodEPT::PageTableForAddress(): creating page "
"directory for va %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n",
virtualAddress, physicalPageDir);
SetTableEntry(pdpte, (physicalPageDir & EPT_PDPTE_ADDRESS_MASK)
| EPT_PDPTE_PRESENT
| EPT_PDPTE_WRITABLE
| EPT_PDPTE_EXECUTABLE);
mapCount++;
}
return (uint64*)pageMapper->GetPageTableAt(
*pdpte & EPT_PDPTE_ADDRESS_MASK);
}
/*static*/ uint64*
X86PagingMethodEPT::PageDirectoryEntryForAddress(uint64* virtualPMLTop,
addr_t virtualAddress, bool isKernel, bool allocateTables,
vm_page_reservation* reservation,
TranslationMapPhysicalPageMapper* pageMapper, int32& mapCount)
{
uint64* virtualPageDirectory = PageDirectoryForAddress(virtualPMLTop,
virtualAddress, isKernel, allocateTables, reservation, pageMapper,
mapCount);
if (virtualPageDirectory == NULL)
return NULL;
return &virtualPageDirectory[VADDR_TO_PDE(virtualAddress)];
}
/*! Traverses down the paging structure hierarchy to find the page table for a
virtual address, allocating new tables if required.
*/
/*static*/ uint64*
X86PagingMethodEPT::PageTableForAddress(uint64* virtualPMLTop,
addr_t virtualAddress, bool isKernel, bool allocateTables,
vm_page_reservation* reservation,
TranslationMapPhysicalPageMapper* pageMapper, int32& mapCount)
{
TRACE("X86PagingMethodEPT::PageTableForAddress(%#" B_PRIxADDR ", "
"%d)\n", virtualAddress, allocateTables);
uint64* pde = PageDirectoryEntryForAddress(virtualPMLTop, virtualAddress,
isKernel, allocateTables, reservation, pageMapper, mapCount);
if (pde == NULL)
return NULL;
if ((*pde & EPT_PDE_PRESENT) == 0) {
if (!allocateTables)
return NULL;
// Allocate a new page table.
vm_page* page = vm_page_allocate_page(reservation,
PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
DEBUG_PAGE_ACCESS_END(page);
phys_addr_t physicalPageTable
= (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;
TRACE("X86PagingMethodEPT::PageTableForAddress(): creating page "
"table for va %#" B_PRIxADDR " at %#" B_PRIxPHYSADDR "\n",
virtualAddress, physicalPageTable);
SetTableEntry(pde, (physicalPageTable & EPT_PDE_ADDRESS_MASK)
| EPT_PDE_PRESENT
| EPT_PDE_WRITABLE
| EPT_PDE_EXECUTABLE);
mapCount++;
}
return (uint64*)pageMapper->GetPageTableAt(*pde & EPT_PDE_ADDRESS_MASK);
}
/*static*/ uint64*
X86PagingMethodEPT::PageTableEntryForAddress(uint64* virtualPMLTop,
addr_t virtualAddress, bool isKernel, bool allocateTables,
vm_page_reservation* reservation,
TranslationMapPhysicalPageMapper* pageMapper, int32& mapCount)
{
uint64* virtualPageTable = PageTableForAddress(virtualPMLTop, virtualAddress,
isKernel, allocateTables, reservation, pageMapper, mapCount);
if (virtualPageTable == NULL)
return NULL;
return &virtualPageTable[VADDR_TO_PTE(virtualAddress)];
}
/*static*/ void
X86PagingMethodEPT::PutPageTableEntryInTable(uint64* entry,
phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType,
bool globalPage)
{
uint64 page = (physicalAddress & EPT_PTE_ADDRESS_MASK)
| EPT_PTE_PRESENT | EPT_PTE_IGNORE_PAT
| MemoryTypeToPageTableEntryFlags(memoryType);
if ((attributes & B_WRITE_AREA) != 0)
page |= EPT_PTE_WRITABLE;
if ((attributes & B_EXECUTE_AREA) != 0)
page |= EPT_PTE_EXECUTABLE;
// put it in the page table
SetTableEntry(entry, page);
}
src/system/kernel/arch/x86/paging/nested/X86PagingMethodEPT.h
+157
View File
@@ -0,0 +1,157 @@
/*
* Copyright 2024, Daniel Martin, dalmemail@gmail.com
* Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org.
* Copyright 2012, Alex Smith, alex@alex-smith.me.uk.
* Copyright 2010, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
#ifndef KERNEL_ARCH_X86_PAGING_METHOD_EPT_H
#define KERNEL_ARCH_X86_PAGING_METHOD_EPT_H
#include <atomic>
#include <KernelExport.h>
#include <lock.h>
#include <vm/vm_types.h>
#include "paging.h"
#include "paging/X86PagingMethod.h"
#include "paging/X86PagingStructures.h"
#include "paging/64bit/X86PagingMethod64Bit.h"
class TranslationMapPhysicalPageMapper;
class X86PhysicalPageMapper;
struct vm_page_reservation;
class X86PagingMethodEPT {
public:
X86PagingMethodEPT();
virtual ~X86PagingMethodEPT();
static X86PhysicalPageMapper* PhysicalPageMapper();
static uint64* PageDirectoryForAddress(uint64* virtualPML4,
addr_t virtualAddress, bool isKernel,
bool allocateTables,
vm_page_reservation* reservation,
TranslationMapPhysicalPageMapper*
pageMapper, int32& mapCount);
static uint64* PageDirectoryEntryForAddress(
uint64* virtualPML4, addr_t virtualAddress,
bool isKernel, bool allocateTables,
vm_page_reservation* reservation,
TranslationMapPhysicalPageMapper*
pageMapper, int32& mapCount);
static uint64* PageTableForAddress(uint64* virtualPML4,
addr_t virtualAddress, bool isKernel,
bool allocateTables,
vm_page_reservation* reservation,
TranslationMapPhysicalPageMapper*
pageMapper, int32& mapCount);
static uint64* PageTableEntryForAddress(uint64* virtualPML4,
addr_t virtualAddress, bool isKernel,
bool allocateTables,
vm_page_reservation* reservation,
TranslationMapPhysicalPageMapper*
pageMapper, int32& mapCount);
static void PutPageTableEntryInTable(
uint64* entry, phys_addr_t physicalAddress,
uint32 attributes, uint32 memoryType,
bool globalPage);
static void SetTableEntry(uint64_t* entry,
uint64_t newEntry);
static uint64_t SetTableEntryFlags(uint64_t* entryPointer,
uint64_t flags);
static uint64 TestAndSetTableEntry(uint64* entry,
uint64 newEntry, uint64 oldEntry);
static uint64_t ClearTableEntry(uint64_t* entryPointer);
static uint64_t ClearTableEntryFlags(uint64_t* entryPointer,
uint64_t flags);
static uint64 MemoryTypeToPageTableEntryFlags(
uint32 memoryType);
private:
X86PhysicalPageMapper* fPhysicalPageMapper;
};
static_assert(sizeof(std::atomic<uint64_t>) == sizeof(uint64_t),
"Non-trivial representation of atomic uint64_t.");
/*static*/ inline X86PhysicalPageMapper*
X86PagingMethodEPT::PhysicalPageMapper()
{
return static_cast<X86PagingMethod64Bit*>(gX86PagingMethod)->PhysicalPageMapper();
}
/*static*/ inline void
X86PagingMethodEPT::SetTableEntry(uint64_t* entryPointer, uint64_t newEntry)
{
auto& entry = *reinterpret_cast<std::atomic<uint64_t>*>(entryPointer);
entry.store(newEntry, std::memory_order_relaxed);
}
/*static*/ inline uint64_t
X86PagingMethodEPT::SetTableEntryFlags(uint64_t* entryPointer, uint64_t flags)
{
auto& entry = *reinterpret_cast<std::atomic<uint64_t>*>(entryPointer);
return entry.fetch_or(flags);
}
/*static*/ inline uint64
X86PagingMethodEPT::TestAndSetTableEntry(uint64* entry, uint64 newEntry, uint64 oldEntry)
{
return atomic_test_and_set64((int64*)entry, newEntry, oldEntry);
}
/*static*/ inline uint64_t
X86PagingMethodEPT::ClearTableEntry(uint64_t* entryPointer)
{
auto& entry = *reinterpret_cast<std::atomic<uint64_t>*>(entryPointer);
return entry.exchange(0);
}
/*static*/ inline uint64_t
X86PagingMethodEPT::ClearTableEntryFlags(uint64_t* entryPointer,
uint64_t flags)
{
auto& entry = *reinterpret_cast<std::atomic<uint64_t>*>(entryPointer);
return entry.fetch_and(~flags);
}
/*static*/ inline uint64
X86PagingMethodEPT::MemoryTypeToPageTableEntryFlags(uint32 memoryType)
{
switch (memoryType) {
case B_UNCACHED_MEMORY:
return EPT_PTE_CACHING_DISABLED;
case B_WRITE_COMBINING_MEMORY:
return EPT_PTE_WRITE_COMBINING;
case B_WRITE_THROUGH_MEMORY:
return EPT_PTE_WRITE_THROUGH;
case B_WRITE_PROTECTED_MEMORY:
return EPT_PTE_WRITE_PROTECT;
case B_WRITE_BACK_MEMORY:
default:
return EPT_PTE_WRITE_BACK;
}
}
#endif // KERNEL_ARCH_X86_PAGING_METHOD_EPT_H
src/system/kernel/arch/x86/paging/nested/X86VMTranslationMapEPT.cpp
+537
View File
@@ -0,0 +1,537 @@
/*
* Copyright 2024, Daniel Martin, dalmemail@gmail.com
* Copyright 2012, Alex Smith, alex@alex-smith.me.uk
* Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de.
* Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de.
* Distributed under the terms of the MIT License.
*
* Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
* Distributed under the terms of the NewOS License.
*/
#include "paging/nested/X86VMTranslationMapEPT.h"
#include <interrupts.h>
#include <slab/Slab.h>
#include <thread.h>
#include <util/AutoLock.h>
#include <util/ThreadAutoLock.h>
#include <vm/vm_page.h>
#include <vm/VMAddressSpace.h>
#include <vm/VMCache.h>
#include "paging/nested/X86PagingMethodEPT.h"
#include "paging/64bit/X86PagingStructures64Bit.h"
#include "paging/x86_physical_page_mapper.h"
//#define TRACE_X86_VM_TRANSLATION_MAP_EPT
#ifdef TRACE_X86_VM_TRANSLATION_MAP_EPT
# define TRACE(x...) dprintf(x)
#else
# define TRACE(x...) ;
#endif
// #pragma mark - X86VMTranslationMapEPT
X86VMTranslationMapEPT::X86VMTranslationMapEPT()
:
fPagingStructures(NULL)
{
}
X86VMTranslationMapEPT::~X86VMTranslationMapEPT()
{
TRACE("X86VMTranslationMapEPT::~X86VMTranslationMapEPT()\n");
if (fPagingStructures == NULL)
return;
if (fPageMapper != NULL) {
vm_page_reservation reservation = {};
phys_addr_t address;
vm_page* page;
// Free all structures in the PMLTop.
uint64* virtualPML4 = fPagingStructures->VirtualPMLTop();
for (uint32 i = 0; i < 512; i++) {
if ((virtualPML4[i] & EPT_PML4E_PRESENT) == 0)
continue;
uint64* virtualPDPT = (uint64*)fPageMapper->GetPageTableAt(
virtualPML4[i] & EPT_PML4E_ADDRESS_MASK);
for (uint32 j = 0; j < 512; j++) {
if ((virtualPDPT[j] & EPT_PDPTE_PRESENT) == 0)
continue;
uint64* virtualPageDir = (uint64*)fPageMapper->GetPageTableAt(
virtualPDPT[j] & EPT_PDPTE_ADDRESS_MASK);
for (uint32 k = 0; k < 512; k++) {
if ((virtualPageDir[k] & EPT_PDE_PRESENT) == 0)
continue;
address = virtualPageDir[k] & EPT_PDE_ADDRESS_MASK;
page = vm_lookup_page(address / B_PAGE_SIZE);
if (page == NULL) {
panic("page table %u %u %u on invalid page %#"
B_PRIxPHYSADDR "\n", i, j, k, address);
}
DEBUG_PAGE_ACCESS_START(page);
vm_page_free_etc(NULL, page, &reservation);
}
address = virtualPDPT[j] & EPT_PDPTE_ADDRESS_MASK;
page = vm_lookup_page(address / B_PAGE_SIZE);
if (page == NULL) {
panic("page directory %u %u on invalid page %#"
B_PRIxPHYSADDR "\n", i, j, address);
}
DEBUG_PAGE_ACCESS_START(page);
vm_page_free_etc(NULL, page, &reservation);
}
address = virtualPML4[i] & EPT_PML4E_ADDRESS_MASK;
page = vm_lookup_page(address / B_PAGE_SIZE);
if (page == NULL) {
panic("PDPT %u on invalid page %#" B_PRIxPHYSADDR "\n", i,
address);
}
DEBUG_PAGE_ACCESS_START(page);
vm_page_free_etc(NULL, page, &reservation);
}
vm_page_unreserve_pages(&reservation);
fPageMapper->Delete();
}
fPagingStructures->RemoveReference();
}
status_t
X86VMTranslationMapEPT::Init()
{
TRACE("X86VMTranslationMapEPT::Init()\n");
X86VMTranslationMap::Init(false);
fPagingStructures = new(std::nothrow) X86PagingStructures64Bit;
if (fPagingStructures == NULL)
return B_NO_MEMORY;
// Allocate a physical page mapper.
status_t error = X86PagingMethodEPT::PhysicalPageMapper()
->CreateTranslationMapPhysicalPageMapper(&fPageMapper);
if (error != B_OK)
return error;
// Allocate and clear the PMLTop.
uint64* virtualPMLTop = (uint64*)memalign(B_PAGE_SIZE, B_PAGE_SIZE);
if (virtualPMLTop == NULL)
return B_NO_MEMORY;
memset(virtualPMLTop, 0, B_PAGE_SIZE);
// Look up the PMLTop physical address.
phys_addr_t physicalPMLTop;
vm_get_page_mapping(VMAddressSpace::KernelID(), (addr_t)virtualPMLTop,
&physicalPMLTop);
// Initialize the paging structures.
fPagingStructures->Init(virtualPMLTop, physicalPMLTop);
return B_OK;
}
size_t
X86VMTranslationMapEPT::MaxPagesNeededToMap(addr_t start, addr_t end) const
{
// If start == 0, the actual base address is not yet known to the caller and
// we shall assume the worst case, which is where the start address is the
// last page covered by a PDPT or PML4.
if (start == 0) {
start = k64BitPDPTRange - B_PAGE_SIZE;
end += start;
}
size_t requiredPML4s = 0;
size_t requiredPDPTs = end / k64BitPDPTRange + 1
- start / k64BitPDPTRange;
size_t requiredPageDirs = end / k64BitPageDirectoryRange + 1
- start / k64BitPageDirectoryRange;
size_t requiredPageTables = end / k64BitPageTableRange + 1
- start / k64BitPageTableRange;
return requiredPML4s + requiredPDPTs + requiredPageDirs
+ requiredPageTables;
}
status_t
X86VMTranslationMapEPT::Map(addr_t virtualAddress, phys_addr_t physicalAddress,
uint32 attributes, uint32 memoryType, vm_page_reservation* reservation)
{
TRACE("X86VMTranslationMapEPT::Map(%#" B_PRIxADDR ", %#" B_PRIxPHYSADDR
")\n", virtualAddress, physicalAddress);
ThreadCPUPinner pinner(thread_get_current_thread());
// Look up the page table for the virtual address, allocating new tables
// if required. Shouldn't fail.
uint64* entry = X86PagingMethodEPT::PageTableEntryForAddress(
fPagingStructures->VirtualPMLTop(), virtualAddress, fIsKernelMap,
true, reservation, fPageMapper, fMapCount);
ASSERT(entry != NULL);
// The entry should not already exist.
ASSERT_PRINT((*entry & EPT_PTE_PRESENT) == 0,
"virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx64,
virtualAddress, *entry);
// Fill in the table entry.
X86PagingMethodEPT::PutPageTableEntryInTable(entry, physicalAddress,
attributes, memoryType, fIsKernelMap);
// Note: We don't need to invalidate the TLB for this address, as previously
// the entry was not present and the TLB doesn't cache those entries.
fMapCount++;
return 0;
}
status_t
X86VMTranslationMapEPT::Unmap(addr_t start, addr_t end)
{
ASSERT_UNREACHABLE();
return B_ERROR;
}
status_t
X86VMTranslationMapEPT::UnmapPage(VMArea* area, addr_t address,
bool updatePageQueue, bool deletingAddressSpace, uint32* _flags)
{
ASSERT(address % B_PAGE_SIZE == 0);
ASSERT(_flags == NULL || !updatePageQueue);
TRACE("X86VMTranslationMapEPT::UnmapPage(%#" B_PRIxADDR ")\n", address);
ThreadCPUPinner pinner(thread_get_current_thread());
// Look up the page table for the virtual address.
uint64* entry = X86PagingMethodEPT::PageTableEntryForAddress(
fPagingStructures->VirtualPMLTop(), address, fIsKernelMap,
false, NULL, fPageMapper, fMapCount);
if (entry == NULL)
return B_ENTRY_NOT_FOUND;
RecursiveLocker locker(fLock);
uint64 oldEntry = X86PagingMethodEPT::ClearTableEntry(entry);
pinner.Unlock();
if ((oldEntry & EPT_PTE_PRESENT) == 0)
return B_ENTRY_NOT_FOUND;
fMapCount--;
if ((oldEntry & EPT_PTE_ACCESSED) != 0) {
// Note, that we only need to invalidate the address, if the
// accessed flags was set, since only then the entry could have been
// in any TLB.
if (!deletingAddressSpace)
InvalidatePage(address);
if (_flags == NULL) {
Flush();
// flush explicitly, since we directly use the lock
}
// NOTE: Between clearing the page table entry and Flush() other
// processors (actually even this processor with another thread of the
// same team) could still access the page in question via their cached
// entry. We can obviously lose a modified flag in this case, with the
// effect that the page looks unmodified (and might thus be recycled),
// but is actually modified.
// In most cases this is harmless, but for vm_remove_all_page_mappings()
// this is actually a problem.
// Interestingly FreeBSD seems to ignore this problem as well
// (cf. pmap_remove_all()), unless I've missed something.
}
if (_flags == NULL) {
locker.Detach();
// PageUnmapped() will unlock for us
PageUnmapped(area, (oldEntry & EPT_PTE_ADDRESS_MASK) / B_PAGE_SIZE,
(oldEntry & EPT_PTE_ACCESSED) != 0,
(oldEntry & EPT_PTE_DIRTY) != 0, updatePageQueue);
} else {
uint32 flags = PAGE_PRESENT;
if ((oldEntry & EPT_PTE_ACCESSED) != 0)
flags |= PAGE_ACCESSED;
if ((oldEntry & EPT_PTE_DIRTY) != 0)
flags |= PAGE_MODIFIED;
*_flags = flags;
}
return B_OK;
}
void
X86VMTranslationMapEPT::UnmapPages(VMArea* area, addr_t base, size_t size,
bool updatePageQueue, bool deletingAddressSpace)
{
if (size == 0)
return;
addr_t start = base;
addr_t end = base + size - 1;
TRACE("X86VMTranslationMapEPT::UnmapPages(%p, %#" B_PRIxADDR ", %#"
B_PRIxADDR ")\n", area, start, end);
VMAreaMappings queue;
RecursiveLocker locker(fLock);
ThreadCPUPinner pinner(thread_get_current_thread());
do {
uint64* pageTable = X86PagingMethodEPT::PageTableForAddress(
fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false,
NULL, fPageMapper, fMapCount);
if (pageTable == NULL) {
// Move on to the next page table.
start = ROUNDUP(start + 1, k64BitPageTableRange);
continue;
}
for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount;
index < k64BitTableEntryCount && start < end;
index++, start += B_PAGE_SIZE) {
uint64 oldEntry = X86PagingMethodEPT::ClearTableEntry(
&pageTable[index]);
if ((oldEntry & EPT_PTE_PRESENT) == 0)
continue;
fMapCount--;
if ((oldEntry & EPT_PTE_ACCESSED) != 0) {
// Note, that we only need to invalidate the address, if the
// accessed flags was set, since only then the entry could have
// been in any TLB.
if (!deletingAddressSpace)
InvalidatePage(start);
}
if (area->cache_type != CACHE_TYPE_DEVICE) {
page_num_t page = (oldEntry & EPT_PTE_ADDRESS_MASK) / B_PAGE_SIZE;
PageUnmapped(area, page,
(oldEntry & EPT_PTE_ACCESSED) != 0,
(oldEntry & EPT_PTE_DIRTY) != 0,
updatePageQueue, &queue);
}
}
Flush();
// flush explicitly, since we directly use the lock
} while (start != 0 && start < end);
// TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not
// really critical here, as in all cases this method is used, the unmapped
// area range is unmapped for good (resized/cut) and the pages will likely
// be freed.
locker.Unlock();
// free removed mappings
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
status_t
X86VMTranslationMapEPT::Query(addr_t virtualAddress,
phys_addr_t* _physicalAddress, uint32* _flags)
{
*_flags = 0;
*_physicalAddress = 0;
ThreadCPUPinner pinner(thread_get_current_thread());
// This function may be called on the physical map area, so we must handle
// large pages here. Look up the page directory entry for the virtual
// address.
uint64* pde = X86PagingMethodEPT::PageDirectoryEntryForAddress(
fPagingStructures->VirtualPMLTop(), virtualAddress, fIsKernelMap,
false, NULL, fPageMapper, fMapCount);
if (pde == NULL || (*pde & EPT_PDE_PRESENT) == 0)
return B_OK;
uint64 entry;
uint64* virtualPageTable = (uint64*)fPageMapper->GetPageTableAt(
*pde & EPT_PDE_ADDRESS_MASK);
entry = virtualPageTable[VADDR_TO_PTE(virtualAddress)];
*_physicalAddress = entry & EPT_PTE_ADDRESS_MASK;
// Translate the page state flags.
*_flags |= ((entry & EPT_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0)
| B_READ_AREA
| ((entry & EPT_PTE_EXECUTABLE) != 0 ? B_EXECUTE_AREA : 0)
| ((entry & EPT_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0)
| ((entry & EPT_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0)
| ((entry & EPT_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0);
TRACE("X86VMTranslationMapEPT::Query(%#" B_PRIxADDR ") -> %#"
B_PRIxPHYSADDR " %#" B_PRIx32 " (entry: %#" B_PRIx64 ")\n",
virtualAddress, *_physicalAddress, *_flags, entry);
return B_OK;
}
status_t
X86VMTranslationMapEPT::QueryInterrupt(addr_t virtualAddress,
phys_addr_t* _physicalAddress, uint32* _flags)
{
// With our page mapper, there is no difference in getting a page table
// when interrupts are enabled or disabled, so just call Query().
return Query(virtualAddress, _physicalAddress, _flags);
}
status_t
X86VMTranslationMapEPT::Protect(addr_t start, addr_t end, uint32 attributes,
uint32 memoryType)
{
ASSERT_UNREACHABLE();
return B_ERROR;
}
status_t
X86VMTranslationMapEPT::ClearFlags(addr_t address, uint32 flags)
{
TRACE("X86VMTranslationMapEPT::ClearFlags(%#" B_PRIxADDR ", %#" B_PRIx32
")\n", address, flags);
ThreadCPUPinner pinner(thread_get_current_thread());
uint64* entry = X86PagingMethodEPT::PageTableEntryForAddress(
fPagingStructures->VirtualPMLTop(), address, fIsKernelMap,
false, NULL, fPageMapper, fMapCount);
if (entry == NULL)
return B_OK;
uint64 flagsToClear = ((flags & PAGE_MODIFIED) ? EPT_PTE_DIRTY : 0)
| ((flags & PAGE_ACCESSED) ? EPT_PTE_ACCESSED : 0);
uint64 oldEntry = X86PagingMethodEPT::ClearTableEntryFlags(entry,
flagsToClear);
if ((oldEntry & flagsToClear) != 0)
InvalidatePage(address);
return B_OK;
}
bool
X86VMTranslationMapEPT::ClearAccessedAndModified(VMArea* area, addr_t address,
bool unmapIfUnaccessed, bool& _modified)
{
ASSERT(address % B_PAGE_SIZE == 0);
TRACE("X86VMTranslationMapEPT::ClearAccessedAndModified(%#" B_PRIxADDR
")\n", address);
RecursiveLocker locker(fLock);
ThreadCPUPinner pinner(thread_get_current_thread());
uint64* entry = X86PagingMethodEPT::PageTableEntryForAddress(
fPagingStructures->VirtualPMLTop(), address, fIsKernelMap,
false, NULL, fPageMapper, fMapCount);
if (entry == NULL)
return false;
uint64 oldEntry;
if (unmapIfUnaccessed) {
while (true) {
oldEntry = *entry;
if ((oldEntry & EPT_PTE_PRESENT) == 0) {
// page mapping not valid
return false;
}
if (oldEntry & EPT_PTE_ACCESSED) {
// page was accessed -- just clear the flags
oldEntry = X86PagingMethodEPT::ClearTableEntryFlags(entry,
EPT_PTE_ACCESSED | EPT_PTE_DIRTY);
break;
}
// page hasn't been accessed -- unmap it
if (X86PagingMethodEPT::TestAndSetTableEntry(entry, 0, oldEntry)
== oldEntry) {
break;
}
// something changed -- check again
}
} else {
oldEntry = X86PagingMethodEPT::ClearTableEntryFlags(entry,
EPT_PTE_ACCESSED | EPT_PTE_DIRTY);
}
pinner.Unlock();
_modified = (oldEntry & EPT_PTE_DIRTY) != 0;
if ((oldEntry & EPT_PTE_ACCESSED) != 0) {
// Note, that we only need to invalidate the address, if the
// accessed flags was set, since only then the entry could have been
// in any TLB.
InvalidatePage(address);
Flush();
return true;
}
if (!unmapIfUnaccessed)
return false;
// We have unmapped the address. Do the "high level" stuff.
fMapCount--;
locker.Detach();
// UnaccessedPageUnmapped() will unlock for us
UnaccessedPageUnmapped(area,
(oldEntry & EPT_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
return false;
}
X86PagingStructures*
X86VMTranslationMapEPT::PagingStructures() const
{
return fPagingStructures;
}
src/system/kernel/arch/x86/paging/nested/X86VMTranslationMapEPT.h
+62
View File
@@ -0,0 +1,62 @@
/*
* Copyright 2024, Daniel Martin, dalmemail@gmail.com
* Copyright 2012, Alex Smith, alex@alex-smith.me.uk.
* Copyright 2010, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
#ifndef KERNEL_ARCH_X86_PAGING_NESTED_X86_VM_TRANSLATION_MAP_EPT_H
#define KERNEL_ARCH_X86_PAGING_NESTED_X86_VM_TRANSLATION_MAP_EPT_H
#include "paging/X86VMTranslationMap.h"
#include "paging/64bit/X86PagingStructures64Bit.h"
struct X86VMTranslationMapEPT final : X86VMTranslationMap {
X86VMTranslationMapEPT();
virtual ~X86VMTranslationMapEPT();
status_t Init();
virtual size_t MaxPagesNeededToMap(addr_t start,
addr_t end) const;
virtual status_t Map(addr_t virtualAddress,
phys_addr_t physicalAddress,
uint32 attributes, uint32 memoryType,
vm_page_reservation* reservation);
virtual status_t Unmap(addr_t start, addr_t end);
virtual status_t UnmapPage(VMArea* area, addr_t address,
bool updatePageQueue,
bool deletingAddressSpace, uint32* _flags);
virtual void UnmapPages(VMArea* area, addr_t base,
size_t size, bool updatePageQueue,
bool deletingAddressSpace);
virtual status_t Query(addr_t virtualAddress,
phys_addr_t* _physicalAddress,
uint32* _flags);
virtual status_t QueryInterrupt(addr_t virtualAddress,
phys_addr_t* _physicalAddress,
uint32* _flags);
virtual status_t Protect(addr_t base, addr_t top,
uint32 attributes, uint32 memoryType);
virtual status_t ClearFlags(addr_t virtualAddress,
uint32 flags);
virtual bool ClearAccessedAndModified(
VMArea* area, addr_t address,
bool unmapIfUnaccessed,
bool& _modified);
virtual X86PagingStructures* PagingStructures() const;
private:
X86PagingStructures64Bit* fPagingStructures;
};
#endif // KERNEL_ARCH_X86_PAGING_NESTED_X86_VM_TRANSLATION_MAP_EPT_H
src/system/kernel/arch/x86/paging/nested/paging.h
+47
View File
@@ -0,0 +1,47 @@
/*
* Copyright 2026, Haiku, Inc. All rights reserved.
* Distributed under the terms of the MIT License.
*/
#ifndef KERNEL_ARCH_X86_PAGING_NESTED_PAGING_H
#define KERNEL_ARCH_X86_PAGING_NESTED_PAGING_H
#include "../64bit/paging.h"
// EPT PML4E bits
#define EPT_PML4E_PRESENT (1LL << 0)
#define EPT_PML4E_WRITABLE (1LL << 1)
#define EPT_PML4E_EXECUTABLE (1LL << 2)
#define EPT_PML4E_ADDRESS_MASK 0x0000fffffffff000L
// EPT PDPTE bits
#define EPT_PDPTE_PRESENT (1LL << 0)
#define EPT_PDPTE_WRITABLE (1LL << 1)
#define EPT_PDPTE_EXECUTABLE (1LL << 2)
#define EPT_PDPTE_ADDRESS_MASK 0x0000fffffffff000L
// EPT PDE bits
#define EPT_PDE_PRESENT (1LL << 0)
#define EPT_PDE_WRITABLE (1LL << 1)
#define EPT_PDE_EXECUTABLE (1LL << 2)
#define EPT_PDE_ADDRESS_MASK 0x0000fffffffff000L
// EPT PTE bits.
#define EPT_PTE_PRESENT (1LL << 0) // FIXME: actually READABLE
#define EPT_PTE_WRITABLE (1LL << 1)
#define EPT_PTE_EXECUTABLE (1LL << 2)
#define EPT_PTE_IGNORE_PAT (1LL << 6)
#define EPT_PTE_ACCESSED (1LL << 8)
#define EPT_PTE_DIRTY (1LL << 9)
#define EPT_PTE_ADDRESS_MASK 0x0000fffffffff000L
// Memory types
#define EPT_PTE_CACHING_DISABLED (0LL << 3)
#define EPT_PTE_WRITE_COMBINING (1LL << 3)
#define EPT_PTE_WRITE_THROUGH (4LL << 3)
#define EPT_PTE_WRITE_PROTECT (5LL << 3)
#define EPT_PTE_WRITE_BACK (6LL << 3)
#endif // KERNEL_ARCH_X86_PAGING_NESTED_PAGING_H
src/tests/add-ons/kernel/drivers/Jamfile
+1
View File
@@ -2,5 +2,6 @@ SubDir HAIKU_TOP src tests add-ons kernel drivers ;
SubInclude HAIKU_TOP src tests add-ons kernel drivers audio ;
SubInclude HAIKU_TOP src tests add-ons kernel drivers hpet ;
SubInclude HAIKU_TOP src tests add-ons kernel drivers nvmm ;
SubInclude HAIKU_TOP src tests add-ons kernel drivers random ;
SubInclude HAIKU_TOP src tests add-ons kernel drivers tty ;
src/tests/add-ons/kernel/drivers/nvmm/Jamfile
+23
View File
@@ -0,0 +1,23 @@
SubDir HAIKU_TOP src tests add-ons kernel drivers nvmm ;
UseHeaders [ FDirName $(HAIKU_TOP) headers compatibility bsd ] : true ;
UseHeaders [ FDirName $(HAIKU_TOP) headers libs nvmm ] : true ;
UseHeaders [ FDirName $(HAIKU_TOP) src libs nvmm ] : true ;
UseHeaders [ FDirName $(HAIKU_TOP) src libs nvmm driver include ] : true ;
UsePrivateKernelHeaders ;
SEARCH_SOURCE += [ FDirName $(SUBDIR) h_io_assist ] ;
SEARCH_SOURCE += [ FDirName $(SUBDIR) h_mem_assist ] ;
SimpleTest libnvmm_io_assist :
h_io_assist.c
h_io_assist_asm.S
: libnvmm.so
;
SimpleTest libnvmm_mem_assist :
h_mem_assist.c
h_mem_assist_asm.S
: libnvmm.so
;
src/tests/add-ons/kernel/drivers/nvmm/h_io_assist/h_io_assist.c
+10 -2
View File
@@ -32,12 +32,16 @@
#include <stdbool.h>
#include <unistd.h>
#include <string.h>
#include <err.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/mman.h>
#if defined(__HAIKU__)
#include <machine/specialreg.h>
#else
#include <err.h>
#include <machine/segments.h>
#include <machine/psl.h>
#endif
#include <nvmm.h>
@@ -46,7 +50,7 @@
#include <machine/pte.h>
#define PAGE_SIZE 4096
#else /* DragonFly */
#elif defined(__DragonFly__) /* DragonFly */
#include <machine/pmap.h>
#define PTE_P X86_PG_V /* 0x001: P (Valid) */
@@ -54,6 +58,10 @@
#define PSL_MBO PSL_RESERVED_DEFAULT /* 0x00000002 */
#define SDT_SYS386BSY SDT_SYSBSY /* 11: system 64-bit TSS busy */
#elif defined(__HAIKU__)
#include "../haiku_defs.h"
#endif /* __NetBSD__ */
#define IO_SIZE 128
src/tests/add-ons/kernel/drivers/nvmm/h_mem_assist/h_mem_assist.c
+10 -2
View File
@@ -32,12 +32,16 @@
#include <stdbool.h>
#include <unistd.h>
#include <string.h>
#include <err.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/mman.h>
#if defined(__HAIKU__)
#include <machine/specialreg.h>
#else
#include <err.h>
#include <machine/segments.h>
#include <machine/psl.h>
#endif
#include <nvmm.h>
@@ -46,7 +50,7 @@
#include <machine/pte.h>
#define PAGE_SIZE 4096
#else /* DragonFly */
#elif defined(__DragonFly__) /* DragonFly */
#include <machine/pmap.h>
#define PTE_P X86_PG_V /* 0x001: P (Valid) */
@@ -54,6 +58,10 @@
#define PSL_MBO PSL_RESERVED_DEFAULT /* 0x00000002 */
#define SDT_SYS386BSY SDT_SYSBSY /* 11: system 64-bit TSS busy */
#elif defined(__HAIKU__)
#include "../haiku_defs.h"
#endif /* __NetBSD__ */
static uint8_t mmiobuf[PAGE_SIZE];
src/tests/add-ons/kernel/drivers/nvmm/haiku_defs.h
+39
View File
@@ -0,0 +1,39 @@
#ifndef LIBNVMM_TESTS_HAIKU_DEFS
#define LIBNVMM_TESTS_HAIKU_DEFS
#include <OS.h>
#define PAGE_SIZE B_PAGE_SIZE
#define _ASSEMBLER
#include <arch/x86/descriptors.h>
#undef _ASSEMBLER
// taken from nvmm_os.h / DragonFlyBSD's segments.h
#define GSEL(s,r) (((s) << 3) | r) /* a global selector */
#define GCODE_SEL KERNEL_CODE_SEGMENT /* Kernel Code Descriptor */
#define GDATA_SEL KERNEL_DATA_SEGMENT /* Kernel Data Descriptor */
#define SEL_KPL DPL_KERNEL /* kernel privilege level */
// taken from libnvmm_x86.c
#define PTE_P 0x0000000000000001 /* Present */
#define PTE_W 0x0000000000000002 /* Write */
// taken from NetBSD
#define PSL_MBO 0x00000002
#define SDT_MEMRWA 19
#define SDT_MEMERA 27
#define SDT_SYSLDT 2
#define SDT_SYS386BSY 11
typedef uint64_t pt_entry_t;
static void err(int error_code, char *str)
{
printf(str);
printf("\n");
exit(error_code);
}
#define errx(error_code, str) err(error_code, str)
#endif /* LIBNVMM_TESTS_HAIKU_DEFS */