riscv: use atomic CSR bit set/clear operations, refactor
Fix race conditions that cause broken timer interrupts. Change-Id: I78e13a18d394b1566977e894a1def16a66c9ca5f Reviewed-on: https://review.haiku-os.org/c/haiku/+/5883 Reviewed-by: X512 <[email protected]> Reviewed-by: Alex von Gluck IV <[email protected]> Tested-by: Commit checker robot <[email protected]>
This commit is contained in:
@@ -19,28 +19,21 @@
|
|||||||
static inline bool
|
static inline bool
|
||||||
get_ac()
|
get_ac()
|
||||||
{
|
{
|
||||||
SstatusReg status(Sstatus());
|
return SstatusReg{.val = Sstatus()}.sum;
|
||||||
return status.sum != 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static inline void
|
static inline void
|
||||||
set_ac()
|
set_ac()
|
||||||
{
|
{
|
||||||
// TODO: Could be done atomically via CSRRS?
|
SetBitsSstatus(SstatusReg{.sum = 1}.val);
|
||||||
SstatusReg status(Sstatus());
|
|
||||||
status.sum = 1;
|
|
||||||
SetSstatus(status.val);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static inline void
|
static inline void
|
||||||
clear_ac()
|
clear_ac()
|
||||||
{
|
{
|
||||||
// TODO: Could be done atomically with CSRRC?
|
ClearBitsSstatus(SstatusReg{.sum = 1}.val);
|
||||||
SstatusReg status(Sstatus());
|
|
||||||
status.sum = 0;
|
|
||||||
SetSstatus(status.val);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -20,22 +20,15 @@
|
|||||||
static inline void
|
static inline void
|
||||||
arch_int_enable_interrupts_inline(void)
|
arch_int_enable_interrupts_inline(void)
|
||||||
{
|
{
|
||||||
// TODO: Use atomic CSRRS?
|
SetBitsSstatus(SstatusReg{.ie = 1 << modeS}.val);
|
||||||
SstatusReg status(Sstatus());
|
|
||||||
status.ie |= (1 << modeS);
|
|
||||||
SetSstatus(status.val);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static inline int
|
static inline int
|
||||||
arch_int_disable_interrupts_inline(void)
|
arch_int_disable_interrupts_inline(void)
|
||||||
{
|
{
|
||||||
// TODO: Use atomic CSRRC?
|
SstatusReg oldStatus{.val = GetAndClearBitsSstatus(SstatusReg{.ie = 1 << modeS}.val)};
|
||||||
SstatusReg status(Sstatus());
|
return ((1 << modeS) & oldStatus.ie) != 0;
|
||||||
int oldState = ((1 << modeS) & status.ie) != 0;
|
|
||||||
status.ie &= ~(1 << modeS);
|
|
||||||
SetSstatus(status.val);
|
|
||||||
return oldState;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -50,7 +43,7 @@ arch_int_restore_interrupts_inline(int oldState)
|
|||||||
static inline bool
|
static inline bool
|
||||||
arch_int_are_interrupts_enabled_inline(void)
|
arch_int_are_interrupts_enabled_inline(void)
|
||||||
{
|
{
|
||||||
SstatusReg status(Sstatus());
|
SstatusReg status{.val = Sstatus()};
|
||||||
return ((1 << modeS) & status.ie) != 0;
|
return ((1 << modeS) & status.ie) != 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -27,59 +27,49 @@ enum {
|
|||||||
extStatusDirty = 3,
|
extStatusDirty = 3,
|
||||||
};
|
};
|
||||||
|
|
||||||
struct MstatusReg {
|
union MstatusReg {
|
||||||
union {
|
struct {
|
||||||
struct {
|
uint64 ie: 4; // interrupt enable
|
||||||
uint64 ie: 4; // interrupt enable
|
uint64 pie: 4; // previous interrupt enable
|
||||||
uint64 pie: 4; // previous interrupt enable
|
uint64 spp: 1; // previous mode (supervisor)
|
||||||
uint64 spp: 1; // previous mode (supervisor)
|
uint64 unused1: 2;
|
||||||
uint64 unused1: 2;
|
uint64 mpp: 2; // previous mode (machine)
|
||||||
uint64 mpp: 2; // previous mode (machine)
|
uint64 fs: 2; // FPU status
|
||||||
uint64 fs: 2; // FPU status
|
uint64 xs: 2; // extensions status
|
||||||
uint64 xs: 2; // extensions status
|
uint64 mprv: 1; // modify privilege
|
||||||
uint64 mprv: 1; // modify privilege
|
uint64 sum: 1; // permit supervisor user memory access
|
||||||
uint64 sum: 1; // permit supervisor user memory access
|
uint64 mxr: 1; // make executable readable
|
||||||
uint64 mxr: 1; // make executable readable
|
uint64 tvm: 1; // trap virtual memory
|
||||||
uint64 tvm: 1; // trap virtual memory
|
uint64 tw: 1; // timeout wait (trap WFI)
|
||||||
uint64 tw: 1; // timeout wait (trap WFI)
|
uint64 tsr: 1; // trap SRET
|
||||||
uint64 tsr: 1; // trap SRET
|
uint64 unused2: 9;
|
||||||
uint64 unused2: 9;
|
uint64 uxl: 2; // U-mode XLEN
|
||||||
uint64 uxl: 2; // U-mode XLEN
|
uint64 sxl: 2; // S-mode XLEN
|
||||||
uint64 sxl: 2; // S-mode XLEN
|
uint64 unused3: 27;
|
||||||
uint64 unused3: 27;
|
uint64 sd: 1; // status dirty
|
||||||
uint64 sd: 1; // status dirty
|
|
||||||
};
|
|
||||||
uint64 val;
|
|
||||||
};
|
};
|
||||||
|
uint64 val;
|
||||||
MstatusReg() {}
|
|
||||||
MstatusReg(uint64 val): val(val) {}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct SstatusReg {
|
union SstatusReg {
|
||||||
union {
|
struct {
|
||||||
struct {
|
uint64 ie: 2; // interrupt enable
|
||||||
uint64 ie: 2; // interrupt enable
|
uint64 unused1: 2;
|
||||||
uint64 unused1: 2;
|
uint64 pie: 2; // previous interrupt enable
|
||||||
uint64 pie: 2; // previous interrupt enable
|
uint64 unused2: 2;
|
||||||
uint64 unused2: 2;
|
uint64 spp: 1; // previous mode (supervisor)
|
||||||
uint64 spp: 1; // previous mode (supervisor)
|
uint64 unused3: 4;
|
||||||
uint64 unused3: 4;
|
uint64 fs: 2; // FPU status
|
||||||
uint64 fs: 2; // FPU status
|
uint64 xs: 2; // extensions status
|
||||||
uint64 xs: 2; // extensions status
|
uint64 unused4: 1;
|
||||||
uint64 unused4: 1;
|
uint64 sum: 1; // permit supervisor user memory access
|
||||||
uint64 sum: 1; // permit supervisor user memory access
|
uint64 mxr: 1; // make executable readable
|
||||||
uint64 mxr: 1; // make executable readable
|
uint64 unused5: 12;
|
||||||
uint64 unused5: 12;
|
uint64 uxl: 2; // U-mode XLEN
|
||||||
uint64 uxl: 2; // U-mode XLEN
|
uint64 unused6: 29;
|
||||||
uint64 unused6: 29;
|
uint64 sd: 1; // status dirty
|
||||||
uint64 sd: 1; // status dirty
|
|
||||||
};
|
|
||||||
uint64 val;
|
|
||||||
};
|
};
|
||||||
|
uint64 val;
|
||||||
SstatusReg() {}
|
|
||||||
SstatusReg(uint64 val): val(val) {}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
enum {
|
enum {
|
||||||
@@ -129,7 +119,6 @@ enum {
|
|||||||
};
|
};
|
||||||
|
|
||||||
enum {
|
enum {
|
||||||
pageSize = 4096,
|
|
||||||
pageBits = 12,
|
pageBits = 12,
|
||||||
pteCount = 512,
|
pteCount = 512,
|
||||||
pteIdxBits = 9,
|
pteIdxBits = 9,
|
||||||
@@ -146,19 +135,14 @@ enum {
|
|||||||
pteDirty = 7,
|
pteDirty = 7,
|
||||||
};
|
};
|
||||||
|
|
||||||
struct Pte {
|
union Pte {
|
||||||
union {
|
struct {
|
||||||
struct {
|
uint64 flags: 8;
|
||||||
uint64 flags: 8;
|
uint64 rsw: 2;
|
||||||
uint64 rsw: 2;
|
uint64 ppn: 44;
|
||||||
uint64 ppn: 44;
|
uint64 reserved: 10;
|
||||||
uint64 reserved: 10;
|
|
||||||
};
|
|
||||||
uint64 val;
|
|
||||||
};
|
};
|
||||||
|
uint64 val;
|
||||||
Pte() {}
|
|
||||||
Pte(uint64 val): val(val) {}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
enum {
|
enum {
|
||||||
@@ -169,18 +153,13 @@ enum {
|
|||||||
satpModeSv64 = 11,
|
satpModeSv64 = 11,
|
||||||
};
|
};
|
||||||
|
|
||||||
struct SatpReg {
|
union SatpReg {
|
||||||
union {
|
struct {
|
||||||
struct {
|
uint64 ppn: 44;
|
||||||
uint64 ppn: 44;
|
uint64 asid: 16;
|
||||||
uint64 asid: 16;
|
uint64 mode: 4;
|
||||||
uint64 mode: 4;
|
|
||||||
};
|
|
||||||
uint64 val;
|
|
||||||
};
|
};
|
||||||
|
uint64 val;
|
||||||
SatpReg() {}
|
|
||||||
SatpReg(uint64 val): val(val) {}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
static B_ALWAYS_INLINE uint64 VirtAdrPte(uint64 physAdr, uint32 level)
|
static B_ALWAYS_INLINE uint64 VirtAdrPte(uint64 physAdr, uint32 level)
|
||||||
@@ -190,134 +169,84 @@ static B_ALWAYS_INLINE uint64 VirtAdrPte(uint64 physAdr, uint32 level)
|
|||||||
|
|
||||||
static B_ALWAYS_INLINE uint64 VirtAdrOfs(uint64 physAdr)
|
static B_ALWAYS_INLINE uint64 VirtAdrOfs(uint64 physAdr)
|
||||||
{
|
{
|
||||||
return physAdr % pageSize;
|
return physAdr % PAGESIZE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define CSR_REG_MACRO(Name, value) \
|
||||||
|
static B_ALWAYS_INLINE uint64 Name() { \
|
||||||
|
uint64 x; asm volatile("csrr %0, " #value : "=r" (x)); return x;} \
|
||||||
|
static B_ALWAYS_INLINE void Set##Name(uint64 x) { \
|
||||||
|
asm volatile("csrw " #value ", %0" : : "r" (x));} \
|
||||||
|
static B_ALWAYS_INLINE void SetBits##Name(uint64 x) { \
|
||||||
|
asm volatile("csrs " #value ", %0" : : "r" (x));} \
|
||||||
|
static B_ALWAYS_INLINE void ClearBits##Name(uint64 x) { \
|
||||||
|
asm volatile("csrc " #value ", %0" : : "r" (x));} \
|
||||||
|
static B_ALWAYS_INLINE uint64 GetAndSetBits##Name(uint64 x) { \
|
||||||
|
uint64 res; \
|
||||||
|
asm volatile("csrrs %0, " #value ", %1" : "=r" (res) : "r" (x)); \
|
||||||
|
return res; \
|
||||||
|
} \
|
||||||
|
static B_ALWAYS_INLINE uint64 GetAndClearBits##Name(uint64 x) { \
|
||||||
|
uint64 res; \
|
||||||
|
asm volatile("csrrc %0, " #value ", %1" : "=r" (res) : "r" (x)); \
|
||||||
|
return res; \
|
||||||
|
} \
|
||||||
|
|
||||||
// CPU core ID
|
// CPU core ID
|
||||||
static B_ALWAYS_INLINE uint64 Mhartid() {
|
CSR_REG_MACRO(Mhartid, mhartid)
|
||||||
uint64 x; asm volatile("csrr %0, mhartid" : "=r" (x)); return x;}
|
|
||||||
|
|
||||||
// status register
|
// status register
|
||||||
static B_ALWAYS_INLINE uint64 Mstatus() {
|
CSR_REG_MACRO(Mstatus, mstatus)
|
||||||
uint64 x; asm volatile("csrr %0, mstatus" : "=r" (x)); return x;}
|
CSR_REG_MACRO(Sstatus, sstatus)
|
||||||
static B_ALWAYS_INLINE void SetMstatus(uint64 x) {
|
|
||||||
asm volatile("csrw mstatus, %0" : : "r" (x));}
|
|
||||||
static B_ALWAYS_INLINE uint64 Sstatus() {
|
|
||||||
uint64 x; asm volatile("csrr %0, sstatus" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetSstatus(uint64 x) {
|
|
||||||
asm volatile("csrw sstatus, %0" : : "r" (x));}
|
|
||||||
|
|
||||||
// exception program counter
|
// exception program counter
|
||||||
static B_ALWAYS_INLINE uint64 Mepc() {
|
CSR_REG_MACRO(Mepc, mepc)
|
||||||
uint64 x; asm volatile("csrr %0, mepc" : "=r" (x)); return x;}
|
CSR_REG_MACRO(Sepc, sepc)
|
||||||
static B_ALWAYS_INLINE void SetMepc(uint64 x) {
|
|
||||||
asm volatile("csrw mepc, %0" : : "r" (x));}
|
|
||||||
static B_ALWAYS_INLINE uint64 Sepc() {
|
|
||||||
uint64 x; asm volatile("csrr %0, sepc" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetSepc(uint64 x) {
|
|
||||||
asm volatile("csrw sepc, %0" : : "r" (x));}
|
|
||||||
|
|
||||||
// interrupt pending
|
// interrupt pending
|
||||||
static B_ALWAYS_INLINE uint64 Mip() {
|
CSR_REG_MACRO(Mip, mip)
|
||||||
uint64 x; asm volatile("csrr %0, mip" : "=r" (x)); return x;}
|
CSR_REG_MACRO(Sip, sip)
|
||||||
static B_ALWAYS_INLINE void SetMip(uint64 x) {
|
|
||||||
asm volatile("csrw mip, %0" : : "r" (x));}
|
|
||||||
static B_ALWAYS_INLINE uint64 Sip() {
|
|
||||||
uint64 x; asm volatile("csrr %0, sip" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetSip(uint64 x) {
|
|
||||||
asm volatile("csrw sip, %0" : : "r" (x));}
|
|
||||||
|
|
||||||
// interrupt enable
|
// interrupt enable
|
||||||
static B_ALWAYS_INLINE uint64 Sie() {
|
CSR_REG_MACRO(Mie, mie)
|
||||||
uint64 x; asm volatile("csrr %0, sie" : "=r" (x)); return x;}
|
CSR_REG_MACRO(Sie, sie)
|
||||||
static B_ALWAYS_INLINE void SetSie(uint64 x) {
|
|
||||||
asm volatile("csrw sie, %0" : : "r" (x));}
|
|
||||||
static B_ALWAYS_INLINE uint64 Mie() {
|
|
||||||
uint64 x; asm volatile("csrr %0, mie" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetMie(uint64 x) {
|
|
||||||
asm volatile("csrw mie, %0" : : "r" (x));}
|
|
||||||
|
|
||||||
// exception delegation
|
// exception delegation
|
||||||
static B_ALWAYS_INLINE uint64 Medeleg() {
|
CSR_REG_MACRO(Medeleg, medeleg)
|
||||||
uint64 x; asm volatile("csrr %0, medeleg" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetMedeleg(uint64 x) {
|
|
||||||
asm volatile("csrw medeleg, %0" : : "r" (x));}
|
|
||||||
// interrupt delegation
|
// interrupt delegation
|
||||||
static B_ALWAYS_INLINE uint64 Mideleg() {
|
CSR_REG_MACRO(Mideleg, mideleg)
|
||||||
uint64 x; asm volatile("csrr %0, mideleg" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetMideleg(uint64 x) {
|
|
||||||
asm volatile("csrw mideleg, %0" : : "r" (x));}
|
|
||||||
|
|
||||||
// trap vector, 2 low bits: mode
|
// trap vector, 2 low bits: mode
|
||||||
static B_ALWAYS_INLINE uint64 Mtvec() {
|
CSR_REG_MACRO(Mtvec, mtvec)
|
||||||
uint64 x; asm volatile("csrr %0, mtvec" : "=r" (x)); return x;}
|
CSR_REG_MACRO(Stvec, stvec)
|
||||||
static B_ALWAYS_INLINE void SetMtvec(uint64 x) {
|
|
||||||
asm volatile("csrw mtvec, %0" : : "r" (x));}
|
|
||||||
static B_ALWAYS_INLINE uint64 Stvec() {
|
|
||||||
uint64 x; asm volatile("csrr %0, stvec" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetStvec(uint64 x) {
|
|
||||||
asm volatile("csrw stvec, %0" : : "r" (x));}
|
|
||||||
|
|
||||||
// address translation and protection (pointer to page table and flags)
|
// address translation and protection (pointer to page table and flags)
|
||||||
static B_ALWAYS_INLINE uint64 Satp() {
|
CSR_REG_MACRO(Satp, satp)
|
||||||
uint64 x; asm volatile("csrr %0, satp" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetSatp(uint64 x) {
|
|
||||||
asm volatile("csrw satp, %0" : : "r" (x) : "memory");}
|
|
||||||
|
|
||||||
// scratch register
|
// scratch register
|
||||||
static B_ALWAYS_INLINE uint64 Mscratch() {
|
CSR_REG_MACRO(Mscratch, mscratch)
|
||||||
uint64 x; asm volatile("csrr %0, mscratch" : "=r" (x)); return x;}
|
CSR_REG_MACRO(Sscratch, sscratch)
|
||||||
static B_ALWAYS_INLINE void SetMscratch(uint64 x) {
|
|
||||||
asm volatile("csrw mscratch, %0" : : "r" (x));}
|
|
||||||
static B_ALWAYS_INLINE uint64 Sscratch() {
|
|
||||||
uint64 x; asm volatile("csrr %0, sscratch" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetSscratch(uint64 x) {
|
|
||||||
asm volatile("csrw sscratch, %0" : : "r" (x));}
|
|
||||||
|
|
||||||
// trap cause
|
// trap cause
|
||||||
static B_ALWAYS_INLINE uint64 Mcause() {
|
CSR_REG_MACRO(Mcause, mcause)
|
||||||
uint64 x; asm volatile("csrr %0, mcause" : "=r" (x)); return x;}
|
CSR_REG_MACRO(Scause, scause)
|
||||||
static B_ALWAYS_INLINE void SetMcause(uint64 x) {
|
|
||||||
asm volatile("csrw mcause, %0" : : "r" (x));}
|
|
||||||
static B_ALWAYS_INLINE uint64 Scause() {
|
|
||||||
uint64 x; asm volatile("csrr %0, scause" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetScause(uint64 x) {
|
|
||||||
asm volatile("csrw scause, %0" : : "r" (x));}
|
|
||||||
|
|
||||||
// trap value
|
// trap value
|
||||||
static B_ALWAYS_INLINE uint64 Mtval() {
|
CSR_REG_MACRO(Mtval, mtval)
|
||||||
uint64 x; asm volatile("csrr %0, mtval" : "=r" (x)); return x;}
|
CSR_REG_MACRO(Stval, stval)
|
||||||
static B_ALWAYS_INLINE void SetMtval(uint64 x) {
|
|
||||||
asm volatile("csrw mtval, %0" : : "r" (x));}
|
|
||||||
static B_ALWAYS_INLINE uint64 Stval() {
|
|
||||||
uint64 x; asm volatile("csrr %0, stval" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetStval(uint64 x) {
|
|
||||||
asm volatile("csrw stval, %0" : : "r" (x));}
|
|
||||||
|
|
||||||
// machine-mode counter enable
|
// machine-mode counter enable
|
||||||
static B_ALWAYS_INLINE uint64 Mcounteren() {
|
CSR_REG_MACRO(Mcounteren, mcounteren)
|
||||||
uint64 x; asm volatile("csrr %0, mcounteren" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetMcounteren(uint64 x) {
|
|
||||||
asm volatile("csrw mcounteren, %0" : : "r" (x));}
|
|
||||||
|
|
||||||
// cycle counter
|
// cycle counter
|
||||||
static B_ALWAYS_INLINE uint64 CpuMcycle() {
|
CSR_REG_MACRO(CpuMcycle, mcycle)
|
||||||
uint64 x; asm volatile("csrr %0, mcycle" : "=r" (x)); return x;}
|
CSR_REG_MACRO(CpuCycle, cycle)
|
||||||
static B_ALWAYS_INLINE uint64 CpuCycle() {
|
|
||||||
uint64 x; asm volatile("csrr %0, cycle" : "=r" (x)); return x;}
|
|
||||||
// monotonic timer
|
// monotonic timer
|
||||||
static B_ALWAYS_INLINE uint64 CpuTime() {
|
CSR_REG_MACRO(CpuTime, time)
|
||||||
uint64 x; asm volatile("csrr %0, time" : "=r" (x)); return x;}
|
|
||||||
|
|
||||||
// physical memory protection
|
// physical memory protection
|
||||||
static B_ALWAYS_INLINE uint64 Pmpaddr0() {
|
CSR_REG_MACRO(Pmpaddr0, pmpaddr0)
|
||||||
uint64 x; asm volatile("csrr %0, pmpaddr0" : "=r" (x)); return x;}
|
CSR_REG_MACRO(Pmpcfg0, pmpcfg0)
|
||||||
static B_ALWAYS_INLINE uint64 Pmpcfg0() {
|
|
||||||
uint64 x; asm volatile("csrr %0, pmpcfg0" : "=r" (x)); return x;}
|
|
||||||
static B_ALWAYS_INLINE void SetPmpaddr0(uint64 x) {
|
|
||||||
asm volatile("csrw pmpaddr0, %0" : : "r" (x));}
|
|
||||||
static B_ALWAYS_INLINE void SetPmpcfg0(uint64 x) {
|
|
||||||
asm volatile("csrw pmpcfg0, %0" : : "r" (x));}
|
|
||||||
|
|
||||||
// flush the TLB
|
// flush the TLB
|
||||||
static B_ALWAYS_INLINE void FlushTlbAll() {
|
static B_ALWAYS_INLINE void FlushTlbAll() {
|
||||||
|
|||||||
@@ -111,7 +111,7 @@ DumpPageTableInt(Pte* pte, uint64_t virtAdr, uint32_t level, uint64& firstVirt,
|
|||||||
if (level == 0)
|
if (level == 0)
|
||||||
panic("internal page table on level 0");
|
panic("internal page table on level 0");
|
||||||
|
|
||||||
DumpPageTableInt((Pte*)VirtFromPhys(pageSize*pte[i].ppn),
|
DumpPageTableInt((Pte*)VirtFromPhys(B_PAGE_SIZE*pte[i].ppn),
|
||||||
virtAdr + ((uint64_t)i << (pageBits + pteIdxBits*level)),
|
virtAdr + ((uint64_t)i << (pageBits + pteIdxBits*level)),
|
||||||
level - 1, firstVirt, firstPhys, firstFlags, len);
|
level - 1, firstVirt, firstPhys, firstFlags, len);
|
||||||
} else {
|
} else {
|
||||||
@@ -130,7 +130,7 @@ DumpPageTableInt(Pte* pte, uint64_t virtAdr, uint32_t level, uint64& firstVirt,
|
|||||||
static int
|
static int
|
||||||
DumpPageTable(uint64 satp)
|
DumpPageTable(uint64 satp)
|
||||||
{
|
{
|
||||||
SatpReg satpReg(satp);
|
SatpReg satpReg{.val = satp};
|
||||||
Pte* root = (Pte*)VirtFromPhys(satpReg.ppn * B_PAGE_SIZE);
|
Pte* root = (Pte*)VirtFromPhys(satpReg.ppn * B_PAGE_SIZE);
|
||||||
|
|
||||||
dprintf("PageTable:\n");
|
dprintf("PageTable:\n");
|
||||||
@@ -237,14 +237,14 @@ PreallocKernelRange()
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
uint64
|
static uint64
|
||||||
GetSatp()
|
GetSatp()
|
||||||
{
|
{
|
||||||
SatpReg satp;
|
return SatpReg{
|
||||||
satp.ppn = sPageTable / B_PAGE_SIZE;
|
.ppn = sPageTable / B_PAGE_SIZE,
|
||||||
satp.asid = 0;
|
.asid = 0,
|
||||||
satp.mode = satpModeSv39;
|
.mode = satpModeSv39
|
||||||
return satp.val;
|
}.val;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ WriteExt(uint64_t val)
|
|||||||
void
|
void
|
||||||
WriteSstatus(uint64_t val)
|
WriteSstatus(uint64_t val)
|
||||||
{
|
{
|
||||||
SstatusReg status(val);
|
SstatusReg status{.val = val};
|
||||||
dprintf("%#" B_PRIx64, val);
|
dprintf("%#" B_PRIx64, val);
|
||||||
dprintf(" (");
|
dprintf(" (");
|
||||||
dprintf("ie: "); WriteModeSet(status.ie);
|
dprintf("ie: "); WriteModeSet(status.ie);
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ cpu_init()
|
|||||||
{
|
{
|
||||||
gKernelArgs.num_cpus = 1;
|
gKernelArgs.num_cpus = 1;
|
||||||
|
|
||||||
SstatusReg status(Sstatus());
|
SstatusReg status{.val = Sstatus()};
|
||||||
status.fs = extStatusInitial; // enable FPU
|
status.fs = extStatusInitial; // enable FPU
|
||||||
status.xs = extStatusOff;
|
status.xs = extStatusOff;
|
||||||
SetSstatus(status.val);
|
SetSstatus(status.val);
|
||||||
|
|||||||
@@ -270,11 +270,11 @@ SetupPageTable()
|
|||||||
static uint64
|
static uint64
|
||||||
GetSatp()
|
GetSatp()
|
||||||
{
|
{
|
||||||
SatpReg satp;
|
return SatpReg{
|
||||||
satp.ppn = sPageTable / B_PAGE_SIZE;
|
.ppn = sPageTable / B_PAGE_SIZE,
|
||||||
satp.asid = 0;
|
.asid = 0,
|
||||||
satp.mode = satpModeSv39;
|
.mode = satpModeSv39
|
||||||
return satp.val;
|
}.val;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -104,7 +104,7 @@ MTrap(iframe* frame)
|
|||||||
frame->a0 = B_NOT_ALLOWED;
|
frame->a0 = B_NOT_ALLOWED;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
MstatusReg status(Mstatus());
|
MstatusReg status{.val = Mstatus()};
|
||||||
status.mpp = modeS;
|
status.mpp = modeS;
|
||||||
SetMedeleg(
|
SetMedeleg(
|
||||||
0xffff & ~((1 << causeMEcall) | (1 << causeSEcall)));
|
0xffff & ~((1 << causeMEcall) | (1 << causeSEcall)));
|
||||||
@@ -125,12 +125,12 @@ MTrap(iframe* frame)
|
|||||||
enable, frame->a2);
|
enable, frame->a2);
|
||||||
*/
|
*/
|
||||||
// dprintf(" mtime: %" B_PRIu64 "\n", gClintRegs->mTime);
|
// dprintf(" mtime: %" B_PRIu64 "\n", gClintRegs->mTime);
|
||||||
SetMip(Mip() & ~(1 << sTimerInt));
|
ClearBitsMip(1 << sTimerInt);
|
||||||
if (!enable) {
|
if (!enable) {
|
||||||
SetMie(Mie() & ~(1 << mTimerInt));
|
ClearBitsMie(1 << mTimerInt);
|
||||||
} else {
|
} else {
|
||||||
gClintRegs->mtimecmp[0] = frame->a2;
|
gClintRegs->mtimecmp[0] = frame->a2;
|
||||||
SetMie(Mie() | (1 << mTimerInt));
|
SetBitsMie(1 << mTimerInt);
|
||||||
}
|
}
|
||||||
frame->a0 = B_OK;
|
frame->a0 = B_OK;
|
||||||
return;
|
return;
|
||||||
@@ -142,8 +142,8 @@ MTrap(iframe* frame)
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case causeInterrupt + mTimerInt: {
|
case causeInterrupt + mTimerInt: {
|
||||||
SetMie(Mie() & ~(1 << mTimerInt));
|
ClearBitsMie(1 << mTimerInt);
|
||||||
SetMip(Mip() | (1 << sTimerInt));
|
SetBitsMip(1 << sTimerInt);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -156,7 +156,7 @@ void
|
|||||||
traps_init()
|
traps_init()
|
||||||
{
|
{
|
||||||
SetMtvec((uint64)MVec);
|
SetMtvec((uint64)MVec);
|
||||||
MstatusReg mstatus(Mstatus());
|
MstatusReg mstatus{.val = Mstatus()};
|
||||||
mstatus.ie = 1 << modeM;
|
mstatus.ie = 1 << modeM;
|
||||||
SetMstatus(mstatus.val);
|
SetMstatus(mstatus.val);
|
||||||
InitPmp();
|
InitPmp();
|
||||||
|
|||||||
@@ -363,7 +363,7 @@ RISCV64VMTranslationMap::Unmap(addr_t start, addr_t end)
|
|||||||
Pte* pte = LookupPte(page, false, NULL);
|
Pte* pte = LookupPte(page, false, NULL);
|
||||||
if (pte != NULL) {
|
if (pte != NULL) {
|
||||||
fMapCount--;
|
fMapCount--;
|
||||||
Pte oldPte((uint64)atomic_get_and_set64((int64*)&pte->val, 0));
|
Pte oldPte{.val = (uint64)atomic_get_and_set64((int64*)&pte->val, 0)};
|
||||||
if ((oldPte.flags & (1 << pteAccessed)) != 0)
|
if ((oldPte.flags & (1 << pteAccessed)) != 0)
|
||||||
InvalidatePage(page);
|
InvalidatePage(page);
|
||||||
}
|
}
|
||||||
@@ -410,7 +410,7 @@ RISCV64VMTranslationMap::UnmapPage(VMArea* area, addr_t address,
|
|||||||
|
|
||||||
RecursiveLocker locker(fLock);
|
RecursiveLocker locker(fLock);
|
||||||
|
|
||||||
Pte oldPte((uint64)atomic_get_and_set64((int64*)&pte->val, 0));
|
Pte oldPte{.val = (uint64)atomic_get_and_set64((int64*)&pte->val, 0)};
|
||||||
fMapCount--;
|
fMapCount--;
|
||||||
pinner.Unlock();
|
pinner.Unlock();
|
||||||
|
|
||||||
@@ -448,7 +448,7 @@ RISCV64VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size,
|
|||||||
if (pte == NULL)
|
if (pte == NULL)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
Pte oldPte((uint64)atomic_get_and_set64((int64*)&pte->val, 0));
|
Pte oldPte{.val = (uint64)atomic_get_and_set64((int64*)&pte->val, 0)};
|
||||||
if ((oldPte.flags & (1 << pteValid)) == 0)
|
if ((oldPte.flags & (1 << pteValid)) == 0)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
@@ -579,7 +579,7 @@ RISCV64VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace,
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
Pte oldPte((uint64)atomic_get_and_set64((int64*)&pte->val, 0));
|
Pte oldPte{.val = (uint64)atomic_get_and_set64((int64*)&pte->val, 0)};
|
||||||
|
|
||||||
// transfer the accessed/dirty flags to the page and
|
// transfer the accessed/dirty flags to the page and
|
||||||
// invalidate the mapping, if necessary
|
// invalidate the mapping, if necessary
|
||||||
|
|||||||
@@ -32,12 +32,12 @@ status_t
|
|||||||
arch_cpu_init_percpu(kernel_args *args, int curr_cpu)
|
arch_cpu_init_percpu(kernel_args *args, int curr_cpu)
|
||||||
{
|
{
|
||||||
SetStvec((uint64)SVec);
|
SetStvec((uint64)SVec);
|
||||||
SstatusReg sstatus(Sstatus());
|
SstatusReg sstatus{.val = Sstatus()};
|
||||||
sstatus.ie = 0;
|
sstatus.ie = 0;
|
||||||
sstatus.fs = extStatusInitial; // enable FPU
|
sstatus.fs = extStatusInitial; // enable FPU
|
||||||
sstatus.xs = extStatusOff;
|
sstatus.xs = extStatusOff;
|
||||||
SetSstatus(sstatus.val);
|
SetSstatus(sstatus.val);
|
||||||
SetSie(Sie() | (1 << sTimerInt) | (1 << sSoftInt) | (1 << sExternInt));
|
SetBitsSie((1 << sTimerInt) | (1 << sSoftInt) | (1 << sExternInt));
|
||||||
|
|
||||||
return B_OK;
|
return B_OK;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ WriteExt(uint64_t val)
|
|||||||
static void
|
static void
|
||||||
WriteSstatus(uint64_t val)
|
WriteSstatus(uint64_t val)
|
||||||
{
|
{
|
||||||
SstatusReg status(val);
|
SstatusReg status{.val = val};
|
||||||
dprintf("(");
|
dprintf("(");
|
||||||
dprintf("ie: "); WriteModeSet(status.ie);
|
dprintf("ie: "); WriteModeSet(status.ie);
|
||||||
dprintf(", pie: "); WriteModeSet(status.pie);
|
dprintf(", pie: "); WriteModeSet(status.pie);
|
||||||
@@ -253,7 +253,7 @@ static void
|
|||||||
SendSignal(debug_exception_type type, uint32 signalNumber, int32 signalCode,
|
SendSignal(debug_exception_type type, uint32 signalNumber, int32 signalCode,
|
||||||
addr_t signalAddress = 0, int32 signalError = B_ERROR)
|
addr_t signalAddress = 0, int32 signalError = B_ERROR)
|
||||||
{
|
{
|
||||||
if (SstatusReg(Sstatus()).spp == modeU) {
|
if (SstatusReg{.val = Sstatus()}.spp == modeU) {
|
||||||
struct sigaction action;
|
struct sigaction action;
|
||||||
Thread* thread = thread_get_current_thread();
|
Thread* thread = thread_get_current_thread();
|
||||||
|
|
||||||
@@ -390,13 +390,13 @@ STrap(iframe* frame)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (SstatusReg(frame->status).spp == modeU) {
|
if (SstatusReg{.val = frame->status}.spp == modeU) {
|
||||||
thread_get_current_thread()->arch_info.userFrame = frame;
|
thread_get_current_thread()->arch_info.userFrame = frame;
|
||||||
thread_get_current_thread()->arch_info.oldA0 = frame->a0;
|
thread_get_current_thread()->arch_info.oldA0 = frame->a0;
|
||||||
thread_at_kernel_entry(system_time());
|
thread_at_kernel_entry(system_time());
|
||||||
}
|
}
|
||||||
const auto& kernelExit = ScopeExit([&]() {
|
const auto& kernelExit = ScopeExit([&]() {
|
||||||
if (SstatusReg(frame->status).spp == modeU) {
|
if (SstatusReg{.val = frame->status}.spp == modeU) {
|
||||||
disable_interrupts();
|
disable_interrupts();
|
||||||
atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED);
|
atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED);
|
||||||
if ((thread_get_current_thread()->flags
|
if ((thread_get_current_thread()->flags
|
||||||
@@ -431,7 +431,7 @@ STrap(iframe* frame)
|
|||||||
Stval());
|
Stval());
|
||||||
}
|
}
|
||||||
case causeBreakpoint: {
|
case causeBreakpoint: {
|
||||||
if (SstatusReg(frame->status).spp == modeU) {
|
if (SstatusReg{.val = frame->status}.spp == modeU) {
|
||||||
user_debug_breakpoint_hit(false);
|
user_debug_breakpoint_hit(false);
|
||||||
} else {
|
} else {
|
||||||
panic("hit kernel breakpoint");
|
panic("hit kernel breakpoint");
|
||||||
@@ -478,7 +478,7 @@ STrap(iframe* frame)
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (SstatusReg(frame->status).pie == 0) {
|
if (SstatusReg{.val = frame->status}.pie == 0) {
|
||||||
// user_memcpy() failure
|
// user_memcpy() failure
|
||||||
Thread* thread = thread_get_current_thread();
|
Thread* thread = thread_get_current_thread();
|
||||||
if (thread != NULL && thread->fault_handler != 0) {
|
if (thread != NULL && thread->fault_handler != 0) {
|
||||||
@@ -496,7 +496,7 @@ STrap(iframe* frame)
|
|||||||
|
|
||||||
vm_page_fault(stval, frame->epc, frame->cause == causeStorePageFault,
|
vm_page_fault(stval, frame->epc, frame->cause == causeStorePageFault,
|
||||||
frame->cause == causeExecPageFault,
|
frame->cause == causeExecPageFault,
|
||||||
SstatusReg(frame->status).spp == modeU, &newIP);
|
SstatusReg{.val = frame->status}.spp == modeU, &newIP);
|
||||||
|
|
||||||
if (newIP != 0)
|
if (newIP != 0)
|
||||||
frame->epc = newIP;
|
frame->epc = newIP;
|
||||||
|
|||||||
@@ -148,7 +148,7 @@ arch_thread_enter_userspace(Thread *thread, addr_t entry, void *arg1,
|
|||||||
iframe frame;
|
iframe frame;
|
||||||
memset(&frame, 0, sizeof(frame));
|
memset(&frame, 0, sizeof(frame));
|
||||||
|
|
||||||
SstatusReg status(Sstatus());
|
SstatusReg status{.val = Sstatus()};
|
||||||
status.pie = (1 << modeS); // enable interrupts when enter userspace
|
status.pie = (1 << modeS); // enable interrupts when enter userspace
|
||||||
status.spp = modeU;
|
status.spp = modeU;
|
||||||
|
|
||||||
@@ -381,7 +381,7 @@ arch_restore_fork_frame(struct arch_fork_arg *arg)
|
|||||||
|
|
||||||
arch_stack* stackHeader = (arch_stack*)thread_get_current_thread()->kernel_stack_top - 1;
|
arch_stack* stackHeader = (arch_stack*)thread_get_current_thread()->kernel_stack_top - 1;
|
||||||
stackHeader->thread = thread_get_current_thread();
|
stackHeader->thread = thread_get_current_thread();
|
||||||
SstatusReg status(Sstatus());
|
SstatusReg status{.val = Sstatus()};
|
||||||
status.pie = (1 << modeS); // enable interrupts when enter userspace
|
status.pie = (1 << modeS); // enable interrupts when enter userspace
|
||||||
status.spp = modeU;
|
status.spp = modeU;
|
||||||
arg->frame.status = status.val;
|
arg->frame.status = status.val;
|
||||||
|
|||||||
@@ -110,7 +110,7 @@ public:
|
|||||||
firstFlags(0),
|
firstFlags(0),
|
||||||
len(0)
|
len(0)
|
||||||
{}
|
{}
|
||||||
|
|
||||||
~PageTableDumper()
|
~PageTableDumper()
|
||||||
{
|
{
|
||||||
Write(0, 0, 0, 0);
|
Write(0, 0, 0, 0);
|
||||||
@@ -147,7 +147,7 @@ DumpPageTableInt(Pte* pte, uint64_t virtAdr, uint32_t level, PageTableDumper& du
|
|||||||
if (level == 0)
|
if (level == 0)
|
||||||
kprintf(" internal page table on level 0\n");
|
kprintf(" internal page table on level 0\n");
|
||||||
|
|
||||||
DumpPageTableInt((Pte*)VirtFromPhys(pageSize*pte[i].ppn),
|
DumpPageTableInt((Pte*)VirtFromPhys(B_PAGE_SIZE*pte[i].ppn),
|
||||||
virtAdr + ((uint64_t)i << (pageBits + pteIdxBits * level)),
|
virtAdr + ((uint64_t)i << (pageBits + pteIdxBits * level)),
|
||||||
level - 1, dumper);
|
level - 1, dumper);
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -116,12 +116,9 @@ arch_vm_translation_map_init(kernel_args *args,
|
|||||||
TRACE(" %" B_PRIxPHYSADDR " - %" B_PRIxPHYSADDR "\n", start, end);
|
TRACE(" %" B_PRIxPHYSADDR " - %" B_PRIxPHYSADDR "\n", start, end);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
{
|
sPageTable = SatpReg{.val = Satp()}.ppn * B_PAGE_SIZE;
|
||||||
SatpReg satp(Satp());
|
|
||||||
sPageTable = satp.ppn * B_PAGE_SIZE;
|
|
||||||
}
|
|
||||||
|
|
||||||
dprintf("physMapBase: %#" B_PRIxADDR "\n", args->arch_args.physMap.start);
|
dprintf("physMapBase: %#" B_PRIxADDR "\n", args->arch_args.physMap.start);
|
||||||
dprintf("physMemBase: %#" B_PRIxADDR "\n", args->physical_memory_range[0].start);
|
dprintf("physMemBase: %#" B_PRIxADDR "\n", args->physical_memory_range[0].start);
|
||||||
gVirtFromPhysOffset = args->arch_args.physMap.start - args->physical_memory_range[0].start;
|
gVirtFromPhysOffset = args->arch_args.physMap.start - args->physical_memory_range[0].start;
|
||||||
|
|||||||
Reference in New Issue
Block a user