From 91e7f347f9320c9f186f0b3af876778b9c56fb5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dziepak?= Date: Thu, 29 May 2014 23:28:01 +0200 Subject: [PATCH] kernel/x86_64: try to inline physical page mapper functions The way we handle paging is very wasteful and relies heavily on virtual funcions even if there is absolutely no reason to do so. The proper solution would be to do a major rework of paging code (including arch-independent parts). On x86_64 physical page mapper is very simple what makes the overhead resulting from the desing of paging interface very expensive. This patch attempts to make things a bit better by helping GCC with devirtualization and allowing inlining physical page mapper impementation (well, only when it is devirtualized). --- src/system/kernel/arch/x86/Jamfile | 1 - .../x86/paging/x86_physical_page_mapper.cpp | 16 -- .../x86/paging/x86_physical_page_mapper.h | 54 +++- .../x86_physical_page_mapper_mapped.cpp | 231 +----------------- .../paging/x86_physical_page_mapper_mapped.h | 182 +++++++++++++- 5 files changed, 234 insertions(+), 250 deletions(-) delete mode 100644 src/system/kernel/arch/x86/paging/x86_physical_page_mapper.cpp diff --git a/src/system/kernel/arch/x86/Jamfile b/src/system/kernel/arch/x86/Jamfile index 3941e18b5c..773bc39153 100644 --- a/src/system/kernel/arch/x86/Jamfile +++ b/src/system/kernel/arch/x86/Jamfile @@ -94,7 +94,6 @@ local archGenericSources = pic.cpp # paging - x86_physical_page_mapper.cpp X86PagingMethod.cpp X86PagingStructures.cpp X86VMTranslationMap.cpp diff --git a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper.cpp b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper.cpp deleted file mode 100644 index 1098b46cf3..0000000000 --- a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper.cpp +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de. - * Distributed under the terms of the MIT License. - */ - -#include "paging/x86_physical_page_mapper.h" - - -TranslationMapPhysicalPageMapper::~TranslationMapPhysicalPageMapper() -{ -} - - -X86PhysicalPageMapper::~X86PhysicalPageMapper() -{ -} diff --git a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper.h b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper.h index a29459b9f0..b9c7cb4eaf 100644 --- a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper.h +++ b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper.h @@ -12,9 +12,12 @@ struct kernel_args; +#ifndef __x86_64__ + + class TranslationMapPhysicalPageMapper { public: - virtual ~TranslationMapPhysicalPageMapper(); + virtual ~TranslationMapPhysicalPageMapper() { } virtual void Delete() = 0; @@ -25,8 +28,6 @@ public: class X86PhysicalPageMapper : public VMPhysicalPageMapper { public: - virtual ~X86PhysicalPageMapper(); - virtual status_t CreateTranslationMapPhysicalPageMapper( TranslationMapPhysicalPageMapper** _mapper) = 0; @@ -36,4 +37,51 @@ public: }; +#else + + +class TranslationMapPhysicalPageMapper { +public: + void Delete(); + + void* GetPageTableAt(phys_addr_t physicalAddress); + // Must be invoked with thread pinned to current CPU. +}; + + +class X86PhysicalPageMapper final : public VMPhysicalPageMapper { +public: + status_t CreateTranslationMapPhysicalPageMapper( + TranslationMapPhysicalPageMapper** _mapper); + + void* InterruptGetPageTableAt(phys_addr_t physicalAddress); + + status_t GetPage(phys_addr_t physicalAddress, addr_t* virtualAddress, + void** handle) override; + status_t PutPage(addr_t virtualAddress, void* handle) override; + + status_t GetPageCurrentCPU(phys_addr_t physicalAddress, + addr_t* virtualAddress, void** handle) override; + status_t PutPageCurrentCPU(addr_t virtualAddress, void* handle) override; + + status_t GetPageDebug(phys_addr_t physicalAddress, + addr_t* virtualAddress, void** handle) override; + status_t PutPageDebug(addr_t virtualAddress, void* handle) override; + + status_t MemsetPhysical(phys_addr_t address, int value, + phys_size_t length) override; + status_t MemcpyFromPhysical(void* to, phys_addr_t from, size_t length, + bool user) override; + status_t MemcpyToPhysical(phys_addr_t to, const void* from, + size_t length, bool user) override; + void MemcpyPhysicalPage(phys_addr_t to, phys_addr_t from) override; +}; + + +#include "paging/x86_physical_page_mapper_mapped.h" + + +#endif // __x86_64__ + + #endif // KERNEL_ARCH_X86_PAGING_X86_PHYSICAL_PAGE_MAPPER_H diff --git a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_mapped.cpp b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_mapped.cpp index babdbb87c6..92df73a360 100644 --- a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_mapped.cpp +++ b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_mapped.cpp @@ -4,239 +4,16 @@ */ -/*! Physical page mapper implementation for use where the whole of physical - memory is permanently mapped into the kernel address space. - - This is used on x86_64 where the virtual address space is likely a great - deal larger than the amount of physical memory in the machine, so it can - all be mapped in permanently, which is faster and makes life much easier. -*/ - - -#include "paging/x86_physical_page_mapper_mapped.h" - #include -#include -#include -#include -#include -#include -#include - #include "paging/x86_physical_page_mapper.h" -#include "paging/X86PagingStructures.h" -#include "paging/X86VMTranslationMap.h" // #pragma mark - -class MappedTranslationMapPhysicalPageMapper - : public TranslationMapPhysicalPageMapper { -public: - virtual void Delete(); - - virtual void* GetPageTableAt(phys_addr_t physicalAddress); -}; - - -class MappedPhysicalPageMapper : public X86PhysicalPageMapper { -public: - virtual status_t CreateTranslationMapPhysicalPageMapper( - TranslationMapPhysicalPageMapper** _mapper); - - virtual void* InterruptGetPageTableAt( - phys_addr_t physicalAddress); - - virtual status_t GetPage(phys_addr_t physicalAddress, - addr_t* virtualAddress, void** handle); - virtual status_t PutPage(addr_t virtualAddress, void* handle); - - virtual status_t GetPageCurrentCPU(phys_addr_t physicalAddress, - addr_t* virtualAddress, void** handle); - virtual status_t PutPageCurrentCPU(addr_t virtualAddress, - void* handle); - - virtual status_t GetPageDebug(phys_addr_t physicalAddress, - addr_t* virtualAddress, void** handle); - virtual status_t PutPageDebug(addr_t virtualAddress, - void* handle); - - virtual status_t MemsetPhysical(phys_addr_t address, int value, - phys_size_t length); - virtual status_t MemcpyFromPhysical(void* to, phys_addr_t from, - size_t length, bool user); - virtual status_t MemcpyToPhysical(phys_addr_t to, - const void* from, size_t length, bool user); - virtual void MemcpyPhysicalPage(phys_addr_t to, - phys_addr_t from); -}; - - -static MappedPhysicalPageMapper sPhysicalPageMapper; -static MappedTranslationMapPhysicalPageMapper sKernelPageMapper; - - -// #pragma mark - MappedTranslationMapPhysicalPageMapper - - -void -MappedTranslationMapPhysicalPageMapper::Delete() -{ - delete this; -} - - -void* -MappedTranslationMapPhysicalPageMapper::GetPageTableAt( - phys_addr_t physicalAddress) -{ - ASSERT(physicalAddress % B_PAGE_SIZE == 0); - - return (void*)(physicalAddress + KERNEL_PMAP_BASE); -} - - -// #pragma mark - MappedPhysicalPageMapper - - -status_t -MappedPhysicalPageMapper::CreateTranslationMapPhysicalPageMapper( - TranslationMapPhysicalPageMapper** _mapper) -{ - MappedTranslationMapPhysicalPageMapper* mapper - = new(std::nothrow) MappedTranslationMapPhysicalPageMapper; - if (mapper == NULL) - return B_NO_MEMORY; - - *_mapper = mapper; - return B_OK; -} - - -void* -MappedPhysicalPageMapper::InterruptGetPageTableAt( - phys_addr_t physicalAddress) -{ - ASSERT(physicalAddress % B_PAGE_SIZE == 0); - - return (void*)(physicalAddress + KERNEL_PMAP_BASE); -} - - -status_t -MappedPhysicalPageMapper::GetPage(phys_addr_t physicalAddress, - addr_t* virtualAddress, void** handle) -{ - if (physicalAddress >= KERNEL_PMAP_BASE) - return B_BAD_ADDRESS; - - *virtualAddress = physicalAddress + KERNEL_PMAP_BASE; - return B_OK; -} - - -status_t -MappedPhysicalPageMapper::PutPage(addr_t virtualAddress, void* handle) -{ - return B_OK; -} - - -status_t -MappedPhysicalPageMapper::GetPageCurrentCPU(phys_addr_t physicalAddress, - addr_t* virtualAddress, void** handle) -{ - if (physicalAddress >= KERNEL_PMAP_BASE) - return B_BAD_ADDRESS; - - *virtualAddress = physicalAddress + KERNEL_PMAP_BASE; - return B_OK; -} - - -status_t -MappedPhysicalPageMapper::PutPageCurrentCPU(addr_t virtualAddress, - void* handle) -{ - return B_OK; -} - - -status_t -MappedPhysicalPageMapper::GetPageDebug(phys_addr_t physicalAddress, - addr_t* virtualAddress, void** handle) -{ - if (physicalAddress >= KERNEL_PMAP_BASE) - return B_BAD_ADDRESS; - - *virtualAddress = physicalAddress + KERNEL_PMAP_BASE; - return B_OK; -} - - -status_t -MappedPhysicalPageMapper::PutPageDebug(addr_t virtualAddress, void* handle) -{ - return B_OK; -} - - -status_t -MappedPhysicalPageMapper::MemsetPhysical(phys_addr_t address, int value, - phys_size_t length) -{ - if (address >= KERNEL_PMAP_SIZE || address + length > KERNEL_PMAP_SIZE) - return B_BAD_ADDRESS; - - memset((void*)(address + KERNEL_PMAP_BASE), value, length); - return B_OK; -} - - -status_t -MappedPhysicalPageMapper::MemcpyFromPhysical(void* to, phys_addr_t _from, - size_t length, bool user) -{ - if (_from >= KERNEL_PMAP_SIZE || _from + length > KERNEL_PMAP_SIZE) - return B_BAD_ADDRESS; - - void* from = (void*)(_from + KERNEL_PMAP_BASE); - - if (user) - return user_memcpy(to, from, length); - else - memcpy(to, from, length); - - return B_OK; -} - - -status_t -MappedPhysicalPageMapper::MemcpyToPhysical(phys_addr_t _to, const void* from, - size_t length, bool user) -{ - if (_to >= KERNEL_PMAP_SIZE || _to + length > KERNEL_PMAP_SIZE) - return B_BAD_ADDRESS; - - void* to = (void*)(_to + KERNEL_PMAP_BASE); - - if (user) - return user_memcpy(to, from, length); - - memcpy(to, from, length); - return B_OK; -} - - -void -MappedPhysicalPageMapper::MemcpyPhysicalPage(phys_addr_t to, - phys_addr_t from) -{ - memcpy((void*)(to + KERNEL_PMAP_BASE), (void*)(from + KERNEL_PMAP_BASE), - B_PAGE_SIZE); -} +static X86PhysicalPageMapper sPhysicalPageMapper; +static TranslationMapPhysicalPageMapper sKernelPageMapper; // #pragma mark - Initialization @@ -247,8 +24,8 @@ mapped_physical_page_ops_init(kernel_args* args, X86PhysicalPageMapper*& _pageMapper, TranslationMapPhysicalPageMapper*& _kernelPageMapper) { - new(&sPhysicalPageMapper) MappedPhysicalPageMapper; - new(&sKernelPageMapper) MappedTranslationMapPhysicalPageMapper; + new(&sPhysicalPageMapper) X86PhysicalPageMapper; + new(&sKernelPageMapper) TranslationMapPhysicalPageMapper; _pageMapper = &sPhysicalPageMapper; _kernelPageMapper = &sKernelPageMapper; diff --git a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_mapped.h b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_mapped.h index d0c4390729..6e5339edb6 100644 --- a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_mapped.h +++ b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_mapped.h @@ -8,14 +8,190 @@ #include -#include +#include +#include +#include +#include +#include +#include + +#include "paging/x86_physical_page_mapper.h" +#include "paging/X86PagingStructures.h" +#include "paging/X86VMTranslationMap.h" -class TranslationMapPhysicalPageMapper; -class X86PhysicalPageMapper; struct kernel_args; +/*! Physical page mapper implementation for use where the whole of physical + memory is permanently mapped into the kernel address space. + + This is used on x86_64 where the virtual address space is likely a great + deal larger than the amount of physical memory in the machine, so it can + all be mapped in permanently, which is faster and makes life much easier. +*/ + + +// #pragma mark - TranslationMapPhysicalPageMapper + + +inline void +TranslationMapPhysicalPageMapper::Delete() +{ + delete this; +} + + +inline void* +TranslationMapPhysicalPageMapper::GetPageTableAt( + phys_addr_t physicalAddress) +{ + ASSERT(physicalAddress % B_PAGE_SIZE == 0); + + return (void*)(physicalAddress + KERNEL_PMAP_BASE); +} + + +// #pragma mark - X86PhysicalPageMapper + + +inline status_t +X86PhysicalPageMapper::CreateTranslationMapPhysicalPageMapper( + TranslationMapPhysicalPageMapper** _mapper) +{ + auto mapper = new(std::nothrow) TranslationMapPhysicalPageMapper; + if (mapper == NULL) + return B_NO_MEMORY; + + *_mapper = mapper; + return B_OK; +} + + +inline void* +X86PhysicalPageMapper::InterruptGetPageTableAt( + phys_addr_t physicalAddress) +{ + ASSERT(physicalAddress % B_PAGE_SIZE == 0); + + return (void*)(physicalAddress + KERNEL_PMAP_BASE); +} + + +inline status_t +X86PhysicalPageMapper::GetPage(phys_addr_t physicalAddress, + addr_t* virtualAddress, void** handle) +{ + if (physicalAddress >= KERNEL_PMAP_BASE) + return B_BAD_ADDRESS; + + *virtualAddress = physicalAddress + KERNEL_PMAP_BASE; + return B_OK; +} + + +inline status_t +X86PhysicalPageMapper::PutPage(addr_t virtualAddress, void* handle) +{ + return B_OK; +} + + +inline status_t +X86PhysicalPageMapper::GetPageCurrentCPU(phys_addr_t physicalAddress, + addr_t* virtualAddress, void** handle) +{ + if (physicalAddress >= KERNEL_PMAP_BASE) + return B_BAD_ADDRESS; + + *virtualAddress = physicalAddress + KERNEL_PMAP_BASE; + return B_OK; +} + + +inline status_t +X86PhysicalPageMapper::PutPageCurrentCPU(addr_t virtualAddress, + void* handle) +{ + return B_OK; +} + + +inline status_t +X86PhysicalPageMapper::GetPageDebug(phys_addr_t physicalAddress, + addr_t* virtualAddress, void** handle) +{ + if (physicalAddress >= KERNEL_PMAP_BASE) + return B_BAD_ADDRESS; + + *virtualAddress = physicalAddress + KERNEL_PMAP_BASE; + return B_OK; +} + + +inline status_t +X86PhysicalPageMapper::PutPageDebug(addr_t virtualAddress, void* handle) +{ + return B_OK; +} + + +inline status_t +X86PhysicalPageMapper::MemsetPhysical(phys_addr_t address, int value, + phys_size_t length) +{ + if (address >= KERNEL_PMAP_SIZE || address + length > KERNEL_PMAP_SIZE) + return B_BAD_ADDRESS; + + memset((void*)(address + KERNEL_PMAP_BASE), value, length); + return B_OK; +} + + +inline status_t +X86PhysicalPageMapper::MemcpyFromPhysical(void* to, phys_addr_t _from, + size_t length, bool user) +{ + if (_from >= KERNEL_PMAP_SIZE || _from + length > KERNEL_PMAP_SIZE) + return B_BAD_ADDRESS; + + auto from = (void*)(_from + KERNEL_PMAP_BASE); + + if (user) + return user_memcpy(to, from, length); + else + memcpy(to, from, length); + + return B_OK; +} + + +inline status_t +X86PhysicalPageMapper::MemcpyToPhysical(phys_addr_t _to, const void* from, + size_t length, bool user) +{ + if (_to >= KERNEL_PMAP_SIZE || _to + length > KERNEL_PMAP_SIZE) + return B_BAD_ADDRESS; + + auto to = (void*)(_to + KERNEL_PMAP_BASE); + + if (user) + return user_memcpy(to, from, length); + + memcpy(to, from, length); + return B_OK; +} + + +inline void +X86PhysicalPageMapper::MemcpyPhysicalPage(phys_addr_t to, + phys_addr_t from) +{ + memcpy((void*)(to + KERNEL_PMAP_BASE), (void*)(from + KERNEL_PMAP_BASE), + B_PAGE_SIZE); +} + + status_t mapped_physical_page_ops_init(kernel_args* args, X86PhysicalPageMapper*& _pageMapper, TranslationMapPhysicalPageMapper*& _kernelPageMapper);