i2p/haiku-beta6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kernel/vm: Use more than one object_cache for the page mappings.

Browse Source 
Every time a page is mapped into an area on fault, we have to
allocate a mapping object for it. While the object_cache
does have per-CPU depots, these depots only store a limited
number of items, and once they run out the object_cache's lock
must be acquired.

So, to reduce lock contention on SMP systems, create a number
of object caches corresponding to the nearest power of 2
that is equal or smaller than the count of CPUs. (We already
allocate dozens of object caches for the block allocator
no matter how many CPUs there are, so a few more depending
on CPU count shouldn't impact memory use too much. Besides,
the object_caches are wired into the low_resource system.)

This significantly reduces lock contention on SMP systems.
Same benchmark setup as yesterday (compile mime_db and relink
HaikuDepot, VMware, -j4), before:
real    0m16.981s
user    0m14.357s
sys     0m6.060s

after:
real    0m14.522s
user    0m14.194s
sys     0m4.337s

And the page_mappings object_cache locks went from having 200,000+ waits
and ~14 seconds waiting time (across all threads) down to ~900 (yes,
that's not a typo) and ~0.05s wait time (though these numbers were captured
in conjunction with the following commit.)
This commit is contained in:
Augustin Cavalier
2024-07-24 16:31:20 -04:00
parent f44ebcb614
commit 08c53ca964
11 changed files with 81 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
headers/private/kernel/vm/vm.h
+2 -4
View File
@@ -17,7 +17,6 @@
struct generic_io_vec;
struct kernel_args;
struct ObjectCache;
struct system_memory_info;
struct VMAddressSpace;
struct VMArea;
@@ -48,9 +47,6 @@ struct VMPageWiringInfo;
#define VM_MEMORY_RESERVE_SYSTEM (VM_PAGE_RESERVE_SYSTEM * B_PAGE_SIZE)
extern struct ObjectCache* gPageMappingsObjectCache;
#ifdef __cplusplus
extern "C" {
#endif
@@ -86,6 +82,8 @@ area_id transfer_area(area_id id, void** _address, uint32 addressSpec,
const char* vm_cache_type_to_string(int32 type);
void vm_free_page_mapping(page_num_t page, vm_page_mapping* mapping, uint32 flags);
status_t vm_prepare_kernel_area_debug_protection(area_id id, void** cookie);
status_t vm_set_kernel_area_debug_protection(void* cookie, void* _address,
size_t size, uint32 protection);
src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.cpp
+2 -2
View File
@@ -517,7 +517,7 @@ ARMVMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
@@ -619,7 +619,7 @@ ARMVMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.cpp
+2 -2
View File
@@ -638,7 +638,7 @@ M68KVMTranslationMap040::UnmapPages(VMArea* area, addr_t base, size_t size,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
@@ -755,7 +755,7 @@ M68KVMTranslationMap040::UnmapArea(VMArea* area, bool deletingAddressSpace,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.cpp
+2 -2
View File
@@ -863,7 +863,7 @@ PPCVMTranslationMap460::UnmapPages(VMArea* area, addr_t base, size_t size,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
#endif
}
@@ -968,7 +968,7 @@ PPCVMTranslationMap460::UnmapArea(VMArea* area, bool deletingAddressSpace,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
#endif
}
src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.cpp
+2 -2
View File
@@ -863,7 +863,7 @@ PPCVMTranslationMapClassic::UnmapPages(VMArea* area, addr_t base, size_t size,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
#endif
}
@@ -968,7 +968,7 @@ PPCVMTranslationMapClassic::UnmapArea(VMArea* area, bool deletingAddressSpace,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
#endif
}
src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp
+2 -2
View File
@@ -455,7 +455,7 @@ RISCV64VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size,
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
@@ -553,7 +553,7 @@ RISCV64VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace,
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp
+2 -2
View File
@@ -500,7 +500,7 @@ X86VMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
@@ -602,7 +602,7 @@ X86VMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.cpp
+2 -2
View File
@@ -502,7 +502,7 @@ X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
@@ -598,7 +598,7 @@ X86VMTranslationMap64Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.cpp
+2 -2
View File
@@ -747,7 +747,7 @@ X86VMTranslationMapPAE::UnmapPages(VMArea* area, addr_t base, size_t size,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = queue.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
@@ -875,7 +875,7 @@ X86VMTranslationMapPAE::UnmapArea(VMArea* area, bool deletingAddressSpace,
uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
while (vm_page_mapping* mapping = mappings.RemoveHead())
object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
}
src/system/kernel/vm/VMTranslationMap.cpp
+2 -2
View File
@@ -206,7 +206,7 @@ VMTranslationMap::PageUnmapped(VMArea* area, page_num_t pageNumber,
if (mapping != NULL) {
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
object_cache_free(gPageMappingsObjectCache, mapping,
vm_free_page_mapping(pageNumber, mapping,
CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0));
}
@@ -252,7 +252,7 @@ VMTranslationMap::UnaccessedPageUnmapped(VMArea* area, page_num_t pageNumber)
atomic_add(&gMappedPagesCount, -1);
if (mapping != NULL) {
object_cache_free(gPageMappingsObjectCache, mapping,
vm_free_page_mapping(pageNumber, mapping,
CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE);
// Since this is called by the page daemon, we never want to lock
// the kernel address space.
src/system/kernel/vm/vm.cpp
+61 -12
View File
@@ -244,7 +244,8 @@ static const size_t kMemoryReserveForPriority[] = {
};
ObjectCache* gPageMappingsObjectCache;
static ObjectCache** sPageMappingsObjectCaches;
static uint32 sPageMappingsMask;
static rw_lock sAreaCacheLock = RW_LOCK_INITIALIZER("area->cache");
@@ -410,6 +411,61 @@ private:
#endif // VM_PAGE_FAULT_TRACING
// #pragma mark - page mappings allocation
static void
create_page_mappings_object_caches()
{
// We want an even power of 2 smaller than the number of CPUs.
const int32 numCPUs = smp_get_num_cpus();
int32 count = next_power_of_2(numCPUs);
if (count > numCPUs)
count >>= 1;
sPageMappingsMask = count - 1;
sPageMappingsObjectCaches = new object_cache*[count];
if (sPageMappingsObjectCaches == NULL)
panic("failed to allocate page mappings object_cache array");
for (int32 i = 0; i < count; i++) {
char name[32];
snprintf(name, sizeof(name), "page mappings %" B_PRId32, i);
object_cache* cache = create_object_cache_etc(name,
sizeof(vm_page_mapping), 0, 0, 64, 128, CACHE_LARGE_SLAB, NULL, NULL,
NULL, NULL);
if (cache == NULL)
panic("failed to create page mappings object_cache");
object_cache_set_minimum_reserve(cache, 1024);
sPageMappingsObjectCaches[i] = cache;
}
}
static object_cache*
page_mapping_object_cache_for(page_num_t page)
{
return sPageMappingsObjectCaches[page & sPageMappingsMask];
}
static vm_page_mapping*
allocate_page_mapping(page_num_t page, uint32 flags = 0)
{
return (vm_page_mapping*)object_cache_alloc(page_mapping_object_cache_for(page),
flags);
}
void
vm_free_page_mapping(page_num_t page, vm_page_mapping* mapping, uint32 flags)
{
object_cache_free(page_mapping_object_cache_for(page), mapping, flags);
}
// #pragma mark -
@@ -561,8 +617,7 @@ map_page(VMArea* area, vm_page* page, addr_t address, uint32 protection,
DEBUG_PAGE_ACCESS_CHECK(page);
bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
vm_page_mapping* mapping = (vm_page_mapping*)object_cache_alloc(
gPageMappingsObjectCache,
vm_page_mapping* mapping = allocate_page_mapping(page->physical_page_number,
CACHE_DONT_WAIT_FOR_MEMORY
| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0));
if (mapping == NULL)
@@ -4429,14 +4484,7 @@ vm_init(kernel_args* args)
(void *)ROUNDDOWN(0xdeadbeef, B_PAGE_SIZE), B_PAGE_SIZE * 64);
#endif
// create the object cache for the page mappings
gPageMappingsObjectCache = create_object_cache_etc("page mappings",
sizeof(vm_page_mapping), 0, 0, 64, 128, CACHE_LARGE_SLAB, NULL, NULL,
NULL, NULL);
if (gPageMappingsObjectCache == NULL)
panic("failed to create page mappings object cache");
object_cache_set_minimum_reserve(gPageMappingsObjectCache, 1024);
create_page_mappings_object_caches();
#if DEBUG_CACHE_LIST
if (vm_page_num_free_pages() >= 200 * 1024 * 1024 / B_PAGE_SIZE) {
@@ -5073,7 +5121,8 @@ vm_soft_fault(VMAddressSpace* addressSpace, addr_t originalAddress,
context.UnlockAll();
if (object_cache_reserve(gPageMappingsObjectCache, 1, 0)
if (object_cache_reserve(page_mapping_object_cache_for(
context.page->physical_page_number), 1, 0)
!= B_OK) {
// Apparently the situation is serious. Let's get ourselves
// killed.