kernel/vm: Use more than one object_cache for the page mappings.

Every time a page is mapped into an area on fault, we have to
allocate a mapping object for it. While the object_cache
does have per-CPU depots, these depots only store a limited
number of items, and once they run out the object_cache's lock
must be acquired.

So, to reduce lock contention on SMP systems, create a number
of object caches corresponding to the nearest power of 2
that is equal or smaller than the count of CPUs. (We already
allocate dozens of object caches for the block allocator
no matter how many CPUs there are, so a few more depending
on CPU count shouldn't impact memory use too much. Besides,
the object_caches are wired into the low_resource system.)

This significantly reduces lock contention on SMP systems.
Same benchmark setup as yesterday (compile mime_db and relink
HaikuDepot, VMware, -j4), before:
real    0m16.981s
user    0m14.357s
sys     0m6.060s

after:
real    0m14.522s
user    0m14.194s
sys     0m4.337s

And the page_mappings object_cache locks went from having 200,000+ waits
and ~14 seconds waiting time (across all threads) down to ~900 (yes,
that's not a typo) and ~0.05s wait time (though these numbers were captured
in conjunction with the following commit.)

headers/private/kernel/vm/vm.h

@@ -17,7 +17,6 @@
 
 struct generic_io_vec;
 struct kernel_args;
-struct ObjectCache;
 struct system_memory_info;
 struct VMAddressSpace;
 struct VMArea;
@@ -48,9 +47,6 @@ struct VMPageWiringInfo;
 #define VM_MEMORY_RESERVE_SYSTEM	(VM_PAGE_RESERVE_SYSTEM * B_PAGE_SIZE)
 
 
-extern struct ObjectCache* gPageMappingsObjectCache;
-
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -86,6 +82,8 @@ area_id transfer_area(area_id id, void** _address, uint32 addressSpec,
 
 const char* vm_cache_type_to_string(int32 type);
 
+void vm_free_page_mapping(page_num_t page, vm_page_mapping* mapping, uint32 flags);
+
 status_t vm_prepare_kernel_area_debug_protection(area_id id, void** cookie);
 status_t vm_set_kernel_area_debug_protection(void* cookie, void* _address,
 			size_t size, uint32 protection);

src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.cpp

@@ -517,7 +517,7 @@ ARMVMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = queue.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 }
 
 
@@ -619,7 +619,7 @@ ARMVMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = mappings.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 }
 
 

src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.cpp

@@ -638,7 +638,7 @@ M68KVMTranslationMap040::UnmapPages(VMArea* area, addr_t base, size_t size,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = queue.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 }
 
 
@@ -755,7 +755,7 @@ M68KVMTranslationMap040::UnmapArea(VMArea* area, bool deletingAddressSpace,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = mappings.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 }
 
 

src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.cpp

@@ -863,7 +863,7 @@ PPCVMTranslationMap460::UnmapPages(VMArea* area, addr_t base, size_t size,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = queue.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 #endif
 }
 
@@ -968,7 +968,7 @@ PPCVMTranslationMap460::UnmapArea(VMArea* area, bool deletingAddressSpace,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = mappings.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 #endif
 }
 

src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.cpp

@@ -863,7 +863,7 @@ PPCVMTranslationMapClassic::UnmapPages(VMArea* area, addr_t base, size_t size,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = queue.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 #endif
 }
 
@@ -968,7 +968,7 @@ PPCVMTranslationMapClassic::UnmapArea(VMArea* area, bool deletingAddressSpace,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = mappings.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 #endif
 }
 

src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp

@@ -455,7 +455,7 @@ RISCV64VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size,
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 
 	while (vm_page_mapping* mapping = queue.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 }
 
 
@@ -553,7 +553,7 @@ RISCV64VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace,
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 
 	while (vm_page_mapping* mapping = mappings.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 }
 
 

src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp

@@ -500,7 +500,7 @@ X86VMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = queue.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 }
 
 
@@ -602,7 +602,7 @@ X86VMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = mappings.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 }
 
 

src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.cpp

@@ -502,7 +502,7 @@ X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = queue.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 }
 
 
@@ -598,7 +598,7 @@ X86VMTranslationMap64Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = mappings.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 }
 
 

src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.cpp

@@ -747,7 +747,7 @@ X86VMTranslationMapPAE::UnmapPages(VMArea* area, addr_t base, size_t size,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = queue.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 }
 
 
@@ -875,7 +875,7 @@ X86VMTranslationMapPAE::UnmapArea(VMArea* area, bool deletingAddressSpace,
 	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
 		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
 	while (vm_page_mapping* mapping = mappings.RemoveHead())
-		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
+		vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags);
 }
 
 

src/system/kernel/vm/VMTranslationMap.cpp

@@ -206,7 +206,7 @@ VMTranslationMap::PageUnmapped(VMArea* area, page_num_t pageNumber,
 
 	if (mapping != NULL) {
 		bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
-		object_cache_free(gPageMappingsObjectCache, mapping,
+		vm_free_page_mapping(pageNumber, mapping,
 			CACHE_DONT_WAIT_FOR_MEMORY
 				| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0));
 	}
@@ -252,7 +252,7 @@ VMTranslationMap::UnaccessedPageUnmapped(VMArea* area, page_num_t pageNumber)
 		atomic_add(&gMappedPagesCount, -1);
 
 	if (mapping != NULL) {
-		object_cache_free(gPageMappingsObjectCache, mapping,
+		vm_free_page_mapping(pageNumber, mapping,
 			CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE);
 			// Since this is called by the page daemon, we never want to lock
 			// the kernel address space.

src/system/kernel/vm/vm.cpp

@@ -244,7 +244,8 @@ static const size_t kMemoryReserveForPriority[] = {
 };
 
 
-ObjectCache* gPageMappingsObjectCache;
+static ObjectCache** sPageMappingsObjectCaches;
+static uint32 sPageMappingsMask;
 
 static rw_lock sAreaCacheLock = RW_LOCK_INITIALIZER("area->cache");
 
@@ -410,6 +411,61 @@ private:
 #endif	// VM_PAGE_FAULT_TRACING
 
 
+//	#pragma mark - page mappings allocation
+
+
+static void
+create_page_mappings_object_caches()
+{
+	// We want an even power of 2 smaller than the number of CPUs.
+	const int32 numCPUs = smp_get_num_cpus();
+	int32 count = next_power_of_2(numCPUs);
+	if (count > numCPUs)
+		count >>= 1;
+	sPageMappingsMask = count - 1;
+
+	sPageMappingsObjectCaches = new object_cache*[count];
+	if (sPageMappingsObjectCaches == NULL)
+		panic("failed to allocate page mappings object_cache array");
+
+	for (int32 i = 0; i < count; i++) {
+		char name[32];
+		snprintf(name, sizeof(name), "page mappings %" B_PRId32, i);
+
+		object_cache* cache = create_object_cache_etc(name,
+			sizeof(vm_page_mapping), 0, 0, 64, 128, CACHE_LARGE_SLAB, NULL, NULL,
+			NULL, NULL);
+		if (cache == NULL)
+			panic("failed to create page mappings object_cache");
+
+		object_cache_set_minimum_reserve(cache, 1024);
+		sPageMappingsObjectCaches[i] = cache;
+	}
+}
+
+
+static object_cache*
+page_mapping_object_cache_for(page_num_t page)
+{
+	return sPageMappingsObjectCaches[page & sPageMappingsMask];
+}
+
+
+static vm_page_mapping*
+allocate_page_mapping(page_num_t page, uint32 flags = 0)
+{
+	return (vm_page_mapping*)object_cache_alloc(page_mapping_object_cache_for(page),
+		flags);
+}
+
+
+void
+vm_free_page_mapping(page_num_t page, vm_page_mapping* mapping, uint32 flags)
+{
+	object_cache_free(page_mapping_object_cache_for(page), mapping, flags);
+}
+
+
 //	#pragma mark -
 
 
@@ -561,8 +617,7 @@ map_page(VMArea* area, vm_page* page, addr_t address, uint32 protection,
 		DEBUG_PAGE_ACCESS_CHECK(page);
 
 		bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
-		vm_page_mapping* mapping = (vm_page_mapping*)object_cache_alloc(
+		vm_page_mapping* mapping = allocate_page_mapping(page->physical_page_number,
-			gPageMappingsObjectCache,
 			CACHE_DONT_WAIT_FOR_MEMORY
 				| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0));
 		if (mapping == NULL)
@@ -4429,14 +4484,7 @@ vm_init(kernel_args* args)
 		(void *)ROUNDDOWN(0xdeadbeef, B_PAGE_SIZE), B_PAGE_SIZE * 64);
 #endif
 
-	// create the object cache for the page mappings
+	create_page_mappings_object_caches();
-	gPageMappingsObjectCache = create_object_cache_etc("page mappings",
-		sizeof(vm_page_mapping), 0, 0, 64, 128, CACHE_LARGE_SLAB, NULL, NULL,
-		NULL, NULL);
-	if (gPageMappingsObjectCache == NULL)
-		panic("failed to create page mappings object cache");
-
-	object_cache_set_minimum_reserve(gPageMappingsObjectCache, 1024);
 
 #if DEBUG_CACHE_LIST
 	if (vm_page_num_free_pages() >= 200 * 1024 * 1024 / B_PAGE_SIZE) {
@@ -5073,7 +5121,8 @@ vm_soft_fault(VMAddressSpace* addressSpace, addr_t originalAddress,
 
 				context.UnlockAll();
 
-				if (object_cache_reserve(gPageMappingsObjectCache, 1, 0)
+				if (object_cache_reserve(page_mapping_object_cache_for(
+							context.page->physical_page_number), 1, 0)
 						!= B_OK) {
 					// Apparently the situation is serious. Let's get ourselves
 					// killed.