diff --git a/headers/private/kernel/vm/vm.h b/headers/private/kernel/vm/vm.h index 15e5618f7a..9813f41196 100644 --- a/headers/private/kernel/vm/vm.h +++ b/headers/private/kernel/vm/vm.h @@ -17,7 +17,6 @@ struct generic_io_vec; struct kernel_args; -struct ObjectCache; struct system_memory_info; struct VMAddressSpace; struct VMArea; @@ -48,9 +47,6 @@ struct VMPageWiringInfo; #define VM_MEMORY_RESERVE_SYSTEM (VM_PAGE_RESERVE_SYSTEM * B_PAGE_SIZE) -extern struct ObjectCache* gPageMappingsObjectCache; - - #ifdef __cplusplus extern "C" { #endif @@ -86,6 +82,8 @@ area_id transfer_area(area_id id, void** _address, uint32 addressSpec, const char* vm_cache_type_to_string(int32 type); +void vm_free_page_mapping(page_num_t page, vm_page_mapping* mapping, uint32 flags); + status_t vm_prepare_kernel_area_debug_protection(area_id id, void** cookie); status_t vm_set_kernel_area_debug_protection(void* cookie, void* _address, size_t size, uint32 protection); diff --git a/src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.cpp b/src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.cpp index 90b03f6d94..c1462426b6 100644 --- a/src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.cpp +++ b/src/system/kernel/arch/arm/paging/32bit/ARMVMTranslationMap32Bit.cpp @@ -517,7 +517,7 @@ ARMVMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = queue.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); } @@ -619,7 +619,7 @@ ARMVMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = mappings.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); } diff --git a/src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.cpp b/src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.cpp index c54879973f..f2fa356735 100644 --- a/src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.cpp +++ b/src/system/kernel/arch/m68k/paging/040/M68KVMTranslationMap040.cpp @@ -638,7 +638,7 @@ M68KVMTranslationMap040::UnmapPages(VMArea* area, addr_t base, size_t size, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = queue.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); } @@ -755,7 +755,7 @@ M68KVMTranslationMap040::UnmapArea(VMArea* area, bool deletingAddressSpace, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = mappings.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); } diff --git a/src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.cpp b/src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.cpp index b028e02db9..13fdfe6a40 100644 --- a/src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.cpp +++ b/src/system/kernel/arch/ppc/paging/460/PPCVMTranslationMap460.cpp @@ -863,7 +863,7 @@ PPCVMTranslationMap460::UnmapPages(VMArea* area, addr_t base, size_t size, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = queue.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); #endif } @@ -968,7 +968,7 @@ PPCVMTranslationMap460::UnmapArea(VMArea* area, bool deletingAddressSpace, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = mappings.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); #endif } diff --git a/src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.cpp b/src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.cpp index dd7131743a..1e7b70e5c4 100644 --- a/src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.cpp +++ b/src/system/kernel/arch/ppc/paging/classic/PPCVMTranslationMapClassic.cpp @@ -863,7 +863,7 @@ PPCVMTranslationMapClassic::UnmapPages(VMArea* area, addr_t base, size_t size, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = queue.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); #endif } @@ -968,7 +968,7 @@ PPCVMTranslationMapClassic::UnmapArea(VMArea* area, bool deletingAddressSpace, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = mappings.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); #endif } diff --git a/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp b/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp index e2a9cd7415..d88fe3c563 100644 --- a/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp +++ b/src/system/kernel/arch/riscv64/RISCV64VMTranslationMap.cpp @@ -455,7 +455,7 @@ RISCV64VMTranslationMap::UnmapPages(VMArea* area, addr_t base, size_t size, | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = queue.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); } @@ -553,7 +553,7 @@ RISCV64VMTranslationMap::UnmapArea(VMArea* area, bool deletingAddressSpace, | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = mappings.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); } diff --git a/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp b/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp index f11b6955ad..ca55208dbc 100644 --- a/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp +++ b/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp @@ -500,7 +500,7 @@ X86VMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = queue.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); } @@ -602,7 +602,7 @@ X86VMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = mappings.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); } diff --git a/src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.cpp b/src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.cpp index 71e1f891ac..54a8ee857b 100644 --- a/src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.cpp +++ b/src/system/kernel/arch/x86/paging/64bit/X86VMTranslationMap64Bit.cpp @@ -502,7 +502,7 @@ X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = queue.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); } @@ -598,7 +598,7 @@ X86VMTranslationMap64Bit::UnmapArea(VMArea* area, bool deletingAddressSpace, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = mappings.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); } diff --git a/src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.cpp b/src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.cpp index e51d18e963..4128cb1e3a 100644 --- a/src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.cpp +++ b/src/system/kernel/arch/x86/paging/pae/X86VMTranslationMapPAE.cpp @@ -747,7 +747,7 @@ X86VMTranslationMapPAE::UnmapPages(VMArea* area, addr_t base, size_t size, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = queue.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); } @@ -875,7 +875,7 @@ X86VMTranslationMapPAE::UnmapArea(VMArea* area, bool deletingAddressSpace, uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); while (vm_page_mapping* mapping = mappings.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); + vm_free_page_mapping(mapping->page->physical_page_number, mapping, freeFlags); } diff --git a/src/system/kernel/vm/VMTranslationMap.cpp b/src/system/kernel/vm/VMTranslationMap.cpp index ee937db3a3..a93d23cf2f 100644 --- a/src/system/kernel/vm/VMTranslationMap.cpp +++ b/src/system/kernel/vm/VMTranslationMap.cpp @@ -206,7 +206,7 @@ VMTranslationMap::PageUnmapped(VMArea* area, page_num_t pageNumber, if (mapping != NULL) { bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - object_cache_free(gPageMappingsObjectCache, mapping, + vm_free_page_mapping(pageNumber, mapping, CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0)); } @@ -252,7 +252,7 @@ VMTranslationMap::UnaccessedPageUnmapped(VMArea* area, page_num_t pageNumber) atomic_add(&gMappedPagesCount, -1); if (mapping != NULL) { - object_cache_free(gPageMappingsObjectCache, mapping, + vm_free_page_mapping(pageNumber, mapping, CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); // Since this is called by the page daemon, we never want to lock // the kernel address space. diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index 86f06896a5..4009319ae2 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -244,7 +244,8 @@ static const size_t kMemoryReserveForPriority[] = { }; -ObjectCache* gPageMappingsObjectCache; +static ObjectCache** sPageMappingsObjectCaches; +static uint32 sPageMappingsMask; static rw_lock sAreaCacheLock = RW_LOCK_INITIALIZER("area->cache"); @@ -410,6 +411,61 @@ private: #endif // VM_PAGE_FAULT_TRACING +// #pragma mark - page mappings allocation + + +static void +create_page_mappings_object_caches() +{ + // We want an even power of 2 smaller than the number of CPUs. + const int32 numCPUs = smp_get_num_cpus(); + int32 count = next_power_of_2(numCPUs); + if (count > numCPUs) + count >>= 1; + sPageMappingsMask = count - 1; + + sPageMappingsObjectCaches = new object_cache*[count]; + if (sPageMappingsObjectCaches == NULL) + panic("failed to allocate page mappings object_cache array"); + + for (int32 i = 0; i < count; i++) { + char name[32]; + snprintf(name, sizeof(name), "page mappings %" B_PRId32, i); + + object_cache* cache = create_object_cache_etc(name, + sizeof(vm_page_mapping), 0, 0, 64, 128, CACHE_LARGE_SLAB, NULL, NULL, + NULL, NULL); + if (cache == NULL) + panic("failed to create page mappings object_cache"); + + object_cache_set_minimum_reserve(cache, 1024); + sPageMappingsObjectCaches[i] = cache; + } +} + + +static object_cache* +page_mapping_object_cache_for(page_num_t page) +{ + return sPageMappingsObjectCaches[page & sPageMappingsMask]; +} + + +static vm_page_mapping* +allocate_page_mapping(page_num_t page, uint32 flags = 0) +{ + return (vm_page_mapping*)object_cache_alloc(page_mapping_object_cache_for(page), + flags); +} + + +void +vm_free_page_mapping(page_num_t page, vm_page_mapping* mapping, uint32 flags) +{ + object_cache_free(page_mapping_object_cache_for(page), mapping, flags); +} + + // #pragma mark - @@ -561,8 +617,7 @@ map_page(VMArea* area, vm_page* page, addr_t address, uint32 protection, DEBUG_PAGE_ACCESS_CHECK(page); bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - vm_page_mapping* mapping = (vm_page_mapping*)object_cache_alloc( - gPageMappingsObjectCache, + vm_page_mapping* mapping = allocate_page_mapping(page->physical_page_number, CACHE_DONT_WAIT_FOR_MEMORY | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0)); if (mapping == NULL) @@ -4429,14 +4484,7 @@ vm_init(kernel_args* args) (void *)ROUNDDOWN(0xdeadbeef, B_PAGE_SIZE), B_PAGE_SIZE * 64); #endif - // create the object cache for the page mappings - gPageMappingsObjectCache = create_object_cache_etc("page mappings", - sizeof(vm_page_mapping), 0, 0, 64, 128, CACHE_LARGE_SLAB, NULL, NULL, - NULL, NULL); - if (gPageMappingsObjectCache == NULL) - panic("failed to create page mappings object cache"); - - object_cache_set_minimum_reserve(gPageMappingsObjectCache, 1024); + create_page_mappings_object_caches(); #if DEBUG_CACHE_LIST if (vm_page_num_free_pages() >= 200 * 1024 * 1024 / B_PAGE_SIZE) { @@ -5073,7 +5121,8 @@ vm_soft_fault(VMAddressSpace* addressSpace, addr_t originalAddress, context.UnlockAll(); - if (object_cache_reserve(gPageMappingsObjectCache, 1, 0) + if (object_cache_reserve(page_mapping_object_cache_for( + context.page->physical_page_number), 1, 0) != B_OK) { // Apparently the situation is serious. Let's get ourselves // killed.