From 74785e79db32355e0a8ee6b488672ac09ad57b1b Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Tue, 7 Oct 2008 21:39:19 +0000 Subject: [PATCH] * Added "from" address space parameter to vm_swap_address_space()/ arch_vm_aspace_swap(). * The x86 implementation does now maintain a bit mask per vm_translation_map_arch_info indicating on which CPUs the address space is active. This allows flush_tmap() to avoid ICI for user address spaces when the team isn't currently running on any other CPU. In this context ICI is relatively expensive, particularly since we map most pages via vm_map_page() and therefore invoke flush_tmap() pretty much for every single page. This optimization speeds up a "hello world" compilation about 20% on my machine (KDEBUG turned off, freshly booted), but interestingly it has virtually no effect on the "-j2" haiku build time. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@27912 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/private/kernel/arch/vm.h | 3 +- .../kernel/arch/x86/arch_vm_translation_map.h | 3 +- headers/private/kernel/vm_address_space.h | 2 +- src/system/kernel/arch/m68k/arch_vm.cpp | 11 +++--- src/system/kernel/arch/ppc/arch_vm.cpp | 8 ++--- src/system/kernel/arch/x86/arch_thread.cpp | 17 +++++++-- src/system/kernel/arch/x86/arch_vm.cpp | 23 +++++++++--- .../arch/x86/arch_vm_translation_map.cpp | 36 ++++++++++++++----- src/system/kernel/thread.cpp | 2 +- 9 files changed, 76 insertions(+), 29 deletions(-) diff --git a/headers/private/kernel/arch/vm.h b/headers/private/kernel/arch/vm.h index d40cc9bbce..5d62638674 100644 --- a/headers/private/kernel/arch/vm.h +++ b/headers/private/kernel/arch/vm.h @@ -27,7 +27,8 @@ status_t arch_vm_init(struct kernel_args *args); status_t arch_vm_init_post_area(struct kernel_args *args); status_t arch_vm_init_end(struct kernel_args *args); status_t arch_vm_init_post_modules(struct kernel_args *args); -void arch_vm_aspace_swap(struct vm_address_space *aspace); +void arch_vm_aspace_swap(struct vm_address_space *from, + struct vm_address_space *to); bool arch_vm_supports_protection(uint32 protection); status_t arch_vm_set_memory_type(struct vm_area *area, addr_t physicalBase, diff --git a/headers/private/kernel/arch/x86/arch_vm_translation_map.h b/headers/private/kernel/arch/x86/arch_vm_translation_map.h index 29e4db0d9c..6f2a0e58b5 100644 --- a/headers/private/kernel/arch/x86/arch_vm_translation_map.h +++ b/headers/private/kernel/arch/x86/arch_vm_translation_map.h @@ -13,10 +13,11 @@ struct page_directory_entry; - typedef struct vm_translation_map_arch_info { struct page_directory_entry *pgdir_virt; struct page_directory_entry *pgdir_phys; + vint32 active_on_cpus; + // mask indicating on which CPUs the map is currently used int num_invalidate_pages; addr_t pages_to_invalidate[PAGE_INVALIDATE_CACHE_SIZE]; } vm_translation_map_arch_info; diff --git a/headers/private/kernel/vm_address_space.h b/headers/private/kernel/vm_address_space.h index 81c3021840..6155c9d470 100644 --- a/headers/private/kernel/vm_address_space.h +++ b/headers/private/kernel/vm_address_space.h @@ -34,7 +34,7 @@ struct vm_address_space *vm_get_current_user_address_space(void); team_id vm_current_user_address_space_id(void); struct vm_address_space *vm_get_address_space(team_id team); void vm_put_address_space(struct vm_address_space *aspace); -#define vm_swap_address_space(aspace) arch_vm_aspace_swap(aspace) +#define vm_swap_address_space(from, to) arch_vm_aspace_swap(from, to) #ifdef __cplusplus } diff --git a/src/system/kernel/arch/m68k/arch_vm.cpp b/src/system/kernel/arch/m68k/arch_vm.cpp index 46c0f3c8b5..12661dc660 100644 --- a/src/system/kernel/arch/m68k/arch_vm.cpp +++ b/src/system/kernel/arch/m68k/arch_vm.cpp @@ -29,7 +29,7 @@ #warning M68K: WRITEME -status_t +status_t arch_vm_init(kernel_args *args) { return B_OK; @@ -62,7 +62,7 @@ arch_vm_init_end(kernel_args *args) #if 0 TRACE(("arch_vm_init_end(): %lu virtual ranges to keep:\n", args->arch_args.num_virtual_ranges_to_keep)); - + for (int i = 0; i < (int)args->arch_args.num_virtual_ranges_to_keep; i++) { addr_range &range = args->arch_args.virtual_ranges_to_keep[i]; @@ -102,11 +102,10 @@ arch_vm_init_post_modules(kernel_args *args) } -void -arch_vm_aspace_swap(vm_address_space *aspace) +void +arch_vm_aspace_swap(struct vm_address_space *from, struct vm_address_space *to) { - m68k_set_pgdir(m68k_translation_map_get_pgdir( - &aspace->translation_map)); + m68k_set_pgdir(m68k_translation_map_get_pgdir(&to->translation_map)); } diff --git a/src/system/kernel/arch/ppc/arch_vm.cpp b/src/system/kernel/arch/ppc/arch_vm.cpp index a1c6469a87..eef8346f33 100644 --- a/src/system/kernel/arch/ppc/arch_vm.cpp +++ b/src/system/kernel/arch/ppc/arch_vm.cpp @@ -24,7 +24,7 @@ #endif -status_t +status_t arch_vm_init(kernel_args *args) { return B_OK; @@ -105,7 +105,7 @@ arch_vm_init_end(kernel_args *args) { TRACE(("arch_vm_init_end(): %lu virtual ranges to keep:\n", args->arch_args.num_virtual_ranges_to_keep)); - + for (int i = 0; i < (int)args->arch_args.num_virtual_ranges_to_keep; i++) { addr_range &range = args->arch_args.virtual_ranges_to_keep[i]; @@ -143,8 +143,8 @@ arch_vm_init_post_modules(kernel_args *args) } -void -arch_vm_aspace_swap(vm_address_space *aspace) +void +arch_vm_aspace_swap(struct vm_address_space *from, struct vm_address_space *to) { } diff --git a/src/system/kernel/arch/x86/arch_thread.cpp b/src/system/kernel/arch/x86/arch_thread.cpp index c8dbc105be..8df02f64d2 100644 --- a/src/system/kernel/arch/x86/arch_thread.cpp +++ b/src/system/kernel/arch/x86/arch_thread.cpp @@ -358,8 +358,21 @@ arch_thread_context_switch(struct thread *from, struct thread *to) newPageDirectory = (addr_t)x86_next_page_directory(from, to); - if ((newPageDirectory % B_PAGE_SIZE) != 0) - panic("arch_thread_context_switch: bad pgdir 0x%lx\n", newPageDirectory); + ASSERT((newPageDirectory % B_PAGE_SIZE) == 0); + + if (newPageDirectory != 0) { + // update on which CPUs the address space is used + int cpu = smp_get_current_cpu(); + if (vm_address_space* addressSpace = from->team->address_space) { + atomic_and(&addressSpace->translation_map.arch_data->active_on_cpus, + ~((uint32)1 << cpu)); + } + + if (vm_address_space* addressSpace = to->team->address_space) { + atomic_or(&addressSpace->translation_map.arch_data->active_on_cpus, + (uint32)1 << cpu); + } + } gX86SwapFPUFunc(from->arch_info.fpu_state, to->arch_info.fpu_state); i386_context_switch(&from->arch_info, &to->arch_info, newPageDirectory); diff --git a/src/system/kernel/arch/x86/arch_vm.cpp b/src/system/kernel/arch/x86/arch_vm.cpp index 6cdd123a21..894d5784e5 100644 --- a/src/system/kernel/arch/x86/arch_vm.cpp +++ b/src/system/kernel/arch/x86/arch_vm.cpp @@ -8,10 +8,15 @@ */ +#include +#include + #include + #include #include #include +#include #include #include @@ -21,9 +26,6 @@ #include -#include -#include - //#define TRACE_ARCH_VM #ifdef TRACE_ARCH_VM @@ -382,10 +384,21 @@ arch_vm_init_post_modules(kernel_args *args) void -arch_vm_aspace_swap(vm_address_space *aspace) +arch_vm_aspace_swap(struct vm_address_space *from, struct vm_address_space *to) { + int cpu = smp_get_current_cpu(); + if (from != NULL) { + atomic_and(&from->translation_map.arch_data->active_on_cpus, + ~((uint32)1 << cpu)); + } + + if (to != NULL && to != vm_kernel_address_space()) { + atomic_or(&to->translation_map.arch_data->active_on_cpus, + (uint32)1 << cpu); + } + i386_swap_pgdir((addr_t)i386_translation_map_get_pgdir( - &aspace->translation_map)); + &to->translation_map)); } diff --git a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp index 22b9d30bd7..06f638ddca 100644 --- a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp +++ b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp @@ -662,12 +662,18 @@ flush_tmap(vm_translation_map *map) if (IS_KERNEL_MAP(map)) { arch_cpu_global_TLB_invalidate(); - smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, NULL, - SMP_MSG_FLAG_SYNC); + smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, + NULL, SMP_MSG_FLAG_SYNC); } else { arch_cpu_user_TLB_invalidate(); - smp_send_broadcast_ici(SMP_MSG_USER_INVALIDATE_PAGES, 0, 0, 0, NULL, - SMP_MSG_FLAG_SYNC); + + int cpu = smp_get_current_cpu(); + uint32 cpuMask = map->arch_data->active_on_cpus + & ~((uint32)1 << cpu); + if (cpuMask != 0) { + smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES, + 0, 0, 0, NULL, SMP_MSG_FLAG_SYNC); + } } } else { TRACE(("flush_tmap: %d pages to invalidate, invalidate list\n", @@ -675,10 +681,23 @@ flush_tmap(vm_translation_map *map) arch_cpu_invalidate_TLB_list(map->arch_data->pages_to_invalidate, map->arch_data->num_invalidate_pages); - smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, - (uint32)map->arch_data->pages_to_invalidate, - map->arch_data->num_invalidate_pages, 0, NULL, - SMP_MSG_FLAG_SYNC); + + if (IS_KERNEL_MAP(map)) { + smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, + (uint32)map->arch_data->pages_to_invalidate, + map->arch_data->num_invalidate_pages, 0, NULL, + SMP_MSG_FLAG_SYNC); + } else { + int cpu = smp_get_current_cpu(); + uint32 cpuMask = map->arch_data->active_on_cpus + & ~((uint32)1 << cpu); + if (cpuMask != 0) { + smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST, + (uint32)map->arch_data->pages_to_invalidate, + map->arch_data->num_invalidate_pages, 0, NULL, + SMP_MSG_FLAG_SYNC); + } + } } map->arch_data->num_invalidate_pages = 0; @@ -777,6 +796,7 @@ arch_vm_translation_map_init_map(vm_translation_map *map, bool kernel) return B_NO_MEMORY; } + map->arch_data->active_on_cpus = 0; map->arch_data->num_invalidate_pages = 0; if (!kernel) { diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 979787c02a..46a251f74c 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -1441,7 +1441,7 @@ thread_exit(void) RELEASE_TEAM_LOCK(); // swap address spaces, to make sure we're running on the kernel's pgdir - vm_swap_address_space(vm_kernel_address_space()); + vm_swap_address_space(team->address_space, vm_kernel_address_space()); restore_interrupts(state); TRACE(("thread_exit: thread %ld now a kernel thread!\n", thread->id));