diff --git a/headers/private/kernel/arch/vm.h b/headers/private/kernel/arch/vm.h index d40cc9bbce..5d62638674 100644 --- a/headers/private/kernel/arch/vm.h +++ b/headers/private/kernel/arch/vm.h @@ -27,7 +27,8 @@ status_t arch_vm_init(struct kernel_args *args); status_t arch_vm_init_post_area(struct kernel_args *args); status_t arch_vm_init_end(struct kernel_args *args); status_t arch_vm_init_post_modules(struct kernel_args *args); -void arch_vm_aspace_swap(struct vm_address_space *aspace); +void arch_vm_aspace_swap(struct vm_address_space *from, + struct vm_address_space *to); bool arch_vm_supports_protection(uint32 protection); status_t arch_vm_set_memory_type(struct vm_area *area, addr_t physicalBase, diff --git a/headers/private/kernel/arch/x86/arch_vm_translation_map.h b/headers/private/kernel/arch/x86/arch_vm_translation_map.h index 29e4db0d9c..6f2a0e58b5 100644 --- a/headers/private/kernel/arch/x86/arch_vm_translation_map.h +++ b/headers/private/kernel/arch/x86/arch_vm_translation_map.h @@ -13,10 +13,11 @@ struct page_directory_entry; - typedef struct vm_translation_map_arch_info { struct page_directory_entry *pgdir_virt; struct page_directory_entry *pgdir_phys; + vint32 active_on_cpus; + // mask indicating on which CPUs the map is currently used int num_invalidate_pages; addr_t pages_to_invalidate[PAGE_INVALIDATE_CACHE_SIZE]; } vm_translation_map_arch_info; diff --git a/headers/private/kernel/vm_address_space.h b/headers/private/kernel/vm_address_space.h index 81c3021840..6155c9d470 100644 --- a/headers/private/kernel/vm_address_space.h +++ b/headers/private/kernel/vm_address_space.h @@ -34,7 +34,7 @@ struct vm_address_space *vm_get_current_user_address_space(void); team_id vm_current_user_address_space_id(void); struct vm_address_space *vm_get_address_space(team_id team); void vm_put_address_space(struct vm_address_space *aspace); -#define vm_swap_address_space(aspace) arch_vm_aspace_swap(aspace) +#define vm_swap_address_space(from, to) arch_vm_aspace_swap(from, to) #ifdef __cplusplus } diff --git a/src/system/kernel/arch/m68k/arch_vm.cpp b/src/system/kernel/arch/m68k/arch_vm.cpp index 46c0f3c8b5..12661dc660 100644 --- a/src/system/kernel/arch/m68k/arch_vm.cpp +++ b/src/system/kernel/arch/m68k/arch_vm.cpp @@ -29,7 +29,7 @@ #warning M68K: WRITEME -status_t +status_t arch_vm_init(kernel_args *args) { return B_OK; @@ -62,7 +62,7 @@ arch_vm_init_end(kernel_args *args) #if 0 TRACE(("arch_vm_init_end(): %lu virtual ranges to keep:\n", args->arch_args.num_virtual_ranges_to_keep)); - + for (int i = 0; i < (int)args->arch_args.num_virtual_ranges_to_keep; i++) { addr_range &range = args->arch_args.virtual_ranges_to_keep[i]; @@ -102,11 +102,10 @@ arch_vm_init_post_modules(kernel_args *args) } -void -arch_vm_aspace_swap(vm_address_space *aspace) +void +arch_vm_aspace_swap(struct vm_address_space *from, struct vm_address_space *to) { - m68k_set_pgdir(m68k_translation_map_get_pgdir( - &aspace->translation_map)); + m68k_set_pgdir(m68k_translation_map_get_pgdir(&to->translation_map)); } diff --git a/src/system/kernel/arch/ppc/arch_vm.cpp b/src/system/kernel/arch/ppc/arch_vm.cpp index a1c6469a87..eef8346f33 100644 --- a/src/system/kernel/arch/ppc/arch_vm.cpp +++ b/src/system/kernel/arch/ppc/arch_vm.cpp @@ -24,7 +24,7 @@ #endif -status_t +status_t arch_vm_init(kernel_args *args) { return B_OK; @@ -105,7 +105,7 @@ arch_vm_init_end(kernel_args *args) { TRACE(("arch_vm_init_end(): %lu virtual ranges to keep:\n", args->arch_args.num_virtual_ranges_to_keep)); - + for (int i = 0; i < (int)args->arch_args.num_virtual_ranges_to_keep; i++) { addr_range &range = args->arch_args.virtual_ranges_to_keep[i]; @@ -143,8 +143,8 @@ arch_vm_init_post_modules(kernel_args *args) } -void -arch_vm_aspace_swap(vm_address_space *aspace) +void +arch_vm_aspace_swap(struct vm_address_space *from, struct vm_address_space *to) { } diff --git a/src/system/kernel/arch/x86/arch_thread.cpp b/src/system/kernel/arch/x86/arch_thread.cpp index c8dbc105be..8df02f64d2 100644 --- a/src/system/kernel/arch/x86/arch_thread.cpp +++ b/src/system/kernel/arch/x86/arch_thread.cpp @@ -358,8 +358,21 @@ arch_thread_context_switch(struct thread *from, struct thread *to) newPageDirectory = (addr_t)x86_next_page_directory(from, to); - if ((newPageDirectory % B_PAGE_SIZE) != 0) - panic("arch_thread_context_switch: bad pgdir 0x%lx\n", newPageDirectory); + ASSERT((newPageDirectory % B_PAGE_SIZE) == 0); + + if (newPageDirectory != 0) { + // update on which CPUs the address space is used + int cpu = smp_get_current_cpu(); + if (vm_address_space* addressSpace = from->team->address_space) { + atomic_and(&addressSpace->translation_map.arch_data->active_on_cpus, + ~((uint32)1 << cpu)); + } + + if (vm_address_space* addressSpace = to->team->address_space) { + atomic_or(&addressSpace->translation_map.arch_data->active_on_cpus, + (uint32)1 << cpu); + } + } gX86SwapFPUFunc(from->arch_info.fpu_state, to->arch_info.fpu_state); i386_context_switch(&from->arch_info, &to->arch_info, newPageDirectory); diff --git a/src/system/kernel/arch/x86/arch_vm.cpp b/src/system/kernel/arch/x86/arch_vm.cpp index 6cdd123a21..894d5784e5 100644 --- a/src/system/kernel/arch/x86/arch_vm.cpp +++ b/src/system/kernel/arch/x86/arch_vm.cpp @@ -8,10 +8,15 @@ */ +#include +#include + #include + #include #include #include +#include #include #include @@ -21,9 +26,6 @@ #include -#include -#include - //#define TRACE_ARCH_VM #ifdef TRACE_ARCH_VM @@ -382,10 +384,21 @@ arch_vm_init_post_modules(kernel_args *args) void -arch_vm_aspace_swap(vm_address_space *aspace) +arch_vm_aspace_swap(struct vm_address_space *from, struct vm_address_space *to) { + int cpu = smp_get_current_cpu(); + if (from != NULL) { + atomic_and(&from->translation_map.arch_data->active_on_cpus, + ~((uint32)1 << cpu)); + } + + if (to != NULL && to != vm_kernel_address_space()) { + atomic_or(&to->translation_map.arch_data->active_on_cpus, + (uint32)1 << cpu); + } + i386_swap_pgdir((addr_t)i386_translation_map_get_pgdir( - &aspace->translation_map)); + &to->translation_map)); } diff --git a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp index 22b9d30bd7..06f638ddca 100644 --- a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp +++ b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp @@ -662,12 +662,18 @@ flush_tmap(vm_translation_map *map) if (IS_KERNEL_MAP(map)) { arch_cpu_global_TLB_invalidate(); - smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, NULL, - SMP_MSG_FLAG_SYNC); + smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, + NULL, SMP_MSG_FLAG_SYNC); } else { arch_cpu_user_TLB_invalidate(); - smp_send_broadcast_ici(SMP_MSG_USER_INVALIDATE_PAGES, 0, 0, 0, NULL, - SMP_MSG_FLAG_SYNC); + + int cpu = smp_get_current_cpu(); + uint32 cpuMask = map->arch_data->active_on_cpus + & ~((uint32)1 << cpu); + if (cpuMask != 0) { + smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES, + 0, 0, 0, NULL, SMP_MSG_FLAG_SYNC); + } } } else { TRACE(("flush_tmap: %d pages to invalidate, invalidate list\n", @@ -675,10 +681,23 @@ flush_tmap(vm_translation_map *map) arch_cpu_invalidate_TLB_list(map->arch_data->pages_to_invalidate, map->arch_data->num_invalidate_pages); - smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, - (uint32)map->arch_data->pages_to_invalidate, - map->arch_data->num_invalidate_pages, 0, NULL, - SMP_MSG_FLAG_SYNC); + + if (IS_KERNEL_MAP(map)) { + smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, + (uint32)map->arch_data->pages_to_invalidate, + map->arch_data->num_invalidate_pages, 0, NULL, + SMP_MSG_FLAG_SYNC); + } else { + int cpu = smp_get_current_cpu(); + uint32 cpuMask = map->arch_data->active_on_cpus + & ~((uint32)1 << cpu); + if (cpuMask != 0) { + smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST, + (uint32)map->arch_data->pages_to_invalidate, + map->arch_data->num_invalidate_pages, 0, NULL, + SMP_MSG_FLAG_SYNC); + } + } } map->arch_data->num_invalidate_pages = 0; @@ -777,6 +796,7 @@ arch_vm_translation_map_init_map(vm_translation_map *map, bool kernel) return B_NO_MEMORY; } + map->arch_data->active_on_cpus = 0; map->arch_data->num_invalidate_pages = 0; if (!kernel) { diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 979787c02a..46a251f74c 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -1441,7 +1441,7 @@ thread_exit(void) RELEASE_TEAM_LOCK(); // swap address spaces, to make sure we're running on the kernel's pgdir - vm_swap_address_space(vm_kernel_address_space()); + vm_swap_address_space(team->address_space, vm_kernel_address_space()); restore_interrupts(state); TRACE(("thread_exit: thread %ld now a kernel thread!\n", thread->id));