* Added "from" address space parameter to vm_swap_address_space()/

arch_vm_aspace_swap().
* The x86 implementation does now maintain a bit mask per
  vm_translation_map_arch_info indicating on which CPUs the address
  space is active. This allows flush_tmap() to avoid ICI for user
  address spaces when the team isn't currently running on any other CPU.
  In this context ICI is relatively expensive, particularly since we map
  most pages via vm_map_page() and therefore invoke flush_tmap() pretty
  much for every single page.
  This optimization speeds up a "hello world" compilation about 20% on
  my machine (KDEBUG turned off, freshly booted), but interestingly it
  has virtually no effect on the "-j2" haiku build time.


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@27912 a95241bf-73f2-0310-859d-f6bbb57e9c96

headers/private/kernel/arch/vm.h

@@ -27,7 +27,8 @@ status_t arch_vm_init(struct kernel_args *args);
 status_t arch_vm_init_post_area(struct kernel_args *args);
 status_t arch_vm_init_end(struct kernel_args *args);
 status_t arch_vm_init_post_modules(struct kernel_args *args);
-void arch_vm_aspace_swap(struct vm_address_space *aspace);
+void arch_vm_aspace_swap(struct vm_address_space *from,
+	struct vm_address_space *to);
 bool arch_vm_supports_protection(uint32 protection);
 
 status_t arch_vm_set_memory_type(struct vm_area *area, addr_t physicalBase,

headers/private/kernel/arch/x86/arch_vm_translation_map.h

@@ -13,10 +13,11 @@
 
 struct page_directory_entry;
 
-
 typedef struct vm_translation_map_arch_info {
 	struct page_directory_entry *pgdir_virt;
 	struct page_directory_entry *pgdir_phys;
+	vint32 active_on_cpus;
+		// mask indicating on which CPUs the map is currently used
 	int num_invalidate_pages;
 	addr_t pages_to_invalidate[PAGE_INVALIDATE_CACHE_SIZE];
 } vm_translation_map_arch_info;

headers/private/kernel/vm_address_space.h

@@ -34,7 +34,7 @@ struct vm_address_space *vm_get_current_user_address_space(void);
 team_id vm_current_user_address_space_id(void);
 struct vm_address_space *vm_get_address_space(team_id team);
 void vm_put_address_space(struct vm_address_space *aspace);
-#define vm_swap_address_space(aspace) arch_vm_aspace_swap(aspace)
+#define vm_swap_address_space(from, to) arch_vm_aspace_swap(from, to)
 
 #ifdef __cplusplus
 }

src/system/kernel/arch/m68k/arch_vm.cpp

@@ -29,7 +29,7 @@
 
 #warning M68K: WRITEME
 
-status_t 
+status_t
 arch_vm_init(kernel_args *args)
 {
 	return B_OK;
@@ -62,7 +62,7 @@ arch_vm_init_end(kernel_args *args)
 #if 0
 	TRACE(("arch_vm_init_end(): %lu virtual ranges to keep:\n",
 		args->arch_args.num_virtual_ranges_to_keep));
-	
+
 	for (int i = 0; i < (int)args->arch_args.num_virtual_ranges_to_keep; i++) {
 		addr_range &range = args->arch_args.virtual_ranges_to_keep[i];
 
@@ -102,11 +102,10 @@ arch_vm_init_post_modules(kernel_args *args)
 }
 
 
-void 
-arch_vm_aspace_swap(vm_address_space *aspace)
+void
+arch_vm_aspace_swap(struct vm_address_space *from, struct vm_address_space *to)
 {
-	m68k_set_pgdir(m68k_translation_map_get_pgdir(
-		&aspace->translation_map));
+	m68k_set_pgdir(m68k_translation_map_get_pgdir(&to->translation_map));
 }
 
 

src/system/kernel/arch/ppc/arch_vm.cpp

@@ -24,7 +24,7 @@
 #endif
 
 
-status_t 
+status_t
 arch_vm_init(kernel_args *args)
 {
 	return B_OK;
@@ -105,7 +105,7 @@ arch_vm_init_end(kernel_args *args)
 {
 	TRACE(("arch_vm_init_end(): %lu virtual ranges to keep:\n",
 		args->arch_args.num_virtual_ranges_to_keep));
-	
+
 	for (int i = 0; i < (int)args->arch_args.num_virtual_ranges_to_keep; i++) {
 		addr_range &range = args->arch_args.virtual_ranges_to_keep[i];
 
@@ -143,8 +143,8 @@ arch_vm_init_post_modules(kernel_args *args)
 }
 
 
-void 
-arch_vm_aspace_swap(vm_address_space *aspace)
+void
+arch_vm_aspace_swap(struct vm_address_space *from, struct vm_address_space *to)
 {
 }
 

src/system/kernel/arch/x86/arch_thread.cpp

@@ -358,8 +358,21 @@ arch_thread_context_switch(struct thread *from, struct thread *to)
 
 	newPageDirectory = (addr_t)x86_next_page_directory(from, to);
 
-	if ((newPageDirectory % B_PAGE_SIZE) != 0)
-		panic("arch_thread_context_switch: bad pgdir 0x%lx\n", newPageDirectory);
+	ASSERT((newPageDirectory % B_PAGE_SIZE) == 0);
+
+	if (newPageDirectory != 0) {
+		// update on which CPUs the address space is used
+		int cpu = smp_get_current_cpu();
+		if (vm_address_space* addressSpace = from->team->address_space) {
+			atomic_and(&addressSpace->translation_map.arch_data->active_on_cpus,
+				~((uint32)1 << cpu));
+		}
+
+		if (vm_address_space* addressSpace = to->team->address_space) {
+			atomic_or(&addressSpace->translation_map.arch_data->active_on_cpus,
+				(uint32)1 << cpu);
+		}
+	}
 
 	gX86SwapFPUFunc(from->arch_info.fpu_state, to->arch_info.fpu_state);
 	i386_context_switch(&from->arch_info, &to->arch_info, newPageDirectory);

src/system/kernel/arch/x86/arch_vm.cpp

@@ -8,10 +8,15 @@
  */
 
 
+#include <stdlib.h>
+#include <string.h>
+
 #include <KernelExport.h>
+
 #include <smp.h>
 #include <util/AutoLock.h>
 #include <vm.h>
+#include <vm_address_space.h>
 #include <vm_page.h>
 #include <vm_priv.h>
 
@@ -21,9 +26,6 @@
 
 #include <arch/x86/bios.h>
 
-#include <stdlib.h>
-#include <string.h>
-
 
 //#define TRACE_ARCH_VM
 #ifdef TRACE_ARCH_VM
@@ -382,10 +384,21 @@ arch_vm_init_post_modules(kernel_args *args)
 
 
 void
-arch_vm_aspace_swap(vm_address_space *aspace)
+arch_vm_aspace_swap(struct vm_address_space *from, struct vm_address_space *to)
 {
+	int cpu = smp_get_current_cpu();
+	if (from != NULL) {
+		atomic_and(&from->translation_map.arch_data->active_on_cpus,
+			~((uint32)1 << cpu));
+	}
+
+	if (to != NULL && to != vm_kernel_address_space()) {
+		atomic_or(&to->translation_map.arch_data->active_on_cpus,
+			(uint32)1 << cpu);
+	}
+
 	i386_swap_pgdir((addr_t)i386_translation_map_get_pgdir(
-		&aspace->translation_map));
+		&to->translation_map));
 }
 
 

src/system/kernel/arch/x86/arch_vm_translation_map.cpp

@@ -662,12 +662,18 @@ flush_tmap(vm_translation_map *map)
 
 		if (IS_KERNEL_MAP(map)) {
 			arch_cpu_global_TLB_invalidate();
-			smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, NULL,
-				SMP_MSG_FLAG_SYNC);
+			smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0,
+				NULL, SMP_MSG_FLAG_SYNC);
 		} else {
 			arch_cpu_user_TLB_invalidate();
-			smp_send_broadcast_ici(SMP_MSG_USER_INVALIDATE_PAGES, 0, 0, 0, NULL,
-				SMP_MSG_FLAG_SYNC);
+
+			int cpu = smp_get_current_cpu();
+			uint32 cpuMask = map->arch_data->active_on_cpus
+				& ~((uint32)1 << cpu);
+			if (cpuMask != 0) {
+				smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
+					0, 0, 0, NULL, SMP_MSG_FLAG_SYNC);
+			}
 		}
 	} else {
 		TRACE(("flush_tmap: %d pages to invalidate, invalidate list\n",
@@ -675,10 +681,23 @@ flush_tmap(vm_translation_map *map)
 
 		arch_cpu_invalidate_TLB_list(map->arch_data->pages_to_invalidate,
 			map->arch_data->num_invalidate_pages);
-		smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
-			(uint32)map->arch_data->pages_to_invalidate,
-			map->arch_data->num_invalidate_pages, 0, NULL,
-			SMP_MSG_FLAG_SYNC);
+
+		if (IS_KERNEL_MAP(map)) {
+			smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST,
+				(uint32)map->arch_data->pages_to_invalidate,
+				map->arch_data->num_invalidate_pages, 0, NULL,
+				SMP_MSG_FLAG_SYNC);
+		} else {
+			int cpu = smp_get_current_cpu();
+			uint32 cpuMask = map->arch_data->active_on_cpus
+				& ~((uint32)1 << cpu);
+			if (cpuMask != 0) {
+				smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
+					(uint32)map->arch_data->pages_to_invalidate,
+					map->arch_data->num_invalidate_pages, 0, NULL,
+					SMP_MSG_FLAG_SYNC);
+			}
+		}
 	}
 	map->arch_data->num_invalidate_pages = 0;
 
@@ -777,6 +796,7 @@ arch_vm_translation_map_init_map(vm_translation_map *map, bool kernel)
 		return B_NO_MEMORY;
 	}
 
+	map->arch_data->active_on_cpus = 0;
 	map->arch_data->num_invalidate_pages = 0;
 
 	if (!kernel) {

src/system/kernel/thread.cpp

@@ -1441,7 +1441,7 @@ thread_exit(void)
 		RELEASE_TEAM_LOCK();
 
 		// swap address spaces, to make sure we're running on the kernel's pgdir
-		vm_swap_address_space(vm_kernel_address_space());
+		vm_swap_address_space(team->address_space, vm_kernel_address_space());
 		restore_interrupts(state);
 
 		TRACE(("thread_exit: thread %ld now a kernel thread!\n", thread->id));