When switching to a kernel thread we no longer set the page directory.

This is not necessary, since userland teams' page directories also
contain the kernel mappings, and avoids unnecessary TLB flushes. To make
that possible the vm_translation_map_arch_info objects are reference
counted now.

This optimization reduces the kernel time of the Haiku build on my
machine with SMP disabled a few percent, but interestingly the total
time decreases only marginally. Haven't tested with SMP yet, but for
full impact CPU affinity would be needed.


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@28287 a95241bf-73f2-0310-859d-f6bbb57e9c96

headers/private/kernel/arch/x86/arch_cpu.h

@@ -238,6 +238,8 @@ typedef struct arch_cpu_info {
 	int model;
 	int extended_model;
 
+	struct vm_translation_map_arch_info* active_translation_map;
+
 	// local TSS for this cpu
 	struct tss tss;
 	struct tss double_fault_tss;

src/system/kernel/arch/x86/arch_cpu.cpp

@@ -22,12 +22,15 @@
 #include <smp.h>
 #include <tls.h>
 #include <vm.h>
+#include <vm_address_space.h>
+#include <vm_types.h>
 
 #include <arch_system_info.h>
 #include <arch/x86/selector.h>
 #include <boot/kernel_args.h>
 
 #include "interrupts.h"
+#include "x86_paging.h"
 
 
 #define DUMP_FEATURE_STRING 1
@@ -539,6 +542,10 @@ arch_cpu_init_post_vm(kernel_args *args)
 	// (a fixed number of used GDT entries)
 	//i386_selector_init(gGDT);  // pass the new gdt
 
+
+	vm_translation_map_arch_info* kernelArchTranslationMap
+		= vm_kernel_address_space()->translation_map.arch_data;
+
 	// setup task-state segments
 	for (i = 0; i < args->num_cpus; i++) {
 		// initialize the regular and double fault tss stored in the per-cpu
@@ -553,6 +560,10 @@ arch_cpu_init_post_vm(kernel_args *args)
 
 		// initialize the double fault tss
 		init_double_fault(i);
+
+		// init active translation map
+		gCPU[i].arch.active_translation_map = kernelArchTranslationMap;
+		kernelArchTranslationMap->AddReference();
 	}
 
 	// set the current hardware task on cpu 0

src/system/kernel/arch/x86/arch_thread.cpp

@@ -12,10 +12,12 @@
 
 #include <arch/user_debugger.h>
 #include <arch_cpu.h>
+#include <cpu.h>
 #include <debug.h>
 #include <kernel.h>
 #include <ksignal.h>
 #include <int.h>
+#include <team.h>
 #include <thread.h>
 #include <tls.h>
 #include <tracing.h>
@@ -336,21 +338,6 @@ arch_thread_switch_kstack_and_call(struct thread *t, addr_t new_kstack,
 void
 arch_thread_context_switch(struct thread *from, struct thread *to)
 {
-	addr_t newPageDirectory;
-
-#if 0
-	int i;
-
-	dprintf("arch_thread_context_switch: cpu %d 0x%x -> 0x%x, aspace 0x%x -> 0x%x, old stack = 0x%x:0x%x, stack = 0x%x:0x%x\n",
-		smp_get_current_cpu(), t_from->id, t_to->id,
-		t_from->team->address_space, t_to->team->address_space,
-		t_from->arch_info.current_stack.ss, t_from->arch_info.current_stack.esp,
-		t_to->arch_info.current_stack.ss, t_to->arch_info.current_stack.esp);
-#endif
-#if 0
-	for (i = 0; i < 11; i++)
-		dprintf("*esp[%d] (0x%x) = 0x%x\n", i, ((unsigned int *)new_at->esp + i), *((unsigned int *)new_at->esp + i));
-#endif
 	i386_set_tss_and_kstack(to->kernel_stack_top);
 
 	// set TLS GDT entry to the current thread - since this action is
@@ -358,23 +345,29 @@ arch_thread_context_switch(struct thread *from, struct thread *to)
 	if (to->user_local_storage != 0)
 		set_tls_context(to);
 
-	newPageDirectory = (addr_t)x86_next_page_directory(from, to);
+	struct cpu_ent* cpuData = to->cpu;
+	vm_translation_map_arch_info* activeMap
+		= cpuData->arch.active_translation_map;
+	vm_address_space* toAddressSpace = to->team->address_space;
 
-	ASSERT((newPageDirectory % B_PAGE_SIZE) == 0);
+	addr_t newPageDirectory;
-
+	vm_translation_map_arch_info* toMap;
-	if (newPageDirectory != 0) {
+	if (toAddressSpace != NULL
+		&& (toMap = toAddressSpace->translation_map.arch_data) != activeMap) {
 		// update on which CPUs the address space is used
-		int cpu = smp_get_current_cpu();
+		int cpu = cpuData->cpu_num;
-		if (vm_address_space* addressSpace = from->team->address_space) {
+		atomic_and(&activeMap->active_on_cpus, ~((uint32)1 << cpu));
-			atomic_and(&addressSpace->translation_map.arch_data->active_on_cpus,
+		atomic_or(&toMap->active_on_cpus, (uint32)1 << cpu);
-				~((uint32)1 << cpu));
-		}
 
-		if (vm_address_space* addressSpace = to->team->address_space) {
+		// assign the new map to the CPU
-			atomic_or(&addressSpace->translation_map.arch_data->active_on_cpus,
+		activeMap->RemoveReference();
-				(uint32)1 << cpu);
+		toMap->AddReference();
-		}
+		cpuData->arch.active_translation_map = toMap;
-	}
+
+		// get the new page directory
+		newPageDirectory = (addr_t)toMap->pgdir_phys;
+	} else
+		newPageDirectory = 0;
 
 	gX86SwapFPUFunc(from->arch_info.fpu_state, to->arch_info.fpu_state);
 	i386_context_switch(&from->arch_info, &to->arch_info, newPageDirectory);

src/system/kernel/arch/x86/arch_vm.cpp

@@ -388,19 +388,14 @@ arch_vm_init_post_modules(kernel_args *args)
 void
 arch_vm_aspace_swap(struct vm_address_space *from, struct vm_address_space *to)
 {
-	int cpu = smp_get_current_cpu();
+	// This functions is only invoked when a userland thread is in the process
-	if (from != NULL) {
+	// of dying. It switches to the kernel team and does whatever cleanup is
-		atomic_and(&from->translation_map.arch_data->active_on_cpus,
+	// necessary (in case it is the team's main thread, it will delete the
-			~((uint32)1 << cpu));
+	// team).
-	}
+	// It is however not necessary to change the page directory. Userland team's
-
+	// page directories include all kernel mappings as well. Furthermore our
-	if (to != NULL && to != vm_kernel_address_space()) {
+	// arch specific translation map data objects are ref-counted, so they won't
-		atomic_or(&to->translation_map.arch_data->active_on_cpus,
+	// go away as long as they are still used on any CPU.
-			(uint32)1 << cpu);
-	}
-
-	i386_swap_pgdir((addr_t)i386_translation_map_get_pgdir(
-		&to->translation_map));
 }
 
 

src/system/kernel/arch/x86/arch_vm_translation_map.cpp

@@ -41,7 +41,33 @@ static page_directory_entry *sPageHolePageDir = NULL;
 static page_directory_entry *sKernelPhysicalPageDirectory = NULL;
 static page_directory_entry *sKernelVirtualPageDirectory = NULL;
 
-static vm_translation_map *sTMapList;
+
+// Accessor class to reuse the SinglyLinkedListLink of DeferredDeletable for
+// vm_translation_map_arch_info.
+struct ArchTMapGetLink {
+private:
+	typedef SinglyLinkedListLink<vm_translation_map_arch_info> Link;
+
+public:
+	inline Link* operator()(vm_translation_map_arch_info* element) const
+	{
+		return (Link*)element->GetSinglyLinkedListLink();
+	}
+
+	inline const Link* operator()(
+		const vm_translation_map_arch_info* element) const
+	{
+		return (const Link*)element->GetSinglyLinkedListLink();
+	}
+
+};
+
+
+typedef SinglyLinkedList<vm_translation_map_arch_info, ArchTMapGetLink>
+	ArchTMapList;
+
+
+static ArchTMapList sTMapList;
 static spinlock sTMapListLock;
 
 #define CHATTY_TMAP 0
@@ -69,13 +95,13 @@ i386_translation_map_get_pgdir(vm_translation_map *map)
 void
 x86_update_all_pgdirs(int index, page_directory_entry e)
 {
-	vm_translation_map *entry;
 	unsigned int state = disable_interrupts();
 
 	acquire_spinlock(&sTMapListLock);
 
-	for(entry = sTMapList; entry != NULL; entry = entry->next)
+	ArchTMapList::Iterator it = sTMapList.GetIterator();
-		entry->arch_data->pgdir_virt[index] = e;
+	while (vm_translation_map_arch_info* info = it.Next())
+		info->pgdir_virt[index] = e;
 
 	release_spinlock(&sTMapListLock);
 	restore_interrupts(state);
@@ -143,45 +169,48 @@ unlock_tmap(vm_translation_map *map)
 }
 
 
+vm_translation_map_arch_info::vm_translation_map_arch_info()
+	:
+	pgdir_virt(NULL),
+	ref_count(1)
+{
+}
+
+
+vm_translation_map_arch_info::~vm_translation_map_arch_info()
+{
+	// free the page dir
+	free(pgdir_virt);
+}
+
+
+void
+vm_translation_map_arch_info::Delete()
+{
+	// remove from global list
+	InterruptsSpinLocker locker(sTMapListLock);
+	sTMapList.Remove(this);
+	locker.Unlock();
+
+	if (are_interrupts_enabled())
+		delete this;
+	else
+		deferred_delete(this);
+}
+
+
 static void
 destroy_tmap(vm_translation_map *map)
 {
-	int state;
-	vm_translation_map *entry;
-	vm_translation_map *last = NULL;
-	unsigned int i;
-
 	if (map == NULL)
 		return;
 
-	// remove it from the tmap list
-	state = disable_interrupts();
-	acquire_spinlock(&sTMapListLock);
-
-	// TODO: How about using a doubly linked list?
-	entry = sTMapList;
-	while (entry != NULL) {
-		if (entry == map) {
-			if (last != NULL)
-				last->next = entry->next;
-			else
-				sTMapList = entry->next;
-
-			break;
-		}
-		last = entry;
-		entry = entry->next;
-	}
-
-	release_spinlock(&sTMapListLock);
-	restore_interrupts(state);
-
 	if (map->arch_data->page_mapper != NULL)
 		map->arch_data->page_mapper->Delete();
 
 	if (map->arch_data->pgdir_virt != NULL) {
 		// cycle through and free all of the user space pgtables
-		for (i = VADDR_TO_PDENT(USER_BASE);
+		for (uint32 i = VADDR_TO_PDENT(USER_BASE);
 				i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) {
 			addr_t pgtable_addr;
 			vm_page *page;
@@ -194,10 +223,10 @@ destroy_tmap(vm_translation_map *map)
 				vm_page_set_state(page, PAGE_STATE_FREE);
 			}
 		}
-		free(map->arch_data->pgdir_virt);
 	}
 
-	free(map->arch_data);
+	map->arch_data->RemoveReference();
+
 	recursive_lock_destroy(&map->lock);
 }
 
@@ -710,11 +739,10 @@ arch_vm_translation_map_init_map(vm_translation_map *map, bool kernel)
 	CObjectDeleter<recursive_lock> lockDeleter(&map->lock,
 		&recursive_lock_destroy);
 
-	map->arch_data = (vm_translation_map_arch_info*)
+	map->arch_data = new(std::nothrow) vm_translation_map_arch_info;
-		malloc(sizeof(vm_translation_map_arch_info));
 	if (map->arch_data == NULL)
 		return B_NO_MEMORY;
-	MemoryDeleter archInfoDeleter(map->arch_data);
+	ObjectDeleter<vm_translation_map_arch_info> archInfoDeleter(map->arch_data);
 
 	map->arch_data->active_on_cpus = 0;
 	map->arch_data->num_invalidate_pages = 0;
@@ -763,8 +791,7 @@ arch_vm_translation_map_init_map(vm_translation_map *map, bool kernel)
 			sKernelVirtualPageDirectory + FIRST_KERNEL_PGDIR_ENT,
 			NUM_KERNEL_PGDIR_ENTS * sizeof(page_directory_entry));
 
-		map->next = sTMapList;
+		sTMapList.Add(map->arch_data);
-		sTMapList = map;
 
 		release_spinlock(&sTMapListLock);
 		restore_interrupts(state);
@@ -805,7 +832,7 @@ arch_vm_translation_map_init(kernel_args *args)
 		args->arch_args.vir_pgdir;
 
 	B_INITIALIZE_SPINLOCK(&sTMapListLock);
-	sTMapList = NULL;
+	new (&sTMapList) ArchTMapList;
 
 // TODO: Select the best page mapper!
 	large_memory_physical_page_ops_init(args, &tmap_ops);

src/system/kernel/arch/x86/x86_paging.h

@@ -10,6 +10,9 @@
 
 #include <SupportDefs.h>
 
+#include <heap.h>
+#include <int.h>
+
 
 #define PAGE_INVALIDATE_CACHE_SIZE 64
 
@@ -52,15 +55,24 @@ typedef struct page_directory_entry {
 } page_directory_entry;
 
 
-typedef struct vm_translation_map_arch_info {
+struct vm_translation_map_arch_info : DeferredDeletable {
 	struct page_directory_entry*		pgdir_virt;
 	struct page_directory_entry*		pgdir_phys;
 	TranslationMapPhysicalPageMapper*	page_mapper;
+	vint32 ref_count;
 	vint32 active_on_cpus;
 		// mask indicating on which CPUs the map is currently used
 	int num_invalidate_pages;
 	addr_t pages_to_invalidate[PAGE_INVALIDATE_CACHE_SIZE];
-} vm_translation_map_arch_info;
+
+								vm_translation_map_arch_info();
+	virtual						~vm_translation_map_arch_info();
+
+	inline	void				AddReference();
+	inline	void				RemoveReference();
+
+			void				Delete();
+};
 
 
 void x86_early_prepare_page_tables(page_table_entry* pageTables, addr_t address,
@@ -100,4 +112,19 @@ update_page_table_entry(page_table_entry *entry, page_table_entry *with)
 }
 
 
+inline void
+vm_translation_map_arch_info::AddReference()
+{
+	atomic_add(&ref_count, 1);
+}
+
+
+inline void
+vm_translation_map_arch_info::RemoveReference()
+{
+	if (atomic_add(&ref_count, -1) == 1)
+		Delete();
+}
+
+
 #endif	// _KERNEL_ARCH_X86_PAGING_H