From 9a42ad7a77f11cf1b857e84ec70d21b1afaa71cd Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Wed, 22 Oct 2008 15:12:32 +0000 Subject: [PATCH] When switching to a kernel thread we no longer set the page directory. This is not necessary, since userland teams' page directories also contain the kernel mappings, and avoids unnecessary TLB flushes. To make that possible the vm_translation_map_arch_info objects are reference counted now. This optimization reduces the kernel time of the Haiku build on my machine with SMP disabled a few percent, but interestingly the total time decreases only marginally. Haven't tested with SMP yet, but for full impact CPU affinity would be needed. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@28287 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/private/kernel/arch/x86/arch_cpu.h | 2 + src/system/kernel/arch/x86/arch_cpu.cpp | 11 ++ src/system/kernel/arch/x86/arch_thread.cpp | 51 ++++----- src/system/kernel/arch/x86/arch_vm.cpp | 21 ++-- .../arch/x86/arch_vm_translation_map.cpp | 107 +++++++++++------- src/system/kernel/arch/x86/x86_paging.h | 31 ++++- 6 files changed, 139 insertions(+), 84 deletions(-) diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index 65cb333a41..ac40cca3b8 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -238,6 +238,8 @@ typedef struct arch_cpu_info { int model; int extended_model; + struct vm_translation_map_arch_info* active_translation_map; + // local TSS for this cpu struct tss tss; struct tss double_fault_tss; diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 0a83938304..8a7070b182 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -22,12 +22,15 @@ #include #include #include +#include +#include #include #include #include #include "interrupts.h" +#include "x86_paging.h" #define DUMP_FEATURE_STRING 1 @@ -539,6 +542,10 @@ arch_cpu_init_post_vm(kernel_args *args) // (a fixed number of used GDT entries) //i386_selector_init(gGDT); // pass the new gdt + + vm_translation_map_arch_info* kernelArchTranslationMap + = vm_kernel_address_space()->translation_map.arch_data; + // setup task-state segments for (i = 0; i < args->num_cpus; i++) { // initialize the regular and double fault tss stored in the per-cpu @@ -553,6 +560,10 @@ arch_cpu_init_post_vm(kernel_args *args) // initialize the double fault tss init_double_fault(i); + + // init active translation map + gCPU[i].arch.active_translation_map = kernelArchTranslationMap; + kernelArchTranslationMap->AddReference(); } // set the current hardware task on cpu 0 diff --git a/src/system/kernel/arch/x86/arch_thread.cpp b/src/system/kernel/arch/x86/arch_thread.cpp index 7c8cbcaee6..fe9efb9b58 100644 --- a/src/system/kernel/arch/x86/arch_thread.cpp +++ b/src/system/kernel/arch/x86/arch_thread.cpp @@ -12,10 +12,12 @@ #include #include +#include #include #include #include #include +#include #include #include #include @@ -336,21 +338,6 @@ arch_thread_switch_kstack_and_call(struct thread *t, addr_t new_kstack, void arch_thread_context_switch(struct thread *from, struct thread *to) { - addr_t newPageDirectory; - -#if 0 - int i; - - dprintf("arch_thread_context_switch: cpu %d 0x%x -> 0x%x, aspace 0x%x -> 0x%x, old stack = 0x%x:0x%x, stack = 0x%x:0x%x\n", - smp_get_current_cpu(), t_from->id, t_to->id, - t_from->team->address_space, t_to->team->address_space, - t_from->arch_info.current_stack.ss, t_from->arch_info.current_stack.esp, - t_to->arch_info.current_stack.ss, t_to->arch_info.current_stack.esp); -#endif -#if 0 - for (i = 0; i < 11; i++) - dprintf("*esp[%d] (0x%x) = 0x%x\n", i, ((unsigned int *)new_at->esp + i), *((unsigned int *)new_at->esp + i)); -#endif i386_set_tss_and_kstack(to->kernel_stack_top); // set TLS GDT entry to the current thread - since this action is @@ -358,23 +345,29 @@ arch_thread_context_switch(struct thread *from, struct thread *to) if (to->user_local_storage != 0) set_tls_context(to); - newPageDirectory = (addr_t)x86_next_page_directory(from, to); + struct cpu_ent* cpuData = to->cpu; + vm_translation_map_arch_info* activeMap + = cpuData->arch.active_translation_map; + vm_address_space* toAddressSpace = to->team->address_space; - ASSERT((newPageDirectory % B_PAGE_SIZE) == 0); - - if (newPageDirectory != 0) { + addr_t newPageDirectory; + vm_translation_map_arch_info* toMap; + if (toAddressSpace != NULL + && (toMap = toAddressSpace->translation_map.arch_data) != activeMap) { // update on which CPUs the address space is used - int cpu = smp_get_current_cpu(); - if (vm_address_space* addressSpace = from->team->address_space) { - atomic_and(&addressSpace->translation_map.arch_data->active_on_cpus, - ~((uint32)1 << cpu)); - } + int cpu = cpuData->cpu_num; + atomic_and(&activeMap->active_on_cpus, ~((uint32)1 << cpu)); + atomic_or(&toMap->active_on_cpus, (uint32)1 << cpu); - if (vm_address_space* addressSpace = to->team->address_space) { - atomic_or(&addressSpace->translation_map.arch_data->active_on_cpus, - (uint32)1 << cpu); - } - } + // assign the new map to the CPU + activeMap->RemoveReference(); + toMap->AddReference(); + cpuData->arch.active_translation_map = toMap; + + // get the new page directory + newPageDirectory = (addr_t)toMap->pgdir_phys; + } else + newPageDirectory = 0; gX86SwapFPUFunc(from->arch_info.fpu_state, to->arch_info.fpu_state); i386_context_switch(&from->arch_info, &to->arch_info, newPageDirectory); diff --git a/src/system/kernel/arch/x86/arch_vm.cpp b/src/system/kernel/arch/x86/arch_vm.cpp index 66e67131b0..f1c42c441b 100644 --- a/src/system/kernel/arch/x86/arch_vm.cpp +++ b/src/system/kernel/arch/x86/arch_vm.cpp @@ -388,19 +388,14 @@ arch_vm_init_post_modules(kernel_args *args) void arch_vm_aspace_swap(struct vm_address_space *from, struct vm_address_space *to) { - int cpu = smp_get_current_cpu(); - if (from != NULL) { - atomic_and(&from->translation_map.arch_data->active_on_cpus, - ~((uint32)1 << cpu)); - } - - if (to != NULL && to != vm_kernel_address_space()) { - atomic_or(&to->translation_map.arch_data->active_on_cpus, - (uint32)1 << cpu); - } - - i386_swap_pgdir((addr_t)i386_translation_map_get_pgdir( - &to->translation_map)); + // This functions is only invoked when a userland thread is in the process + // of dying. It switches to the kernel team and does whatever cleanup is + // necessary (in case it is the team's main thread, it will delete the + // team). + // It is however not necessary to change the page directory. Userland team's + // page directories include all kernel mappings as well. Furthermore our + // arch specific translation map data objects are ref-counted, so they won't + // go away as long as they are still used on any CPU. } diff --git a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp index 998ced8e7c..f427c8185b 100644 --- a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp +++ b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp @@ -41,7 +41,33 @@ static page_directory_entry *sPageHolePageDir = NULL; static page_directory_entry *sKernelPhysicalPageDirectory = NULL; static page_directory_entry *sKernelVirtualPageDirectory = NULL; -static vm_translation_map *sTMapList; + +// Accessor class to reuse the SinglyLinkedListLink of DeferredDeletable for +// vm_translation_map_arch_info. +struct ArchTMapGetLink { +private: + typedef SinglyLinkedListLink Link; + +public: + inline Link* operator()(vm_translation_map_arch_info* element) const + { + return (Link*)element->GetSinglyLinkedListLink(); + } + + inline const Link* operator()( + const vm_translation_map_arch_info* element) const + { + return (const Link*)element->GetSinglyLinkedListLink(); + } + +}; + + +typedef SinglyLinkedList + ArchTMapList; + + +static ArchTMapList sTMapList; static spinlock sTMapListLock; #define CHATTY_TMAP 0 @@ -69,13 +95,13 @@ i386_translation_map_get_pgdir(vm_translation_map *map) void x86_update_all_pgdirs(int index, page_directory_entry e) { - vm_translation_map *entry; unsigned int state = disable_interrupts(); acquire_spinlock(&sTMapListLock); - for(entry = sTMapList; entry != NULL; entry = entry->next) - entry->arch_data->pgdir_virt[index] = e; + ArchTMapList::Iterator it = sTMapList.GetIterator(); + while (vm_translation_map_arch_info* info = it.Next()) + info->pgdir_virt[index] = e; release_spinlock(&sTMapListLock); restore_interrupts(state); @@ -143,45 +169,48 @@ unlock_tmap(vm_translation_map *map) } +vm_translation_map_arch_info::vm_translation_map_arch_info() + : + pgdir_virt(NULL), + ref_count(1) +{ +} + + +vm_translation_map_arch_info::~vm_translation_map_arch_info() +{ + // free the page dir + free(pgdir_virt); +} + + +void +vm_translation_map_arch_info::Delete() +{ + // remove from global list + InterruptsSpinLocker locker(sTMapListLock); + sTMapList.Remove(this); + locker.Unlock(); + + if (are_interrupts_enabled()) + delete this; + else + deferred_delete(this); +} + + static void destroy_tmap(vm_translation_map *map) { - int state; - vm_translation_map *entry; - vm_translation_map *last = NULL; - unsigned int i; - if (map == NULL) return; - // remove it from the tmap list - state = disable_interrupts(); - acquire_spinlock(&sTMapListLock); - - // TODO: How about using a doubly linked list? - entry = sTMapList; - while (entry != NULL) { - if (entry == map) { - if (last != NULL) - last->next = entry->next; - else - sTMapList = entry->next; - - break; - } - last = entry; - entry = entry->next; - } - - release_spinlock(&sTMapListLock); - restore_interrupts(state); - if (map->arch_data->page_mapper != NULL) map->arch_data->page_mapper->Delete(); if (map->arch_data->pgdir_virt != NULL) { // cycle through and free all of the user space pgtables - for (i = VADDR_TO_PDENT(USER_BASE); + for (uint32 i = VADDR_TO_PDENT(USER_BASE); i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) { addr_t pgtable_addr; vm_page *page; @@ -194,10 +223,10 @@ destroy_tmap(vm_translation_map *map) vm_page_set_state(page, PAGE_STATE_FREE); } } - free(map->arch_data->pgdir_virt); } - free(map->arch_data); + map->arch_data->RemoveReference(); + recursive_lock_destroy(&map->lock); } @@ -710,11 +739,10 @@ arch_vm_translation_map_init_map(vm_translation_map *map, bool kernel) CObjectDeleter lockDeleter(&map->lock, &recursive_lock_destroy); - map->arch_data = (vm_translation_map_arch_info*) - malloc(sizeof(vm_translation_map_arch_info)); + map->arch_data = new(std::nothrow) vm_translation_map_arch_info; if (map->arch_data == NULL) return B_NO_MEMORY; - MemoryDeleter archInfoDeleter(map->arch_data); + ObjectDeleter archInfoDeleter(map->arch_data); map->arch_data->active_on_cpus = 0; map->arch_data->num_invalidate_pages = 0; @@ -763,8 +791,7 @@ arch_vm_translation_map_init_map(vm_translation_map *map, bool kernel) sKernelVirtualPageDirectory + FIRST_KERNEL_PGDIR_ENT, NUM_KERNEL_PGDIR_ENTS * sizeof(page_directory_entry)); - map->next = sTMapList; - sTMapList = map; + sTMapList.Add(map->arch_data); release_spinlock(&sTMapListLock); restore_interrupts(state); @@ -805,7 +832,7 @@ arch_vm_translation_map_init(kernel_args *args) args->arch_args.vir_pgdir; B_INITIALIZE_SPINLOCK(&sTMapListLock); - sTMapList = NULL; + new (&sTMapList) ArchTMapList; // TODO: Select the best page mapper! large_memory_physical_page_ops_init(args, &tmap_ops); diff --git a/src/system/kernel/arch/x86/x86_paging.h b/src/system/kernel/arch/x86/x86_paging.h index 27b0c3cd7c..4d7d9f0e6c 100644 --- a/src/system/kernel/arch/x86/x86_paging.h +++ b/src/system/kernel/arch/x86/x86_paging.h @@ -10,6 +10,9 @@ #include +#include +#include + #define PAGE_INVALIDATE_CACHE_SIZE 64 @@ -52,15 +55,24 @@ typedef struct page_directory_entry { } page_directory_entry; -typedef struct vm_translation_map_arch_info { +struct vm_translation_map_arch_info : DeferredDeletable { struct page_directory_entry* pgdir_virt; struct page_directory_entry* pgdir_phys; TranslationMapPhysicalPageMapper* page_mapper; + vint32 ref_count; vint32 active_on_cpus; // mask indicating on which CPUs the map is currently used int num_invalidate_pages; addr_t pages_to_invalidate[PAGE_INVALIDATE_CACHE_SIZE]; -} vm_translation_map_arch_info; + + vm_translation_map_arch_info(); + virtual ~vm_translation_map_arch_info(); + + inline void AddReference(); + inline void RemoveReference(); + + void Delete(); +}; void x86_early_prepare_page_tables(page_table_entry* pageTables, addr_t address, @@ -100,4 +112,19 @@ update_page_table_entry(page_table_entry *entry, page_table_entry *with) } +inline void +vm_translation_map_arch_info::AddReference() +{ + atomic_add(&ref_count, 1); +} + + +inline void +vm_translation_map_arch_info::RemoveReference() +{ + if (atomic_add(&ref_count, -1) == 1) + Delete(); +} + + #endif // _KERNEL_ARCH_X86_PAGING_H