From e0e9a3e69f18220cf21cd2fa8ba2d9fb794b922b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Axel=20D=C3=B6rfler?= Date: Wed, 14 Dec 2005 17:07:37 +0000 Subject: [PATCH] * We now support the global page feature of x86 processors that prevents kernel TLBs from being flushed on context switch. * new arch_cpu_user_TLB_invalidate() that now does what arch_cpu_global_TLB_invalidate() did before. * arch_cpu_global_TLB_invalidate() will now flush all TLBs, even those from the kernel. * some cleanups. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@15535 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/private/kernel/arch/cpu.h | 10 ++- headers/private/kernel/arch/x86/arch_cpu.h | 5 ++ headers/private/kernel/smp.h | 7 +- src/system/kernel/arch/x86/arch_cpu.c | 15 ++++ .../kernel/arch/x86/arch_vm_translation_map.c | 89 ++++++++++++++----- src/system/kernel/arch/x86/arch_x86.S | 15 +++- src/system/kernel/smp.c | 12 ++- 7 files changed, 117 insertions(+), 36 deletions(-) diff --git a/headers/private/kernel/arch/cpu.h b/headers/private/kernel/arch/cpu.h index 1d764af843..ecab79391a 100644 --- a/headers/private/kernel/arch/cpu.h +++ b/headers/private/kernel/arch/cpu.h @@ -27,11 +27,15 @@ status_t arch_cpu_shutdown(bool reboot); void arch_cpu_invalidate_TLB_range(addr_t start, addr_t end); void arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages); +void arch_cpu_user_TLB_invalidate(void); void arch_cpu_global_TLB_invalidate(void); -status_t arch_cpu_user_memcpy(void *to, const void *from, size_t size, addr_t *faultHandler); -ssize_t arch_cpu_user_strlcpy(char *to, const char *from, size_t size, addr_t *faultHandler); -status_t arch_cpu_user_memset(void *s, char c, size_t count, addr_t *faultHandler); +status_t arch_cpu_user_memcpy(void *to, const void *from, size_t size, + addr_t *faultHandler); +ssize_t arch_cpu_user_strlcpy(char *to, const char *from, size_t size, + addr_t *faultHandler); +status_t arch_cpu_user_memset(void *s, char c, size_t count, + addr_t *faultHandler); void arch_cpu_idle(void); void arch_cpu_sync_icache(void *address, size_t length); diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index adf22e6d32..f8600a3104 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -21,6 +21,9 @@ #define IA32_MSR_MTRR_PHYSICAL_BASE_0 0x200 #define IA32_MSR_MTRR_PHYSICAL_MASK_0 0x201 +// cr4 flags +#define IA32_CR4_GLOBAL_PAGES (1UL << 7) + // Memory type ranges #define IA32_MTR_UNCACHED 0 #define IA32_MTR_WRITE_COMBINED 1 @@ -104,6 +107,8 @@ void i386_fxsave_swap(void *old_fpu_state, const void *new_fpu_state); uint32 x86_read_ebp(); uint32 x86_read_cr0(); void x86_write_cr0(uint32 value); +uint32 x86_read_cr4(); +void x86_write_cr4(uint32 value); uint64 x86_read_msr(uint32 registerNumber); void x86_write_msr(uint32 registerNumber, uint64 value); void x86_set_task_gate(int32 n, int32 segment); diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index ea88a93639..5341897db8 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -16,9 +16,10 @@ struct kernel_args; // intercpu messages enum { - SMP_MSG_INVL_PAGE_RANGE = 0, - SMP_MSG_INVL_PAGE_LIST, - SMP_MSG_GLOBAL_INVL_PAGE, + SMP_MSG_INVALIDATE_PAGE_RANGE = 0, + SMP_MSG_INVALIDATE_PAGE_LIST, + SMP_MSG_USER_INVALIDATE_PAGES, + SMP_MSG_GLOBAL_INVALIDATE_PAGES, SMP_MSG_RESCHEDULE, SMP_MSG_CPU_HALT, SMP_MSG_CALL_FUNCTION, diff --git a/src/system/kernel/arch/x86/arch_cpu.c b/src/system/kernel/arch/x86/arch_cpu.c index be99a2e432..99a3f0eb04 100644 --- a/src/system/kernel/arch/x86/arch_cpu.c +++ b/src/system/kernel/arch/x86/arch_cpu.c @@ -347,6 +347,21 @@ i386_set_tss_and_kstack(addr_t kstack) } +void +arch_cpu_global_TLB_invalidate(void) +{ + uint32 flags = x86_read_cr4(); + + if (flags & IA32_CR4_GLOBAL_PAGES) { + // disable and reenable the global pages to flush all TLBs regardless + // of the global page bit + x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES); + x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES); + } else + arch_cpu_user_TLB_invalidate(); +} + + void arch_cpu_invalidate_TLB_range(addr_t start, addr_t end) { diff --git a/src/system/kernel/arch/x86/arch_vm_translation_map.c b/src/system/kernel/arch/x86/arch_vm_translation_map.c index 35b4f16670..56e128ae2f 100644 --- a/src/system/kernel/arch/x86/arch_vm_translation_map.c +++ b/src/system/kernel/arch/x86/arch_vm_translation_map.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,8 @@ # define TRACE(x) ; #endif +#define IA32_GLOBAL_PAGE_FEATURE (1UL << 13) + // 256 MB of iospace #define IOSPACE_SIZE (256*1024*1024) // put it 256 MB into kernel space @@ -89,8 +92,8 @@ typedef struct vm_translation_map_arch_info { static page_table_entry *page_hole = NULL; static page_directory_entry *page_hole_pgdir = NULL; -static page_directory_entry *kernel_pgdir_phys = NULL; -static page_directory_entry *kernel_pgdir_virt = NULL; +static page_directory_entry *sKernelPhysicalPageDirectory = NULL; +static page_directory_entry *sKernelVirtualPageDirectory = NULL; static vm_translation_map *tmap_list; static spinlock tmap_list_lock; @@ -107,6 +110,7 @@ static spinlock tmap_list_lock; #define NUM_USER_PGDIR_ENTS (VADDR_TO_PDENT(ROUNDUP(USER_SIZE, B_PAGE_SIZE * 1024))) #define FIRST_KERNEL_PGDIR_ENT (VADDR_TO_PDENT(KERNEL_BASE)) #define NUM_KERNEL_PGDIR_ENTS (VADDR_TO_PDENT(KERNEL_SIZE)) +#define IS_KERNEL_MAP(map) (map->arch_data->pgdir_phys == sKernelPhysicalPageDirectory) static status_t early_query(addr_t va, addr_t *out_physical); static status_t get_physical_page_tmap(addr_t pa, addr_t *va, uint32 flags); @@ -300,7 +304,7 @@ put_pgtable_in_pgdir(page_directory_entry *entry, static void put_page_table_entry_in_pgtable(page_table_entry *entry, - addr_t physicalAddress, uint32 attributes) + addr_t physicalAddress, uint32 attributes, bool globalPage) { page_table_entry page; init_page_table_entry(&page); @@ -317,6 +321,9 @@ put_page_table_entry_in_pgtable(page_table_entry *entry, page.rw = (attributes & B_KERNEL_WRITE_AREA) != 0; page.present = 1; + if (globalPage) + page.global = 1; + // put it in the page table update_page_table_entry(entry, &page); } @@ -363,7 +370,8 @@ map_tmap(vm_translation_map *map, addr_t va, addr_t pa, uint32 attributes) | (attributes & B_USER_PROTECTION ? B_WRITE_AREA : B_KERNEL_WRITE_AREA)); // update any other page directories, if it maps kernel space - if (index >= FIRST_KERNEL_PGDIR_ENT && index < (FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS)) + if (index >= FIRST_KERNEL_PGDIR_ENT + && index < (FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS)) _update_all_pgdirs(index, pd[index]); map->map_count++; @@ -376,7 +384,8 @@ map_tmap(vm_translation_map *map, addr_t va, addr_t pa, uint32 attributes) } while (err < 0); index = VADDR_TO_PTENT(va); - put_page_table_entry_in_pgtable(&pt[index], pa, attributes); + put_page_table_entry_in_pgtable(&pt[index], pa, attributes, + IS_KERNEL_MAP(map)); put_physical_page_tmap((addr_t)pt); @@ -593,24 +602,40 @@ clear_flags_tmap(vm_translation_map *map, addr_t va, uint32 flags) static void flush_tmap(vm_translation_map *map) { - int state; + cpu_status state; if (map->arch_data->num_invalidate_pages <= 0) return; state = disable_interrupts(); + if (map->arch_data->num_invalidate_pages > PAGE_INVALIDATE_CACHE_SIZE) { // invalidate all pages - TRACE(("flush_tmap: %d pages to invalidate, doing global invalidation\n", map->arch_data->num_invalidate_pages)); - arch_cpu_global_TLB_invalidate(); - smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVL_PAGE, 0, 0, 0, NULL, SMP_MSG_FLAG_SYNC); + TRACE(("flush_tmap: %d pages to invalidate, invalidate all\n", + map->arch_data->num_invalidate_pages)); + + if (IS_KERNEL_MAP(map)) { + arch_cpu_global_TLB_invalidate(); + smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, NULL, + SMP_MSG_FLAG_SYNC); + } else { + arch_cpu_user_TLB_invalidate(); + smp_send_broadcast_ici(SMP_MSG_USER_INVALIDATE_PAGES, 0, 0, 0, NULL, + SMP_MSG_FLAG_SYNC); + } } else { - TRACE(("flush_tmap: %d pages to invalidate, doing local invalidation\n", map->arch_data->num_invalidate_pages)); - arch_cpu_invalidate_TLB_list(map->arch_data->pages_to_invalidate, map->arch_data->num_invalidate_pages); - smp_send_broadcast_ici(SMP_MSG_INVL_PAGE_LIST, (unsigned long)map->arch_data->pages_to_invalidate, - map->arch_data->num_invalidate_pages, 0, NULL, SMP_MSG_FLAG_SYNC); + TRACE(("flush_tmap: %d pages to invalidate, invalidate list\n", + map->arch_data->num_invalidate_pages)); + + arch_cpu_invalidate_TLB_list(map->arch_data->pages_to_invalidate, + map->arch_data->num_invalidate_pages); + smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_LIST, + (uint32)map->arch_data->pages_to_invalidate, + map->arch_data->num_invalidate_pages, 0, NULL, + SMP_MSG_FLAG_SYNC); } map->arch_data->num_invalidate_pages = 0; + restore_interrupts(state); } @@ -636,11 +661,13 @@ map_iospace_chunk(addr_t va, addr_t pa) pt[i].user = 0; pt[i].rw = 1; pt[i].present = 1; + pt[i].global = 1; } state = disable_interrupts(); arch_cpu_invalidate_TLB_range(va, va + (IOSPACE_CHUNK_SIZE - B_PAGE_SIZE)); - smp_send_broadcast_ici(SMP_MSG_INVL_PAGE_RANGE, va, va + (IOSPACE_CHUNK_SIZE - B_PAGE_SIZE), 0, + smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE, + va, va + (IOSPACE_CHUNK_SIZE - B_PAGE_SIZE), 0, NULL, SMP_MSG_FLAG_SYNC); restore_interrupts(state); @@ -808,16 +835,18 @@ arch_vm_translation_map_init_map(vm_translation_map *map, bool kernel) recursive_lock_destroy(&map->lock); return B_NO_MEMORY; } - vm_get_page_mapping(vm_get_kernel_aspace_id(), (addr_t)map->arch_data->pgdir_virt, (addr_t *)&map->arch_data->pgdir_phys); + vm_get_page_mapping(vm_get_kernel_aspace_id(), + (addr_t)map->arch_data->pgdir_virt, (addr_t *)&map->arch_data->pgdir_phys); } else { // kernel // we already know the kernel pgdir mapping - map->arch_data->pgdir_virt = kernel_pgdir_virt; - map->arch_data->pgdir_phys = kernel_pgdir_phys; + map->arch_data->pgdir_virt = sKernelVirtualPageDirectory; + map->arch_data->pgdir_phys = sKernelPhysicalPageDirectory; } // zero out the bottom portion of the new pgdir - memset(map->arch_data->pgdir_virt + FIRST_USER_PGDIR_ENT, 0, NUM_USER_PGDIR_ENTS * sizeof(page_directory_entry)); + memset(map->arch_data->pgdir_virt + FIRST_USER_PGDIR_ENT, 0, + NUM_USER_PGDIR_ENTS * sizeof(page_directory_entry)); // insert this new map into the map list { @@ -825,7 +854,8 @@ arch_vm_translation_map_init_map(vm_translation_map *map, bool kernel) acquire_spinlock(&tmap_list_lock); // copy the top portion of the pgdir from the current one - memcpy(map->arch_data->pgdir_virt + FIRST_KERNEL_PGDIR_ENT, kernel_pgdir_virt + FIRST_KERNEL_PGDIR_ENT, + memcpy(map->arch_data->pgdir_virt + FIRST_KERNEL_PGDIR_ENT, + sKernelVirtualPageDirectory + FIRST_KERNEL_PGDIR_ENT, NUM_KERNEL_PGDIR_ENTS * sizeof(page_directory_entry)); map->next = tmap_list; @@ -852,6 +882,8 @@ arch_vm_translation_map_init_kernel_map_post_sem(vm_translation_map *map) status_t arch_vm_translation_map_init(kernel_args *args) { + cpuid_info info; + TRACE(("vm_translation_map_init: entry\n")); // page hole set up in stage2 @@ -861,8 +893,8 @@ arch_vm_translation_map_init(kernel_args *args) // clear out the bottom 2 GB, unmap everything memset(page_hole_pgdir + FIRST_USER_PGDIR_ENT, 0, sizeof(page_directory_entry) * NUM_USER_PGDIR_ENTS); - kernel_pgdir_phys = (page_directory_entry *)args->arch_args.phys_pgdir; - kernel_pgdir_virt = (page_directory_entry *)args->arch_args.vir_pgdir; + sKernelPhysicalPageDirectory = (page_directory_entry *)args->arch_args.phys_pgdir; + sKernelVirtualPageDirectory = (page_directory_entry *)args->arch_args.vir_pgdir; tmap_list_lock = 0; tmap_list = NULL; @@ -907,6 +939,13 @@ arch_vm_translation_map_init(kernel_args *args) } } + // enable global page feature if available + get_current_cpuid(&info, 1); + if (info.eax_1.features & IA32_GLOBAL_PAGE_FEATURE) { + // this prevents kernel pages from being flushed from TLB on context-switch + x86_write_cr4(x86_read_cr4() | IA32_CR4_GLOBAL_PAGES); + } + TRACE(("vm_translation_map_init: done\n")); return B_OK; @@ -933,11 +972,11 @@ arch_vm_translation_map_init_post_area(kernel_args *args) TRACE(("vm_translation_map_init_post_area: entry\n")); // unmap the page hole hack we were using before - kernel_pgdir_virt[1023].present = 0; + sKernelVirtualPageDirectory[1023].present = 0; page_hole_pgdir = NULL; page_hole = NULL; - temp = (void *)kernel_pgdir_virt; + temp = (void *)sKernelVirtualPageDirectory; create_area("kernel_pgdir", &temp, B_EXACT_ADDRESS, B_PAGE_SIZE, B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); @@ -1000,8 +1039,10 @@ arch_vm_translation_map_early_map(kernel_args *args, addr_t va, addr_t pa, // zero it out in it's new mapping memset((unsigned int *)((unsigned int)page_hole + (va / B_PAGE_SIZE / 1024) * B_PAGE_SIZE), 0, B_PAGE_SIZE); } + // now, fill in the pentry - put_page_table_entry_in_pgtable(page_hole + va / B_PAGE_SIZE, pa, attributes); + put_page_table_entry_in_pgtable(page_hole + va / B_PAGE_SIZE, pa, attributes, + IS_KERNEL_ADDRESS(va)); arch_cpu_invalidate_TLB_range(va, va); diff --git a/src/system/kernel/arch/x86/arch_x86.S b/src/system/kernel/arch/x86/arch_x86.S index 25e4b0646c..803b168bf2 100644 --- a/src/system/kernel/arch/x86/arch_x86.S +++ b/src/system/kernel/arch/x86/arch_x86.S @@ -15,8 +15,8 @@ .text -/* void arch_cpu_global_TLB_invalidate(); */ -FUNCTION(arch_cpu_global_TLB_invalidate): +/* void arch_cpu_user_TLB_invalidate(); */ +FUNCTION(arch_cpu_user_TLB_invalidate): movl %cr3,%eax movl %eax,%cr3 ret @@ -77,6 +77,17 @@ FUNCTION(x86_write_cr0): movl %eax, %cr0 ret +/* uint32 x86_read_cr4(); */ +FUNCTION(x86_read_cr4): + movl %cr4, %eax + ret + +/* void x86_write_cr4(uint32 value); */ +FUNCTION(x86_write_cr4): + movl 4(%esp), %eax + movl %eax, %cr4 + ret + /* uint64 x86_read_msr(uint32 register); */ FUNCTION(x86_read_msr): movl 4(%esp), %ecx diff --git a/src/system/kernel/smp.c b/src/system/kernel/smp.c index 62f963f4fb..c438cd851a 100644 --- a/src/system/kernel/smp.c +++ b/src/system/kernel/smp.c @@ -358,13 +358,16 @@ process_pending_ici(int32 currentCPU) TRACE((" cpu %d message = %d\n", curr_cpu, msg->message)); switch (msg->message) { - case SMP_MSG_INVL_PAGE_RANGE: + case SMP_MSG_INVALIDATE_PAGE_RANGE: arch_cpu_invalidate_TLB_range((addr_t)msg->data, (addr_t)msg->data2); break; - case SMP_MSG_INVL_PAGE_LIST: + case SMP_MSG_INVALIDATE_PAGE_LIST: arch_cpu_invalidate_TLB_list((addr_t *)msg->data, (int)msg->data2); break; - case SMP_MSG_GLOBAL_INVL_PAGE: + case SMP_MSG_USER_INVALIDATE_PAGES: + arch_cpu_user_TLB_invalidate(); + break; + case SMP_MSG_GLOBAL_INVALIDATE_PAGES: arch_cpu_global_TLB_invalidate(); break; case SMP_MSG_RESCHEDULE: @@ -571,7 +574,8 @@ smp_wake_up_non_boot_cpus() // invalidate all of the other processors' TLB caches arch_cpu_global_TLB_invalidate(); - smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVL_PAGE, 0, 0, 0, NULL, SMP_MSG_FLAG_SYNC); + smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, NULL, + SMP_MSG_FLAG_SYNC); // start the other processors smp_send_broadcast_ici(SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, SMP_MSG_FLAG_ASYNC);