From badc7b674eca3e33d1c5f865a626966110790f8f Mon Sep 17 00:00:00 2001 From: Travis Geiselbrecht Date: Mon, 19 Feb 2007 00:11:24 +0000 Subject: [PATCH] yet another fix for #1018, which has at this point blossomed into a reorg of how AP cpus are initialized. the new cpuid stuff was apparently exacerbating an existing problem where various bits of low level cpu code (specifically get_current_cpu) weren't really initialized before being used. Changed the order to set up a fake set of threads to point each cpu at really early in boot to make sure that at all points in code it can get the current 'thread' and thus the current cpu. A probably better solution would be to have dr3 point to the current cpu which would then point to the current thread, but that has a race condition that would require an int disable, etc. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@20160 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/build/os/drivers/KernelExport.h | 1 + headers/os/drivers/KernelExport.h | 1 + headers/private/kernel/arch/cpu.h | 2 +- headers/private/kernel/cpu.h | 2 +- headers/private/kernel/thread.h | 2 +- src/system/kernel/arch/ppc/arch_cpu.cpp | 2 +- src/system/kernel/arch/x86/arch_cpu.c | 8 +-- src/system/kernel/cpu.c | 17 ++--- src/system/kernel/main.c | 91 ++++++++++++------------- src/system/kernel/smp.c | 47 +++++++------ src/system/kernel/thread.c | 56 ++++++++------- 11 files changed, 120 insertions(+), 109 deletions(-) diff --git a/headers/build/os/drivers/KernelExport.h b/headers/build/os/drivers/KernelExport.h index 20e8e62c92..8201f402a4 100644 --- a/headers/build/os/drivers/KernelExport.h +++ b/headers/build/os/drivers/KernelExport.h @@ -209,6 +209,7 @@ extern status_t register_kernel_daemon(daemon_hook hook, void *arg, int frequen extern status_t unregister_kernel_daemon(daemon_hook hook, void *arg); extern void call_all_cpus(void (*f)(void *, int), void *cookie); +extern void call_all_cpus_sync(void (*f)(void *, int), void *cookie); /* safe methods to access user memory without having to lock it */ extern status_t user_memcpy(void *to, const void *from, size_t size); diff --git a/headers/os/drivers/KernelExport.h b/headers/os/drivers/KernelExport.h index e91d2c88e5..076821ff7f 100644 --- a/headers/os/drivers/KernelExport.h +++ b/headers/os/drivers/KernelExport.h @@ -216,6 +216,7 @@ extern status_t register_kernel_daemon(daemon_hook hook, void *arg, int frequen extern status_t unregister_kernel_daemon(daemon_hook hook, void *arg); extern void call_all_cpus(void (*f)(void *, int), void *cookie); +extern void call_all_cpus_sync(void (*f)(void *, int), void *cookie); /* safe methods to access user memory without having to lock it */ extern status_t user_memcpy(void *to, const void *from, size_t size); diff --git a/headers/private/kernel/arch/cpu.h b/headers/private/kernel/arch/cpu.h index 53386f072f..725a2f4ad9 100644 --- a/headers/private/kernel/arch/cpu.h +++ b/headers/private/kernel/arch/cpu.h @@ -19,7 +19,7 @@ extern "C" { #endif -status_t arch_cpu_preboot_init(kernel_args *args); +status_t arch_cpu_preboot_init_percpu(kernel_args *args, int curr_cpu); status_t arch_cpu_init(kernel_args *args); status_t arch_cpu_init_percpu(kernel_args *args, int curr_cpu); status_t arch_cpu_init_post_vm(kernel_args *args); diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index 4554cc6151..40f4edcc78 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -43,7 +43,7 @@ extern cpu_ent gCPU[MAX_BOOT_CPUS]; extern "C" { #endif -status_t cpu_preboot_init(struct kernel_args *args); +status_t cpu_preboot_init_percpu(struct kernel_args *args, int curr_cpu); status_t cpu_init(struct kernel_args *args); status_t cpu_init_percpu(kernel_args *ka, int curr_cpu); status_t cpu_init_post_vm(struct kernel_args *args); diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index bcea811e44..873211246a 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -30,7 +30,7 @@ void thread_at_kernel_entry(void); void thread_at_kernel_exit(void); status_t thread_init(struct kernel_args *args); -status_t thread_per_cpu_init(int32 cpuNum); +status_t thread_preboot_init_percpu(struct kernel_args *args, int32 cpuNum); void thread_yield(void); void thread_exit(void); diff --git a/src/system/kernel/arch/ppc/arch_cpu.cpp b/src/system/kernel/arch/ppc/arch_cpu.cpp index c37209df05..a9f9ce0708 100644 --- a/src/system/kernel/arch/ppc/arch_cpu.cpp +++ b/src/system/kernel/arch/ppc/arch_cpu.cpp @@ -17,7 +17,7 @@ static bool sHasTlbia; status_t -arch_cpu_preboot_init(kernel_args *args) +arch_cpu_preboot_init_percpu(kernel_args *args, int curr_cpu) { // enable FPU set_msr(get_msr() | MSR_FP_AVAILABLE); diff --git a/src/system/kernel/arch/x86/arch_cpu.c b/src/system/kernel/arch/x86/arch_cpu.c index b0475e8697..8684db96b5 100644 --- a/src/system/kernel/arch/x86/arch_cpu.c +++ b/src/system/kernel/arch/x86/arch_cpu.c @@ -320,7 +320,7 @@ static void make_feature_string(cpu_ent *cpu, char *str, size_t strlen) strlcat(str, "3dnow ", strlen); } -static int detect_cpu(kernel_args *ka, int curr_cpu) +static int detect_cpu(int curr_cpu) { cpuid_info cpuid; unsigned int data[4]; @@ -431,10 +431,8 @@ bool x86_check_feature(uint32 feature, enum x86_feature_type type) // #pragma mark - status_t -arch_cpu_preboot_init(kernel_args *args) +arch_cpu_preboot_init_percpu(kernel_args *args, int curr_cpu) { - write_dr3(0); - x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); gX86SwapFPUFunc = i386_fnsave_swap; @@ -445,7 +443,7 @@ arch_cpu_preboot_init(kernel_args *args) status_t arch_cpu_init_percpu(kernel_args *args, int curr_cpu) { - detect_cpu(args, curr_cpu); + detect_cpu(curr_cpu); // load the TSS for this cpu // note the main cpu gets initialized in arch_cpu_init_post_vm() diff --git a/src/system/kernel/cpu.c b/src/system/kernel/cpu.c index 06f9aee1d0..5f5a1a3701 100644 --- a/src/system/kernel/cpu.c +++ b/src/system/kernel/cpu.c @@ -26,13 +26,6 @@ static spinlock sSetCpuLock; status_t cpu_init(kernel_args *args) { - int i; - - memset(gCPU, 0, sizeof(gCPU)); - for (i = 0; i < MAX_BOOT_CPUS; i++) { - gCPU[i].cpu_num = i; - } - return arch_cpu_init(args); } @@ -57,11 +50,15 @@ cpu_init_post_modules(kernel_args *args) status_t -cpu_preboot_init(kernel_args *args) +cpu_preboot_init_percpu(kernel_args *args, int curr_cpu) { - return arch_cpu_preboot_init(args); -} + // set the cpu number in the local cpu structure so that + // we can use it for get_current_cpu + memset(&gCPU[curr_cpu], 0, sizeof(gCPU[curr_cpu])); + gCPU[curr_cpu].cpu_num = curr_cpu; + return arch_cpu_preboot_init_percpu(args, curr_cpu); +} bigtime_t cpu_get_active_time(int32 cpu) diff --git a/src/system/kernel/main.c b/src/system/kernel/main.c index c08b18f347..69d75c8a2d 100644 --- a/src/system/kernel/main.c +++ b/src/system/kernel/main.c @@ -42,9 +42,9 @@ //#define TRACE_BOOT #ifdef TRACE_BOOT -# define TRACE(x) dprintf x +# define TRACE(x...) dprintf("INIT : " x) #else -# define TRACE(x) ; +# define TRACE(x...) ; #endif bool kernel_startup; @@ -75,7 +75,8 @@ _start(kernel_args *bootKernelArgs, int currentCPU) smp_set_num_cpus(sKernelArgs.num_cpus); // do any pre-booting cpu config - cpu_preboot_init(&sKernelArgs); + cpu_preboot_init_percpu(&sKernelArgs, currentCPU); + thread_preboot_init_percpu(&sKernelArgs, currentCPU); // if we're not a boot cpu, spin here until someone wakes us up if (smp_trap_non_boot_cpus(currentCPU)) { @@ -93,92 +94,90 @@ _start(kernel_args *bootKernelArgs, int currentCPU) smp_wait_for_non_boot_cpus(); // init modules - TRACE(("init CPU\n")); + TRACE("init CPU\n"); cpu_init(&sKernelArgs); cpu_init_percpu(&sKernelArgs, currentCPU); - TRACE(("init interrupts\n")); + TRACE("init interrupts\n"); int_init(&sKernelArgs); - TRACE(("init VM\n")); + TRACE("init VM\n"); vm_init(&sKernelArgs); // Before vm_init_post_sem() is called, we have to make sure that // the boot loader allocated region is not used anymore // now we can use the heap and create areas arch_platform_init_post_vm(&sKernelArgs); - TRACE(("init driver_settings\n")); + TRACE("init driver_settings\n"); boot_item_init(); driver_settings_init(&sKernelArgs); debug_init_post_vm(&sKernelArgs); int_init_post_vm(&sKernelArgs); cpu_init_post_vm(&sKernelArgs); - TRACE(("init system info\n")); + TRACE("init system info\n"); system_info_init(&sKernelArgs); - TRACE(("init SMP\n")); + TRACE("init SMP\n"); smp_init(&sKernelArgs); - TRACE(("init timer\n")); + TRACE("init timer\n"); timer_init(&sKernelArgs); - TRACE(("init real time clock\n")); + TRACE("init real time clock\n"); rtc_init(&sKernelArgs); - TRACE(("init semaphores\n")); + TRACE("init semaphores\n"); sem_init(&sKernelArgs); // now we can create and use semaphores - TRACE(("init VM semaphores\n")); + TRACE("init VM semaphores\n"); vm_init_post_sem(&sKernelArgs); - TRACE(("init driver_settings\n")); + TRACE("init driver_settings\n"); driver_settings_init_post_sem(&sKernelArgs); - TRACE(("init generic syscall\n")); + TRACE("init generic syscall\n"); generic_syscall_init(); - TRACE(("init cbuf\n")); + TRACE("init cbuf\n"); cbuf_init(); - TRACE(("init teams\n")); + TRACE("init teams\n"); team_init(&sKernelArgs); - TRACE(("init threads\n")); + TRACE("init threads\n"); thread_init(&sKernelArgs); - TRACE(("init ports\n")); + TRACE("init ports\n"); port_init(&sKernelArgs); - TRACE(("init kernel daemons\n")); + TRACE("init kernel daemons\n"); kernel_daemon_init(); arch_platform_init_post_thread(&sKernelArgs); - TRACE(("init VM threads\n")); + TRACE("init VM threads\n"); vm_init_post_thread(&sKernelArgs); - TRACE(("init ELF loader\n")); + TRACE("init ELF loader\n"); elf_init(&sKernelArgs); - TRACE(("init scheduler\n")); + TRACE("init scheduler\n"); scheduler_init(); - TRACE(("init VFS\n")); + TRACE("init VFS\n"); vfs_init(&sKernelArgs); - // start a thread to finish initializing the rest of the system - thread = spawn_kernel_thread(&main2, "main2", B_NORMAL_PRIORITY, NULL); - - smp_wake_up_non_boot_cpus(); - - TRACE(("enable interrupts, exit kernel startup\n")); + TRACE("enable interrupts, exit kernel startup\n"); kernel_startup = false; - enable_interrupts(); + TRACE("waking up AP cpus\n"); + smp_wake_up_non_boot_cpus(); + + enable_interrupts(); scheduler_start(); + + // start a thread to finish initializing the rest of the system + TRACE("starting main2 thread\n"); + thread = spawn_kernel_thread(&main2, "main2", B_NORMAL_PRIORITY, NULL); resume_thread(thread); } else { // this is run for each non boot processor after they've been set loose - - // the order here is pretty important, and kind of arch specific, so it's sort of a hack at the moment. - // thread_* will set the current thread pointer, which lets low level code know what cpu it's on - // cpu_* will detect the current cpu and do any pending low level setup - // smp_* will set up the low level smp routines - thread_per_cpu_init(currentCPU); cpu_init_percpu(&sKernelArgs, currentCPU); smp_per_cpu_init(&sKernelArgs, currentCPU); + // welcome to the machine enable_interrupts(); + scheduler_start(); } - TRACE(("main: done... begin idle loop on cpu %d\n", currentCPU)); + TRACE("main: done... begin idle loop on cpu %d\n", currentCPU); for (;;) arch_cpu_idle(); @@ -191,9 +190,9 @@ main2(void *unused) { (void)(unused); - TRACE(("start of main2: initializing devices\n")); + TRACE("start of main2: initializing devices\n"); - TRACE(("Init modules\n")); + TRACE("Init modules\n"); module_init(&sKernelArgs); // ToDo: the preloaded image debug data is placed in the kernel args, and @@ -208,18 +207,18 @@ main2(void *unused) } // init userland debugging - TRACE(("Init Userland debugging\n")); + TRACE("Init Userland debugging\n"); init_user_debug(); // init the messaging service - TRACE(("Init Messaging Service\n")); + TRACE("Init Messaging Service\n"); init_messaging_service(); /* bootstrap all the filesystems */ - TRACE(("Bootstrap file systems\n")); + TRACE("Bootstrap file systems\n"); vfs_bootstrap_file_systems(); - TRACE(("Init Device Manager\n")); + TRACE("Init Device Manager\n"); device_manager_init(&sKernelArgs); // ToDo: device manager starts here, bus_init()/dev_init() won't be necessary anymore, @@ -227,7 +226,7 @@ main2(void *unused) int_init_post_device_manager(&sKernelArgs); - TRACE(("Mount boot file system\n")); + TRACE("Mount boot file system\n"); vfs_mount_boot_file_system(&sKernelArgs); // CPU specific modules may now be available @@ -259,7 +258,7 @@ main2(void *unused) thread = load_image(argc, args, NULL); if (thread >= B_OK) { resume_thread(thread); - TRACE(("Bootscript started\n")); + TRACE("Bootscript started\n"); } else dprintf("error starting \"%s\" error = %ld \n", args[0], thread); } diff --git a/src/system/kernel/smp.c b/src/system/kernel/smp.c index 31a8fd88b9..5042387098 100644 --- a/src/system/kernel/smp.c +++ b/src/system/kernel/smp.c @@ -551,8 +551,14 @@ smp_trap_non_boot_cpus(int32 cpu) { if (cpu > 0) { boot_cpu_spin[cpu] = 1; - acquire_spinlock(&boot_cpu_spin[cpu]); + acquire_spinlock_nocheck(&boot_cpu_spin[cpu]); return false; + + // lets make sure we're in sync with the main cpu + // the boot processor has probably been sending us + // tlb sync messages all along the way, but we've + // been ignoring them + arch_cpu_global_TLB_invalidate(); } return true; @@ -562,23 +568,16 @@ smp_trap_non_boot_cpus(int32 cpu) void smp_wake_up_non_boot_cpus() { - // resume non boot CPUs int i; - for (i = 1; i < sNumCPUs; i++) { - release_spinlock(&boot_cpu_spin[i]); - } // ICIs were previously being ignored if (sNumCPUs > 1) sICIEnabled = true; - // invalidate all of the other processors' TLB caches - arch_cpu_global_TLB_invalidate(); - smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, NULL, - SMP_MSG_FLAG_SYNC); - - // start the other processors - smp_send_broadcast_ici(SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, SMP_MSG_FLAG_ASYNC); + // resume non boot CPUs + for (i = 1; i < sNumCPUs; i++) { + release_spinlock(&boot_cpu_spin[i]); + } } @@ -651,13 +650,7 @@ smp_get_num_cpus() int32 smp_get_current_cpu(void) { - struct thread *thread = thread_get_current_thread(); - if (thread) - return thread->cpu->cpu_num; - - // this is not always correct during early boot, but it's okay - // for the boot process - return 0; + return thread_get_current_thread()->cpu->cpu_num; } @@ -681,3 +674,19 @@ call_all_cpus(void (*func)(void *, int), void *cookie) restore_interrupts(state); } +void +call_all_cpus_sync(void (*func)(void *, int), void *cookie) +{ + cpu_status state = disable_interrupts(); + + if (smp_get_num_cpus() > 1) { + smp_send_broadcast_ici(SMP_MSG_CALL_FUNCTION, (uint32)cookie, + 0, 0, (void *)func, SMP_MSG_FLAG_SYNC); + } + + // we need to call this function ourselves as well + func(cookie, smp_get_current_cpu()); + + restore_interrupts(state); +} + diff --git a/src/system/kernel/thread.c b/src/system/kernel/thread.c index 8bf780edbe..101380e61d 100644 --- a/src/system/kernel/thread.c +++ b/src/system/kernel/thread.c @@ -63,7 +63,7 @@ static status_t receive_data_etc(thread_id *_sender, void *buffer, spinlock thread_spinlock = 0; // thread list -static struct thread *sIdleThreads[B_MAX_CPU_COUNT]; +static struct thread sIdleThreads[B_MAX_CPU_COUNT]; static void *sThreadHash = NULL; static thread_id sNextThreadID = 1; @@ -162,30 +162,36 @@ thread_struct_hash(void *_t, const void *_key, uint32 range) return (uint32)key->id % range; } - -/** Allocates a thread structure (or reuses one from the dead queue). +/** Allocates and fills in thread structure (or reuses one from the dead queue). * * \param threadID The ID to be assigned to the new thread. If * \code < 0 \endcode a fresh one is allocated. + * \param thread initialize this thread struct if nonnull */ static struct thread * -create_thread_struct(const char *name, thread_id threadID) +create_thread_struct(struct thread *inthread, const char *name, thread_id threadID) { struct thread *thread; cpu_status state; char temp[64]; - state = disable_interrupts(); - GRAB_THREAD_LOCK(); - thread = thread_dequeue(&dead_q); - RELEASE_THREAD_LOCK(); - restore_interrupts(state); + if (inthread == NULL) { + // try to recycle one from the dead queue first + state = disable_interrupts(); + GRAB_THREAD_LOCK(); + thread = thread_dequeue(&dead_q); + RELEASE_THREAD_LOCK(); + restore_interrupts(state); - if (thread == NULL) { - thread = (struct thread *)malloc(sizeof(struct thread)); - if (thread == NULL) - return NULL; + // if not, create a new one + if (thread == NULL) { + thread = (struct thread *)malloc(sizeof(struct thread)); + if (thread == NULL) + return NULL; + } + } else { + thread = inthread; } if (name != NULL) @@ -195,7 +201,8 @@ create_thread_struct(const char *name, thread_id threadID) thread->id = threadID >= 0 ? threadID : allocate_thread_id(); thread->team = NULL; - thread->cpu = NULL; + // XXX terrible hack, this is to leave the early boot cpu pointers alone while being initialized +// thread->cpu = NULL; thread->sem.blocking = -1; thread->fault_handler = 0; thread->page_faults_allowed = 1; @@ -251,7 +258,8 @@ err2: delete_sem(thread->exit.sem); err1: // ToDo: put them in the dead queue instead? - free(thread); + if (inthread == NULL) + free(thread); return NULL; } @@ -352,7 +360,7 @@ create_thread(const char *name, team_id teamID, thread_entry_func entry, TRACE(("create_thread(%s, id = %ld, %s)\n", name, threadID, kernel ? "kernel" : "user")); - thread = create_thread_struct(name, threadID); + thread = create_thread_struct(NULL, name, threadID); if (thread == NULL) return B_NO_MEMORY; @@ -1461,7 +1469,7 @@ thread_init(kernel_args *args) char name[64]; sprintf(name, "idle thread %lu", i + 1); - thread = create_thread_struct(name, + thread = create_thread_struct(&sIdleThreads[i], name, i == 0 ? team_get_kernel_team_id() : -1); if (thread == NULL) { panic("error creating idle thread struct\n"); @@ -1483,11 +1491,6 @@ thread_init(kernel_args *args) hash_insert(sThreadHash, thread); insert_thread_into_team(thread->team, thread); - sIdleThreads[i] = thread; - - if (i == 0) - arch_thread_set_current_thread(thread); - thread->cpu = &gCPU[i]; } sUsedThreads = args->num_cpus; @@ -1539,13 +1542,16 @@ thread_init(kernel_args *args) status_t -thread_per_cpu_init(int32 cpuNum) +thread_preboot_init_percpu(struct kernel_args *args, int32 cpuNum) { - arch_thread_set_current_thread(sIdleThreads[cpuNum]); + // set up the cpu pointer in the not yet initialized per-cpu idle thread + // so that get_current_cpu and friends will work, which is crucial for + // a lot of low level routines + sIdleThreads[cpuNum].cpu = &gCPU[cpuNum]; + arch_thread_set_current_thread(&sIdleThreads[cpuNum]); return B_OK; } - // #pragma mark - public kernel API