diff --git a/headers/build/os/drivers/KernelExport.h b/headers/build/os/drivers/KernelExport.h
index 20e8e62c92..8201f402a4 100644
--- a/headers/build/os/drivers/KernelExport.h
+++ b/headers/build/os/drivers/KernelExport.h
@@ -209,6 +209,7 @@ extern status_t		register_kernel_daemon(daemon_hook hook, void *arg, int frequen
 extern status_t		unregister_kernel_daemon(daemon_hook hook, void *arg);
 
 extern void			call_all_cpus(void (*f)(void *, int), void *cookie);
+extern void			call_all_cpus_sync(void (*f)(void *, int), void *cookie);
 
 /* safe methods to access user memory without having to lock it */
 extern status_t		user_memcpy(void *to, const void *from, size_t size);
diff --git a/headers/os/drivers/KernelExport.h b/headers/os/drivers/KernelExport.h
index e91d2c88e5..076821ff7f 100644
--- a/headers/os/drivers/KernelExport.h
+++ b/headers/os/drivers/KernelExport.h
@@ -216,6 +216,7 @@ extern status_t		register_kernel_daemon(daemon_hook hook, void *arg, int frequen
 extern status_t		unregister_kernel_daemon(daemon_hook hook, void *arg);
 
 extern void			call_all_cpus(void (*f)(void *, int), void *cookie);
+extern void			call_all_cpus_sync(void (*f)(void *, int), void *cookie);
 
 /* safe methods to access user memory without having to lock it */
 extern status_t		user_memcpy(void *to, const void *from, size_t size);
diff --git a/headers/private/kernel/arch/cpu.h b/headers/private/kernel/arch/cpu.h
index 53386f072f..725a2f4ad9 100644
--- a/headers/private/kernel/arch/cpu.h
+++ b/headers/private/kernel/arch/cpu.h
@@ -19,7 +19,7 @@
 extern "C" {
 #endif 
 
-status_t arch_cpu_preboot_init(kernel_args *args);
+status_t arch_cpu_preboot_init_percpu(kernel_args *args, int curr_cpu);
 status_t arch_cpu_init(kernel_args *args);
 status_t arch_cpu_init_percpu(kernel_args *args, int curr_cpu);
 status_t arch_cpu_init_post_vm(kernel_args *args);
diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h
index 4554cc6151..40f4edcc78 100644
--- a/headers/private/kernel/cpu.h
+++ b/headers/private/kernel/cpu.h
@@ -43,7 +43,7 @@ extern cpu_ent gCPU[MAX_BOOT_CPUS];
 extern "C" {
 #endif
 
-status_t cpu_preboot_init(struct kernel_args *args);
+status_t cpu_preboot_init_percpu(struct kernel_args *args, int curr_cpu);
 status_t cpu_init(struct kernel_args *args);
 status_t cpu_init_percpu(kernel_args *ka, int curr_cpu);
 status_t cpu_init_post_vm(struct kernel_args *args);
diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h
index bcea811e44..873211246a 100644
--- a/headers/private/kernel/thread.h
+++ b/headers/private/kernel/thread.h
@@ -30,7 +30,7 @@ void thread_at_kernel_entry(void);
 void thread_at_kernel_exit(void);
 
 status_t thread_init(struct kernel_args *args);
-status_t thread_per_cpu_init(int32 cpuNum);
+status_t thread_preboot_init_percpu(struct kernel_args *args, int32 cpuNum);
 void thread_yield(void);
 void thread_exit(void);
 
diff --git a/src/system/kernel/arch/ppc/arch_cpu.cpp b/src/system/kernel/arch/ppc/arch_cpu.cpp
index c37209df05..a9f9ce0708 100644
--- a/src/system/kernel/arch/ppc/arch_cpu.cpp
+++ b/src/system/kernel/arch/ppc/arch_cpu.cpp
@@ -17,7 +17,7 @@
 static bool sHasTlbia;
 
 status_t 
-arch_cpu_preboot_init(kernel_args *args)
+arch_cpu_preboot_init_percpu(kernel_args *args, int curr_cpu)
 {
 	// enable FPU
 	set_msr(get_msr() | MSR_FP_AVAILABLE);
diff --git a/src/system/kernel/arch/x86/arch_cpu.c b/src/system/kernel/arch/x86/arch_cpu.c
index b0475e8697..8684db96b5 100644
--- a/src/system/kernel/arch/x86/arch_cpu.c
+++ b/src/system/kernel/arch/x86/arch_cpu.c
@@ -320,7 +320,7 @@ static void make_feature_string(cpu_ent *cpu, char *str, size_t strlen)
 		strlcat(str, "3dnow ", strlen);
 }
 
-static int detect_cpu(kernel_args *ka, int curr_cpu) 
+static int detect_cpu(int curr_cpu) 
 {
 	cpuid_info cpuid;
 	unsigned int data[4];
@@ -431,10 +431,8 @@ bool x86_check_feature(uint32 feature, enum x86_feature_type type)
 //	#pragma mark -
 
 status_t
-arch_cpu_preboot_init(kernel_args *args)
+arch_cpu_preboot_init_percpu(kernel_args *args, int curr_cpu)
 {
-	write_dr3(0);
-
 	x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU));
 	gX86SwapFPUFunc = i386_fnsave_swap;
 
@@ -445,7 +443,7 @@ arch_cpu_preboot_init(kernel_args *args)
 status_t 
 arch_cpu_init_percpu(kernel_args *args, int curr_cpu)
 {
-	detect_cpu(args, curr_cpu);
+	detect_cpu(curr_cpu);
 
 	// load the TSS for this cpu
 	// note the main cpu gets initialized in arch_cpu_init_post_vm()
diff --git a/src/system/kernel/cpu.c b/src/system/kernel/cpu.c
index 06f9aee1d0..5f5a1a3701 100644
--- a/src/system/kernel/cpu.c
+++ b/src/system/kernel/cpu.c
@@ -26,13 +26,6 @@ static spinlock sSetCpuLock;
 status_t
 cpu_init(kernel_args *args)
 {
-	int i;
-
-	memset(gCPU, 0, sizeof(gCPU));
-	for (i = 0; i < MAX_BOOT_CPUS; i++) {
-		gCPU[i].cpu_num = i;
-	}
-
 	return arch_cpu_init(args);
 }
 
@@ -57,11 +50,15 @@ cpu_init_post_modules(kernel_args *args)
 
 
 status_t
-cpu_preboot_init(kernel_args *args)
+cpu_preboot_init_percpu(kernel_args *args, int curr_cpu)
 {
-	return arch_cpu_preboot_init(args);
-}
+	// set the cpu number in the local cpu structure so that
+	// we can use it for get_current_cpu
+	memset(&gCPU[curr_cpu], 0, sizeof(gCPU[curr_cpu]));
+	gCPU[curr_cpu].cpu_num = curr_cpu;
 
+	return arch_cpu_preboot_init_percpu(args, curr_cpu);
+}
 
 bigtime_t
 cpu_get_active_time(int32 cpu)
diff --git a/src/system/kernel/main.c b/src/system/kernel/main.c
index c08b18f347..69d75c8a2d 100644
--- a/src/system/kernel/main.c
+++ b/src/system/kernel/main.c
@@ -42,9 +42,9 @@
 
 //#define TRACE_BOOT
 #ifdef TRACE_BOOT
-#	define TRACE(x) dprintf x
+#	define TRACE(x...) dprintf("INIT : " x)
 #else
-#	define TRACE(x) ;
+#	define TRACE(x...) ;
 #endif
 
 bool kernel_startup;
@@ -75,7 +75,8 @@ _start(kernel_args *bootKernelArgs, int currentCPU)
 	smp_set_num_cpus(sKernelArgs.num_cpus);
 
 	// do any pre-booting cpu config
-	cpu_preboot_init(&sKernelArgs);
+	cpu_preboot_init_percpu(&sKernelArgs, currentCPU);
+	thread_preboot_init_percpu(&sKernelArgs, currentCPU);
 
 	// if we're not a boot cpu, spin here until someone wakes us up
 	if (smp_trap_non_boot_cpus(currentCPU)) {
@@ -93,92 +94,90 @@ _start(kernel_args *bootKernelArgs, int currentCPU)
 		smp_wait_for_non_boot_cpus();
 
 		// init modules
-		TRACE(("init CPU\n"));
+		TRACE("init CPU\n");
 		cpu_init(&sKernelArgs);
 		cpu_init_percpu(&sKernelArgs, currentCPU);
-		TRACE(("init interrupts\n"));
+		TRACE("init interrupts\n");
 		int_init(&sKernelArgs);
 
-		TRACE(("init VM\n"));
+		TRACE("init VM\n");
 		vm_init(&sKernelArgs);
 			// Before vm_init_post_sem() is called, we have to make sure that
 			// the boot loader allocated region is not used anymore
 
 		// now we can use the heap and create areas
 		arch_platform_init_post_vm(&sKernelArgs);
-		TRACE(("init driver_settings\n"));
+		TRACE("init driver_settings\n");
 		boot_item_init();
 		driver_settings_init(&sKernelArgs);
 		debug_init_post_vm(&sKernelArgs);
 		int_init_post_vm(&sKernelArgs);
 		cpu_init_post_vm(&sKernelArgs);
-		TRACE(("init system info\n"));
+		TRACE("init system info\n");
 		system_info_init(&sKernelArgs);
 
-		TRACE(("init SMP\n"));
+		TRACE("init SMP\n");
 		smp_init(&sKernelArgs);
-		TRACE(("init timer\n"));
+		TRACE("init timer\n");
 		timer_init(&sKernelArgs);
-		TRACE(("init real time clock\n"));
+		TRACE("init real time clock\n");
 		rtc_init(&sKernelArgs);
 
-		TRACE(("init semaphores\n"));
+		TRACE("init semaphores\n");
 		sem_init(&sKernelArgs);
 
 		// now we can create and use semaphores
-		TRACE(("init VM semaphores\n"));
+		TRACE("init VM semaphores\n");
 		vm_init_post_sem(&sKernelArgs);
-		TRACE(("init driver_settings\n"));
+		TRACE("init driver_settings\n");
 		driver_settings_init_post_sem(&sKernelArgs);
-		TRACE(("init generic syscall\n"));
+		TRACE("init generic syscall\n");
 		generic_syscall_init();
-		TRACE(("init cbuf\n"));
+		TRACE("init cbuf\n");
 		cbuf_init();
-		TRACE(("init teams\n"));
+		TRACE("init teams\n");
 		team_init(&sKernelArgs);
-		TRACE(("init threads\n"));
+		TRACE("init threads\n");
 		thread_init(&sKernelArgs);
-		TRACE(("init ports\n"));
+		TRACE("init ports\n");
 		port_init(&sKernelArgs);
-		TRACE(("init kernel daemons\n"));
+		TRACE("init kernel daemons\n");
 		kernel_daemon_init();
 		arch_platform_init_post_thread(&sKernelArgs);
 
-		TRACE(("init VM threads\n"));
+		TRACE("init VM threads\n");
 		vm_init_post_thread(&sKernelArgs);
-		TRACE(("init ELF loader\n"));
+		TRACE("init ELF loader\n");
 		elf_init(&sKernelArgs);
-		TRACE(("init scheduler\n"));
+		TRACE("init scheduler\n");
 		scheduler_init();
-		TRACE(("init VFS\n"));
+		TRACE("init VFS\n");
 		vfs_init(&sKernelArgs);
 
-		// start a thread to finish initializing the rest of the system
-		thread = spawn_kernel_thread(&main2, "main2", B_NORMAL_PRIORITY, NULL);
-
-		smp_wake_up_non_boot_cpus();
-
-		TRACE(("enable interrupts, exit kernel startup\n"));
+		TRACE("enable interrupts, exit kernel startup\n");
 		kernel_startup = false;
-		enable_interrupts();
 
+		TRACE("waking up AP cpus\n");
+		smp_wake_up_non_boot_cpus();		
+
+		enable_interrupts();
 		scheduler_start();
+
+		// start a thread to finish initializing the rest of the system
+		TRACE("starting main2 thread\n");
+		thread = spawn_kernel_thread(&main2, "main2", B_NORMAL_PRIORITY, NULL);
 		resume_thread(thread);
 	} else {
 		// this is run for each non boot processor after they've been set loose
-
-		// the order here is pretty important, and kind of arch specific, so it's sort of a hack at the moment.
-		// thread_* will set the current thread pointer, which lets low level code know what cpu it's on
-		// cpu_* will detect the current cpu and do any pending low level setup
-		// smp_* will set up the low level smp routines
-		thread_per_cpu_init(currentCPU);
 		cpu_init_percpu(&sKernelArgs, currentCPU);
 		smp_per_cpu_init(&sKernelArgs, currentCPU);
 
+		// welcome to the machine
 		enable_interrupts();
+		scheduler_start();
 	}
 
-	TRACE(("main: done... begin idle loop on cpu %d\n", currentCPU));
+	TRACE("main: done... begin idle loop on cpu %d\n", currentCPU);
 	for (;;)
 		arch_cpu_idle();
 
@@ -191,9 +190,9 @@ main2(void *unused)
 {
 	(void)(unused);
 
-	TRACE(("start of main2: initializing devices\n"));
+	TRACE("start of main2: initializing devices\n");
 
-	TRACE(("Init modules\n"));
+	TRACE("Init modules\n");
 	module_init(&sKernelArgs);
 
 	// ToDo: the preloaded image debug data is placed in the kernel args, and
@@ -208,18 +207,18 @@ main2(void *unused)
 	}
 
 	// init userland debugging
-	TRACE(("Init Userland debugging\n"));
+	TRACE("Init Userland debugging\n");
 	init_user_debug();
 
 	// init the messaging service
-	TRACE(("Init Messaging Service\n"));
+	TRACE("Init Messaging Service\n");
 	init_messaging_service();
 
 	/* bootstrap all the filesystems */
-	TRACE(("Bootstrap file systems\n"));
+	TRACE("Bootstrap file systems\n");
 	vfs_bootstrap_file_systems();
 
-	TRACE(("Init Device Manager\n"));
+	TRACE("Init Device Manager\n");
 	device_manager_init(&sKernelArgs);
 
 	// ToDo: device manager starts here, bus_init()/dev_init() won't be necessary anymore,
@@ -227,7 +226,7 @@ main2(void *unused)
 
 	int_init_post_device_manager(&sKernelArgs);
 
-	TRACE(("Mount boot file system\n"));
+	TRACE("Mount boot file system\n");
 	vfs_mount_boot_file_system(&sKernelArgs);
 
 	// CPU specific modules may now be available
@@ -259,7 +258,7 @@ main2(void *unused)
 		thread = load_image(argc, args, NULL);
 		if (thread >= B_OK) {
 			resume_thread(thread);
-			TRACE(("Bootscript started\n"));
+			TRACE("Bootscript started\n");
 		} else
 			dprintf("error starting \"%s\" error = %ld \n", args[0], thread);
 	}
diff --git a/src/system/kernel/smp.c b/src/system/kernel/smp.c
index 31a8fd88b9..5042387098 100644
--- a/src/system/kernel/smp.c
+++ b/src/system/kernel/smp.c
@@ -551,8 +551,14 @@ smp_trap_non_boot_cpus(int32 cpu)
 {
 	if (cpu > 0) {
 		boot_cpu_spin[cpu] = 1;
-		acquire_spinlock(&boot_cpu_spin[cpu]);
+		acquire_spinlock_nocheck(&boot_cpu_spin[cpu]);
 		return false;
+
+		// lets make sure we're in sync with the main cpu
+		// the boot processor has probably been sending us 
+		// tlb sync messages all along the way, but we've 
+		// been ignoring them
+		arch_cpu_global_TLB_invalidate();
 	}
 
 	return true;
@@ -562,23 +568,16 @@ smp_trap_non_boot_cpus(int32 cpu)
 void
 smp_wake_up_non_boot_cpus()
 {
-	// resume non boot CPUs
 	int i;
-	for (i = 1; i < sNumCPUs; i++) {
-		release_spinlock(&boot_cpu_spin[i]);
-	}
 
 	// ICIs were previously being ignored
 	if (sNumCPUs > 1)
 		sICIEnabled = true;
 
-	// invalidate all of the other processors' TLB caches
-	arch_cpu_global_TLB_invalidate();
-	smp_send_broadcast_ici(SMP_MSG_GLOBAL_INVALIDATE_PAGES, 0, 0, 0, NULL,
-		SMP_MSG_FLAG_SYNC);
-
-	// start the other processors
-	smp_send_broadcast_ici(SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, SMP_MSG_FLAG_ASYNC);
+	// resume non boot CPUs
+	for (i = 1; i < sNumCPUs; i++) {
+		release_spinlock(&boot_cpu_spin[i]);
+	}
 }
 
 
@@ -651,13 +650,7 @@ smp_get_num_cpus()
 int32
 smp_get_current_cpu(void)
 {
-	struct thread *thread = thread_get_current_thread();
-	if (thread)
-		return thread->cpu->cpu_num;
-
-	// this is not always correct during early boot, but it's okay
-	// for the boot process
-	return 0;
+	return thread_get_current_thread()->cpu->cpu_num;
 }
 
 
@@ -681,3 +674,19 @@ call_all_cpus(void (*func)(void *, int), void *cookie)
 	restore_interrupts(state);
 }
 
+void
+call_all_cpus_sync(void (*func)(void *, int), void *cookie)
+{
+	cpu_status state = disable_interrupts();
+
+	if (smp_get_num_cpus() > 1) {
+		smp_send_broadcast_ici(SMP_MSG_CALL_FUNCTION, (uint32)cookie,
+			0, 0, (void *)func, SMP_MSG_FLAG_SYNC);
+	}
+
+	// we need to call this function ourselves as well
+	func(cookie, smp_get_current_cpu());
+
+	restore_interrupts(state);
+}
+
diff --git a/src/system/kernel/thread.c b/src/system/kernel/thread.c
index 8bf780edbe..101380e61d 100644
--- a/src/system/kernel/thread.c
+++ b/src/system/kernel/thread.c
@@ -63,7 +63,7 @@ static status_t receive_data_etc(thread_id *_sender, void *buffer,
 spinlock thread_spinlock = 0;
 
 // thread list
-static struct thread *sIdleThreads[B_MAX_CPU_COUNT];
+static struct thread sIdleThreads[B_MAX_CPU_COUNT];
 static void *sThreadHash = NULL;
 static thread_id sNextThreadID = 1;
 
@@ -162,30 +162,36 @@ thread_struct_hash(void *_t, const void *_key, uint32 range)
 	return (uint32)key->id % range;
 }
 
-
-/**	Allocates a thread structure (or reuses one from the dead queue).
+/**	Allocates and fills in thread structure (or reuses one from the dead queue).
  *
  * \param threadID The ID to be assigned to the new thread. If
  *		  \code < 0 \endcode a fresh one is allocated.
+ * \param thread initialize this thread struct if nonnull
  */
 
 static struct thread *
-create_thread_struct(const char *name, thread_id threadID)
+create_thread_struct(struct thread *inthread, const char *name, thread_id threadID)
 {
 	struct thread *thread;
 	cpu_status state;
 	char temp[64];
 
-	state = disable_interrupts();
-	GRAB_THREAD_LOCK();
-	thread = thread_dequeue(&dead_q);
-	RELEASE_THREAD_LOCK();
-	restore_interrupts(state);
+	if (inthread == NULL) {
+		// try to recycle one from the dead queue first
+		state = disable_interrupts();
+		GRAB_THREAD_LOCK();
+		thread = thread_dequeue(&dead_q);
+		RELEASE_THREAD_LOCK();
+		restore_interrupts(state);
 
-	if (thread == NULL) {
-		thread = (struct thread *)malloc(sizeof(struct thread));
-		if (thread == NULL)
-			return NULL;
+		// if not, create a new one
+		if (thread == NULL) {
+			thread = (struct thread *)malloc(sizeof(struct thread));
+			if (thread == NULL)
+				return NULL;
+		}
+	} else {
+		thread = inthread;
 	}
 
 	if (name != NULL)
@@ -195,7 +201,8 @@ create_thread_struct(const char *name, thread_id threadID)
 
 	thread->id = threadID >= 0 ? threadID : allocate_thread_id();
 	thread->team = NULL;
-	thread->cpu = NULL;
+	// XXX terrible hack, this is to leave the early boot cpu pointers alone while being initialized
+//	thread->cpu = NULL;				
 	thread->sem.blocking = -1;
 	thread->fault_handler = 0;
 	thread->page_faults_allowed = 1;
@@ -251,7 +258,8 @@ err2:
 	delete_sem(thread->exit.sem);
 err1:
 	// ToDo: put them in the dead queue instead?
-	free(thread);
+	if (inthread == NULL)
+		free(thread);
 	return NULL;
 }
 
@@ -352,7 +360,7 @@ create_thread(const char *name, team_id teamID, thread_entry_func entry,
 
 	TRACE(("create_thread(%s, id = %ld, %s)\n", name, threadID, kernel ? "kernel" : "user"));
 
-	thread = create_thread_struct(name, threadID);
+	thread = create_thread_struct(NULL, name, threadID);
 	if (thread == NULL)
 		return B_NO_MEMORY;
 
@@ -1461,7 +1469,7 @@ thread_init(kernel_args *args)
 		char name[64];
 
 		sprintf(name, "idle thread %lu", i + 1);
-		thread = create_thread_struct(name,
+		thread = create_thread_struct(&sIdleThreads[i], name,
 			i == 0 ? team_get_kernel_team_id() : -1);
 		if (thread == NULL) {
 			panic("error creating idle thread struct\n");
@@ -1483,11 +1491,6 @@ thread_init(kernel_args *args)
 
 		hash_insert(sThreadHash, thread);
 		insert_thread_into_team(thread->team, thread);
-		sIdleThreads[i] = thread;
-
-		if (i == 0)
-			arch_thread_set_current_thread(thread);
-
 		thread->cpu = &gCPU[i];
 	}
 	sUsedThreads = args->num_cpus;
@@ -1539,13 +1542,16 @@ thread_init(kernel_args *args)
 
 
 status_t
-thread_per_cpu_init(int32 cpuNum)
+thread_preboot_init_percpu(struct kernel_args *args, int32 cpuNum)
 {
-	arch_thread_set_current_thread(sIdleThreads[cpuNum]);
+	// set up the cpu pointer in the not yet initialized per-cpu idle thread
+	// so that get_current_cpu and friends will work, which is crucial for 
+	// a lot of low level routines
+	sIdleThreads[cpuNum].cpu = &gCPU[cpuNum];
+	arch_thread_set_current_thread(&sIdleThreads[cpuNum]);
 	return B_OK;
 }
 
-
 //	#pragma mark - public kernel API