Merge branch 'scheduler'

Conflicts:
	build/jam/packages/Haiku
	headers/os/kernel/OS.h
	headers/os/opengl/GLRenderer.h
	headers/private/shared/cpu_type.h
	src/add-ons/kernel/drivers/power/acpi_battery/acpi_battery.h
	src/bin/sysinfo.cpp
	src/bin/top.c
	src/system/kernel/arch/x86/arch_system_info.cpp
	src/system/kernel/port.cpp

headers/private/kernel/DPC.h

@@ -62,10 +62,8 @@ public:
 									uint32 reservedSlots);
 			void				Close(bool cancelPending);
 
-			status_t			Add(DPCCallback* callback,
-									bool schedulerLocked);
-			status_t			Add(void (*function)(void*), void* argument,
-									bool schedulerLocked);
+			status_t			Add(DPCCallback* callback);
+			status_t			Add(void (*function)(void*), void* argument);
 			bool				Cancel(DPCCallback* callback);
 
 			thread_id			Thread() const

headers/private/kernel/MessagingServiceDefs.h

@@ -17,7 +17,7 @@ enum {
 };
 
 struct messaging_area_header {
-	vint32	lock_counter;
+	int32	lock_counter;
 	int32	size;				// set to 0, when area is discarded
 	area_id	kernel_area;
 	area_id	next_kernel_area;

headers/private/kernel/UserEvent.h

@@ -92,7 +92,7 @@ private:
 private:
 			ThreadCreationAttributes fCreationAttributes;
 			char				fThreadName[B_OS_NAME_LENGTH];
-			bool				fPendingDPC;
+			int32				fPendingDPC;
 };
 
 

headers/private/kernel/UserTimer.h

@@ -54,6 +54,8 @@ protected:
 	inline	void				UpdatePeriodicStartTime();
 	inline	void				CheckPeriodicOverrun(bigtime_t now);
 
+	inline	void				CancelTimer();
+
 protected:
 			int32				fID;
 			timer				fTimer;
@@ -62,6 +64,7 @@ protected:
 			bigtime_t			fInterval;
 			uint32				fOverrunCount;
 			bool				fScheduled;	// fTimer scheduled
+			int32				fSkip;
 };
 
 

headers/private/kernel/arch/atomic.h

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2014, Paweł Dziepak, [email protected].
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_ARCH_ATOMIC_H
+#define _KERNEL_ARCH_ATOMIC_H
+
+
+#include <SupportDefs.h>
+
+#include <KernelExport.h>
+
+
+#ifdef __x86_64__
+#	include <arch/x86/64/atomic.h>
+#elif __INTEL__
+#	include <arch/x86/32/atomic.h>
+#endif
+
+
+#endif	// _KERNEL_ARCH_ATOMIC_H
+

headers/private/kernel/arch/cpu.h

@@ -41,13 +41,8 @@ ssize_t arch_cpu_user_strlcpy(char *to, const char *from, size_t size,
 status_t arch_cpu_user_memset(void *s, char c, size_t count,
 			addr_t *faultHandler);
 
-void arch_cpu_idle(void);
 void arch_cpu_sync_icache(void *address, size_t length);
 
-void arch_cpu_memory_read_barrier(void);
-void arch_cpu_memory_write_barrier(void);
-void arch_cpu_memory_read_write_barrier(void);
-
 
 #ifdef __cplusplus
 }
@@ -55,4 +50,6 @@ void arch_cpu_memory_read_write_barrier(void);
 
 #include <arch_cpu.h>
 
+#define CACHE_LINE_ALIGN	 __attribute__((aligned(CACHE_LINE_SIZE)))
+
 #endif /* _KERNEL_ARCH_CPU_H */

headers/private/kernel/arch/int.h

@@ -34,6 +34,7 @@ void arch_int_enable_io_interrupt(int irq);
 void arch_int_disable_io_interrupt(int irq);
 void arch_int_configure_io_interrupt(int irq, uint32 config);
 bool arch_int_are_interrupts_enabled(void);
+void arch_int_assign_to_cpu(int32 irq, int32 cpu);
 
 #ifdef __cplusplus
 }

headers/private/kernel/arch/smp.h

@@ -8,23 +8,18 @@
 
 #include <kernel.h>
 
+
 struct kernel_args;
 
+class CPUSet;
 
-// must match MAX_BOOT_CPUS in platform_kernel_args.h
-#define SMP_MAX_CPUS MAX_BOOT_CPUS
 
-#ifdef __cplusplus
-extern "C" {
-#endif
+status_t arch_smp_init(kernel_args* args);
+status_t arch_smp_per_cpu_init(kernel_args* args, int32 cpu);
 
-status_t arch_smp_init(struct kernel_args *args);
-status_t arch_smp_per_cpu_init(struct kernel_args *args, int32 cpu);
 void arch_smp_send_ici(int32 target_cpu);
-void arch_smp_send_broadcast_ici(void);
+void arch_smp_send_broadcast_ici();
+void arch_smp_send_multicast_ici(CPUSet& cpuSet);
 
-#ifdef __cplusplus
-}
-#endif
 
 #endif	/* KERNEL_ARCH_SMP_H */

headers/private/kernel/arch/system_info.h

@@ -18,7 +18,7 @@ extern "C" {
 #endif
 
 status_t arch_system_info_init(struct kernel_args *args);
-status_t arch_get_system_info(system_info *info, size_t size);
+void arch_fill_topology_node(cpu_topology_node_info* node, int32 cpu);
 
 #ifdef __cplusplus
 }

headers/private/kernel/arch/x86/32/atomic.h

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2014, Paweł Dziepak, [email protected].
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_ARCH_X86_32_ATOMIC_H
+#define _KERNEL_ARCH_X86_32_ATOMIC_H
+
+
+static inline void
+memory_read_barrier_inline(void)
+{
+	asm volatile("lock; addl $0, (%%esp)" : : : "memory");
+}
+
+
+static inline void
+memory_write_barrier_inline(void)
+{
+	asm volatile("lock; addl $0, (%%esp)" : : : "memory");
+}
+
+
+static inline void
+memory_full_barrier_inline(void)
+{
+	asm volatile("lock; addl $0, (%%esp)" : : : "memory");
+}
+
+
+#define memory_read_barrier		memory_read_barrier_inline
+#define memory_write_barrier	memory_write_barrier_inline
+#define memory_full_barrier		memory_full_barrier_inline
+
+
+static inline void
+atomic_set_inline(int32* value, int32 newValue)
+{
+	memory_write_barrier();
+	*(volatile int32*)value = newValue;
+}
+
+
+static inline int32
+atomic_get_and_set_inline(int32* value, int32 newValue)
+{
+	asm volatile("xchgl %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_test_and_set_inline(int32* value, int32 newValue, int32 testAgainst)
+{
+	asm volatile("lock; cmpxchgl %2, (%3)"
+		: "=a" (newValue)
+		: "0" (testAgainst), "r" (newValue), "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_add_inline(int32* value, int32 newValue)
+{
+	asm volatile("lock; xaddl %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_get_inline(int32* value)
+{
+	int32 newValue = *(volatile int32*)value;
+	memory_read_barrier();
+	return newValue;
+}
+
+
+#define atomic_set				atomic_set_inline
+#define atomic_get_and_set		atomic_get_and_set_inline
+#ifndef atomic_test_and_set
+#	define atomic_test_and_set	atomic_test_and_set_inline
+#endif
+#ifndef atomic_add
+#	define atomic_add			atomic_add_inline
+#endif
+#define atomic_get				atomic_get_inline
+
+
+#endif	// _KERNEL_ARCH_X86_32_ATOMIC_H
+

headers/private/kernel/arch/x86/32/descriptors.h

@@ -9,30 +9,42 @@
 #define _KERNEL_ARCH_X86_32_DESCRIPTORS_H
 
 
-#define KERNEL_CODE_SEG	0x8
-#define KERNEL_DATA_SEG	0x10
+// Segments common for all CPUs.
+#define KERNEL_CODE_SEGMENT			1
+#define KERNEL_DATA_SEGMENT			2
 
-#define USER_CODE_SEG	0x1b
-#define USER_DATA_SEG	0x23
+#define USER_CODE_SEGMENT			3
+#define USER_DATA_SEGMENT			4
 
-#define APM_CODE32_SEGMENT	0x28
-#define APM_CODE16_SEGMENT	0x30
-#define APM_DATA_SEGMENT	0x38
+#define APM_CODE32_SEGMENT			5
+#define APM_CODE16_SEGMENT			6
+#define APM_DATA_SEGMENT			7
 
-#define BIOS_DATA_SEGMENT	0x40
+#define BIOS_DATA_SEGMENT			8
+
+// Per-CPU segments.
+#define TSS_SEGMENT					9
+#define DOUBLE_FAULT_TSS_SEGMENT	10
+#define KERNEL_TLS_SEGMENT			11
+#define USER_TLS_SEGMENT			12
+#define APM_SEGMENT					13
+
+#define GDT_SEGMENT_COUNT			14
+
+
+#define KERNEL_CODE_SELECTOR	((KERNEL_CODE_SEGMENT << 3) | DPL_KERNEL)
+#define KERNEL_DATA_SELECTOR	((KERNEL_DATA_SEGMENT << 3) | DPL_KERNEL)
+
+#define USER_CODE_SELECTOR	((USER_CODE_SEGMENT << 3) | DPL_USER)
+#define USER_DATA_SELECTOR	((USER_DATA_SEGMENT << 3) | DPL_USER)
+
+#define KERNEL_TLS_SELECTOR	((KERNEL_TLS_SEGMENT << 3) | DPL_KERNEL)
 
 
 #ifndef _ASSEMBLER
 	// this file can also be included from assembler as well
 	// (and is in arch_interrupts.S)
 
-#define DOUBLE_FAULT_TSS_BASE_SEGMENT 9
-#define TSS_BASE_SEGMENT	(DOUBLE_FAULT_TSS_BASE_SEGMENT + smp_get_num_cpus())
-#define TLS_BASE_SEGMENT	(TSS_BASE_SEGMENT + smp_get_num_cpus())
-#define APM_BASE_SEGMENT	(TLS_BASE_SEGMENT + smp_get_num_cpus())
-
-#define TSS_SEGMENT(cpu)	(TSS_BASE_SEGMENT + cpu)
-
 // defines entries in the GDT/LDT
 
 struct segment_descriptor {
@@ -73,6 +85,9 @@ struct tss {
 	uint16 io_map_base;
 };
 
+typedef segment_descriptor global_descriptor_table[GDT_SEGMENT_COUNT];
+extern global_descriptor_table gGDTs[];
+
 
 static inline void
 clear_segment_descriptor(segment_descriptor* desc)
@@ -141,6 +156,13 @@ set_tss_descriptor(segment_descriptor* desc, addr_t base, uint32 limit)
 }
 
 
+static inline segment_descriptor*
+get_gdt(int32 cpu)
+{
+	return gGDTs[cpu];
+}
+
+
 #endif	/* _ASSEMBLER */
 
 #endif	/* _KERNEL_ARCH_X86_32_DESCRIPTORS_H */

headers/private/kernel/arch/x86/32/iframe.h

@@ -37,7 +37,7 @@ struct iframe {
 	uint32 user_ss;
 };
 
-#define IFRAME_IS_USER(f)	((f)->cs == USER_CODE_SEG \
+#define IFRAME_IS_USER(f)	((f)->cs == USER_CODE_SELECTOR \
 								|| ((f)->flags & 0x20000) != 0)
 
 

headers/private/kernel/arch/x86/64/atomic.h

@@ -0,0 +1,153 @@
+/*
+ * Copyright 2014, Paweł Dziepak, [email protected].
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_ARCH_X86_64_ATOMIC_H
+#define _KERNEL_ARCH_X86_64_ATOMIC_H
+
+
+static inline void
+memory_read_barrier_inline(void)
+{
+	asm volatile("lfence" : : : "memory");
+}
+
+
+static inline void
+memory_write_barrier_inline(void)
+{
+	asm volatile("sfence" : : : "memory");
+}
+
+
+static inline void
+memory_full_barrier_inline(void)
+{
+	asm volatile("mfence" : : : "memory");
+}
+
+
+#define memory_read_barrier		memory_read_barrier_inline
+#define memory_write_barrier	memory_write_barrier_inline
+#define memory_full_barrier		memory_full_barrier_inline
+
+
+static inline void
+atomic_set_inline(int32* value, int32 newValue)
+{
+	memory_write_barrier();
+	*(volatile int32*)value = newValue;
+}
+
+
+static inline int32
+atomic_get_and_set_inline(int32* value, int32 newValue)
+{
+	asm volatile("xchg %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_test_and_set_inline(int32* value, int32 newValue, int32 testAgainst)
+{
+	asm volatile("lock; cmpxchgl %2, (%3)"
+		: "=a" (newValue)
+		: "0" (testAgainst), "r" (newValue), "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_add_inline(int32* value, int32 newValue)
+{
+	asm volatile("lock; xaddl %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_get_inline(int32* value)
+{
+	int32 newValue = *(volatile int32*)value;
+	memory_read_barrier();
+	return newValue;
+}
+
+
+static inline void
+atomic_set64_inline(int64* value, int64 newValue)
+{
+	memory_write_barrier();
+	*(volatile int64*)value = newValue;
+}
+
+
+static inline int64
+atomic_get_and_set64_inline(int64* value, int64 newValue)
+{
+	asm volatile("xchgq %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int64
+atomic_test_and_set64_inline(int64* value, int64 newValue, int64 testAgainst)
+{
+	asm volatile("lock; cmpxchgq %2, (%3)"
+		: "=a" (newValue)
+		: "0" (testAgainst), "r" (newValue), "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int64
+atomic_add64_inline(int64* value, int64 newValue)
+{
+	asm volatile("lock; xaddq %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int64
+atomic_get64_inline(int64* value)
+{
+	int64 newValue = *(volatile int64*)value;
+	memory_read_barrier();
+	return newValue;
+}
+
+
+#define atomic_set				atomic_set_inline
+#define atomic_get_and_set		atomic_get_and_set_inline
+#ifndef atomic_test_and_set
+#	define atomic_test_and_set	atomic_test_and_set_inline
+#endif
+#ifndef atomic_add
+#	define atomic_add			atomic_add_inline
+#endif
+#define atomic_get				atomic_get_inline
+
+#define atomic_set64			atomic_set64_inline
+#define atomic_get_and_set64	atomic_get_and_set64_inline
+#define atomic_test_and_set64	atomic_test_and_set64_inline
+#define atomic_add64			atomic_add64_inline
+#define atomic_get64			atomic_get64_inline
+
+
+#endif	// _KERNEL_ARCH_X86_64_ATOMIC_H
+

headers/private/kernel/arch/x86/64/descriptors.h

@@ -8,19 +8,28 @@
 
 // Segment definitions.
 // Note that the ordering of these is important to SYSCALL/SYSRET.
-#define KERNEL_CODE_SEG		0x08
-#define KERNEL_DATA_SEG		0x10
-#define USER_DATA_SEG		0x1b
-#define USER_CODE_SEG		0x23
+#define KERNEL_CODE_SEGMENT		1
+#define KERNEL_DATA_SEGMENT		2
+#define USER_DATA_SEGMENT		3
+#define USER_CODE_SEGMENT		4
+
+#define TSS_BASE_SEGMENT		5
+
+#define TSS_SEGMENT(cpu)		(TSS_BASE_SEGMENT + cpu * 2)
+
+#define GDT_SEGMENT_COUNT		(TSS_BASE_SEGMENT + SMP_MAX_CPUS * 2)
+
+
+#define KERNEL_CODE_SELECTOR	((KERNEL_CODE_SEGMENT << 3) | DPL_KERNEL)
+#define KERNEL_DATA_SELECTOR	((KERNEL_DATA_SEGMENT << 3) | DPL_KERNEL)
+
+#define USER_CODE_SELECTOR	((USER_CODE_SEGMENT << 3) | DPL_USER)
+#define USER_DATA_SELECTOR	((USER_DATA_SEGMENT << 3) | DPL_USER)
 
 
 #ifndef _ASSEMBLER
 
 
-#define TSS_BASE_SEGMENT	5
-#define TSS_SEGMENT(cpu)	(TSS_BASE_SEGMENT + cpu * 2)
-
-
 // Structure of a segment descriptor.
 struct segment_descriptor {
 	uint32 limit0 : 16;

headers/private/kernel/arch/x86/apic.h

@@ -52,7 +52,7 @@
 #define APIC_TRIGGER_MODE_LEVEL					(1 << 15)
 
 /* Interrupt Command defines */
-#define APIC_INTR_COMMAND_1_MASK				0xfff3f000
+#define APIC_INTR_COMMAND_1_MASK				0xfff32000
 #define APIC_INTR_COMMAND_2_MASK				0x00ffffff
 
 #define APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL	0
@@ -110,6 +110,7 @@
 #if !_BOOT_MODE
 
 bool		apic_available();
+bool		x2apic_available();
 uint32		apic_read(uint32 offset);
 void		apic_write(uint32 offset, uint32 data);
 uint32		apic_local_id();
@@ -122,10 +123,9 @@ void		apic_disable_local_ints();
 
 uint32		apic_spurious_intr_vector();
 void		apic_set_spurious_intr_vector(uint32 config);
-uint32		apic_intr_command_1();
-void		apic_set_intr_command_1(uint32 config);
-uint32		apic_intr_command_2();
-void		apic_set_intr_command_2(uint32 config);
+
+void		apic_set_interrupt_command(uint32 destination, uint32 mode);
+bool		apic_interrupt_delivered(void);
 
 uint32		apic_lvt_timer();
 void		apic_set_lvt_timer(uint32 config);

headers/private/kernel/arch/x86/arch_cpu.h

@@ -24,14 +24,25 @@
 #endif	// !_ASSEMBLER
 
 
+#define CPU_MAX_CACHE_LEVEL	8
+
+#define CACHE_LINE_SIZE		64
+
+
 // MSR registers (possibly Intel specific)
 #define IA32_MSR_TSC					0x10
 #define IA32_MSR_APIC_BASE				0x1b
 
+#define IA32_MSR_PLATFORM_INFO			0xce
+#define IA32_MSR_MPERF					0xe7
+#define IA32_MSR_APERF					0xe8
 #define IA32_MSR_MTRR_CAPABILITIES		0xfe
 #define IA32_MSR_SYSENTER_CS			0x174
 #define IA32_MSR_SYSENTER_ESP			0x175
 #define IA32_MSR_SYSENTER_EIP			0x176
+#define IA32_MSR_PERF_STATUS			0x198
+#define IA32_MSR_PERF_CTL				0x199
+#define IA32_MSR_TURBO_RATIO_LIMIT		0x1ad
 #define IA32_MSR_ENERGY_PERF_BIAS		0x1b0
 #define IA32_MSR_MTRR_DEFAULT_TYPE		0x2ff
 #define IA32_MSR_MTRR_PHYSICAL_BASE_0	0x200
@@ -152,6 +163,10 @@
 #define IA32_FEATURE_EXT_RDRND		(1 << 30) // RDRAND instruction
 #define IA32_FEATURE_EXT_HYPERVISOR	(1 << 31) // Running on a hypervisor
 
+// x86 features from cpuid eax 0x80000001, ecx register (AMD)
+#define IA32_FEATURE_AMD_EXT_CMPLEGACY	(1 << 1) // Core MP legacy mode
+#define IA32_FEATURE_AMD_EXT_TOPOLOGY	(1 << 22) // Topology extensions
+
 // x86 features from cpuid eax 0x80000001, edx register (AMD)
 // only care about the ones that are unique to this register
 #define IA32_FEATURE_AMD_EXT_SYSCALL	(1 << 11) // SYSCALL/SYSRET
@@ -170,6 +185,10 @@
 											| IA32_FEATURE_AMD_EXT_RDTSCP	\
 											| IA32_FEATURE_AMD_EXT_LONG)
 
+// x86 defined features from cpuid eax 5, ecx register
+#define IA32_FEATURE_POWER_MWAIT		(1 << 0)
+#define IA32_FEATURE_INTERRUPT_MWAIT	(1 << 1)
+
 // x86 defined features from cpuid eax 6, eax register
 // reference http://www.intel.com/Assets/en_US/PDF/appnote/241618.pdf (Table 5-11)
 #define IA32_FEATURE_DTS	(1 << 0) //Digital Thermal Sensor
@@ -184,6 +203,9 @@
 #define IA32_FEATURE_APERFMPERF	(1 << 0) //IA32_APERF, IA32_MPERF
 #define IA32_FEATURE_EPB	(1 << 3) //IA32_ENERGY_PERF_BIAS
 
+// x86 defined features from cpuid eax 0x80000007, edx register
+#define IA32_FEATURE_INVARIANT_TSC		(1 << 8)
+
 // cr4 flags
 #define IA32_CR4_PAE					(1UL << 5)
 #define IA32_CR4_GLOBAL_PAGES			(1UL << 7)
@@ -265,9 +287,12 @@ typedef struct x86_cpu_module_info {
 enum x86_feature_type {
 	FEATURE_COMMON = 0,     // cpuid eax=1, ecx register
 	FEATURE_EXT,            // cpuid eax=1, edx register
+	FEATURE_EXT_AMD_ECX,	// cpuid eax=0x80000001, ecx register (AMD)
 	FEATURE_EXT_AMD,        // cpuid eax=0x80000001, edx register (AMD)
+	FEATURE_5_ECX,			// cpuid eax=5, ecx register
 	FEATURE_6_EAX,          // cpuid eax=6, eax registers
 	FEATURE_6_ECX,          // cpuid eax=6, ecx registers
+	FEATURE_EXT_7_EDX,		// cpuid eax=0x80000007, edx register
 
 	FEATURE_NUM
 };
@@ -301,6 +326,8 @@ typedef struct arch_cpu_info {
 	int					model;
 	int					extended_model;
 
+	uint32				logical_apic_id;
+
 	struct X86PagingStructures* active_paging_structures;
 
 	size_t				dr6;	// temporary storage for debug registers (cf.
@@ -310,13 +337,11 @@ typedef struct arch_cpu_info {
 	struct tss			tss;
 #ifndef __x86_64__
 	struct tss			double_fault_tss;
+	void*				kernel_tls;
 #endif
 } arch_cpu_info;
 
 
-#undef PAUSE
-#define PAUSE() asm volatile ("pause;")
-
 #define nop() __asm__ ("nop"::)
 
 #define x86_read_cr0() ({ \
@@ -410,6 +435,9 @@ typedef struct arch_cpu_info {
 })
 
 
+extern void (*gCpuIdleFunc)(void);
+
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -462,6 +490,20 @@ void x86_fnsave_swap(void* oldFpuState, const void* newFpuState);
 #endif
 
 
+static inline void
+arch_cpu_idle(void)
+{
+	gCpuIdleFunc();
+}
+
+
+static inline void
+arch_cpu_pause(void)
+{
+	asm volatile("pause" : : : "memory");
+}
+
+
 #ifdef __cplusplus
 }	// extern "C" {
 #endif

headers/private/kernel/arch/x86/arch_int.h

@@ -11,6 +11,13 @@
 #define NUM_IO_VECTORS		(256 - ARCH_INTERRUPT_BASE)
 
 
+enum irq_source {
+	IRQ_SOURCE_INVALID,
+	IRQ_SOURCE_IOAPIC,
+	IRQ_SOURCE_MSI,
+};
+
+
 static inline void
 arch_int_enable_interrupts_inline(void)
 {
@@ -68,9 +75,12 @@ typedef struct interrupt_controller_s {
 	bool	(*is_spurious_interrupt)(int32 num);
 	bool	(*is_level_triggered_interrupt)(int32 num);
 	bool	(*end_of_interrupt)(int32 num);
+	void	(*assign_interrupt_to_cpu)(int32 num, int32 cpu);
 } interrupt_controller;
 
 
+void x86_set_irq_source(int irq, irq_source source);
+
 void arch_int_set_interrupt_controller(const interrupt_controller &controller);
 
 #endif // __cplusplus

headers/private/kernel/arch/x86/arch_kernel_args.h

@@ -40,8 +40,8 @@ typedef struct {
 	uint32	apic_phys;
 	FixedWidthPointer<void> apic;
 	uint32	ioapic_phys;
-	uint32	cpu_apic_id[MAX_BOOT_CPUS];
-	uint32	cpu_apic_version[MAX_BOOT_CPUS];
+	uint32	cpu_apic_id[SMP_MAX_CPUS];
+	uint32	cpu_apic_version[SMP_MAX_CPUS];
 	// hpet stuff
 	uint32	hpet_phys;
 	FixedWidthPointer<void> hpet;

headers/private/kernel/arch/x86/arch_smp.h

@@ -99,4 +99,18 @@ enum {
 	MP_INTR_TYPE_ExtINT,
 };
 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+uint32 x86_get_cpu_apic_id(int32 cpu);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+
 #endif	/* _KERNEL_ARCH_x86_ARCH_SMP_H */

headers/private/kernel/arch/x86/arch_system_info.h

@@ -12,7 +12,7 @@
 extern "C" {
 #endif
 
-status_t get_current_cpuid(cpuid_info* info, uint32 eax);
+status_t get_current_cpuid(cpuid_info* info, uint32 eax, uint32 ecx);
 uint32 get_eflags(void);
 void set_eflags(uint32 value);
 

headers/private/kernel/arch/x86/arch_thread.h

@@ -30,9 +30,6 @@ void x86_restart_syscall(struct iframe* frame);
 void x86_set_tls_context(Thread* thread);
 
 
-#ifdef __x86_64__
-
-
 static inline Thread*
 arch_thread_get_current_thread(void)
 {
@@ -42,6 +39,9 @@ arch_thread_get_current_thread(void)
 }
 
 
+#ifdef __x86_64__
+
+
 static inline void
 arch_thread_set_current_thread(Thread* t)
 {
@@ -59,18 +59,10 @@ arch_thread_set_current_thread(Thread* t)
 void arch_syscall_64_bit_return_value(void);
 
 
-static inline Thread*
-arch_thread_get_current_thread(void)
-{
-	Thread* t = (Thread*)x86_read_dr3();
-	return t;
-}
-
-
 static inline void
 arch_thread_set_current_thread(Thread* t)
 {
-	x86_write_dr3(t);
+	asm volatile("mov %0, %%gs:0" : : "r" (t) : "memory");
 }
 
 
@@ -82,3 +74,4 @@ arch_thread_set_current_thread(Thread* t)
 #endif
 
 #endif /* _KERNEL_ARCH_x86_THREAD_H */
+

headers/private/kernel/arch/x86/arch_user_debugger.h

@@ -9,12 +9,7 @@
 #define ARCH_INIT_USER_DEBUG x86_init_user_debug
 
 // number of breakpoints the CPU supports
-// On 32-bit, DR3 is used to hold the Thread*.
-#ifdef __x86_64__
-#	define X86_BREAKPOINT_COUNT		4
-#else
-#	define X86_BREAKPOINT_COUNT		3
-#endif
+#define X86_BREAKPOINT_COUNT		4
 
 // debug status register DR6
 enum {

headers/private/kernel/arch/x86/descriptors.h

@@ -9,6 +9,10 @@
 #define _KERNEL_ARCH_x86_DESCRIPTORS_H
 
 
+#define DPL_KERNEL	0
+#define DPL_USER	3
+
+
 #ifndef _ASSEMBLER
 
 
@@ -18,11 +22,6 @@
 struct kernel_args;
 
 
-enum descriptor_privilege_levels {
-	DPL_KERNEL = 0,
-	DPL_USER = 3,
-};
-
 enum descriptor_types {
 	// segment types
 	DT_CODE_EXECUTE_ONLY = 0x8,
@@ -48,6 +47,7 @@ enum gate_types {
 };
 
 
+void x86_descriptors_preboot_init_percpu(kernel_args* args, int cpu);
 void x86_descriptors_init(kernel_args* args);
 void x86_descriptors_init_percpu(kernel_args* args, int cpu);
 status_t x86_descriptors_init_post_vm(kernel_args* args);

headers/private/kernel/arch/x86/msi.h

@@ -28,5 +28,6 @@ bool		msi_supported();
 status_t	msi_allocate_vectors(uint8 count, uint8 *startVector,
 				uint64 *address, uint16 *data);
 void		msi_free_vectors(uint8 count, uint8 startVector);
+void		msi_assign_interrupt_to_cpu(uint8 irq, int32 cpu);
 
 #endif // _KERNEL_ARCH_x86_MSI_H

headers/private/kernel/boot/kernel_args.h

@@ -59,7 +59,7 @@ typedef struct kernel_args {
 	uint64		ignored_physical_memory;
 
 	uint32		num_cpus;
-	addr_range	cpu_kstack[MAX_BOOT_CPUS];
+	addr_range	cpu_kstack[SMP_MAX_CPUS];
 
 	// boot volume KMessage data
 	FixedWidthPointer<void> boot_volume;

headers/private/kernel/boot/platform/bios_ia32/platform_kernel_args.h

@@ -15,8 +15,8 @@
 #include <util/FixedWidthPointer.h>
 
 
-// must match SMP_MAX_CPUS in arch_smp.h
-#define MAX_BOOT_CPUS 8
+#define SMP_MAX_CPUS 64
+
 #define MAX_PHYSICAL_MEMORY_RANGE 32
 #define MAX_PHYSICAL_ALLOCATED_RANGE 32
 #define MAX_VIRTUAL_ALLOCATED_RANGE 32

headers/private/kernel/condition_variable.h

@@ -56,19 +56,13 @@ public:
 
 			void				Publish(const void* object,
 									const char* objectType);
-			void				Unpublish(bool schedulerLocked = false);
+			void				Unpublish();
 
-	inline	void				NotifyOne(bool schedulerLocked = false,
-									status_t result = B_OK);
-	inline	void				NotifyAll(bool schedulerLocked = false,
-									status_t result = B_OK);
+	inline	void				NotifyOne(status_t result = B_OK);
+	inline	void				NotifyAll(status_t result = B_OK);
 
-	static	void				NotifyOne(const void* object,
-									bool schedulerLocked = false,
-									status_t result = B_OK);
-	static	void				NotifyAll(const void* object,
-									bool schedulerLocked = false,
-									status_t result = B_OK);
+	static	void				NotifyOne(const void* object, status_t result);
+	static	void				NotifyAll(const void* object, status_t result);
 									// (both methods) caller must ensure that
 									// the variable is not unpublished
 									// concurrently
@@ -86,8 +80,7 @@ public:
 			void				Dump() const;
 
 private:
-			void				_Notify(bool all, bool schedulerLocked,
-									status_t result);
+			void				_Notify(bool all, status_t result);
 			void				_NotifyLocked(bool all, status_t result);
 
 protected:
@@ -124,16 +117,16 @@ ConditionVariableEntry::~ConditionVariableEntry()
 
 
 inline void
-ConditionVariable::NotifyOne(bool schedulerLocked, status_t result)
+ConditionVariable::NotifyOne(status_t result)
 {
-	_Notify(false, schedulerLocked, result);
+	_Notify(false, result);
 }
 
 
 inline void
-ConditionVariable::NotifyAll(bool schedulerLocked, status_t result)
+ConditionVariable::NotifyAll(status_t result)
 {
-	_Notify(true, schedulerLocked, result);
+	_Notify(true, result);
 }
 
 

headers/private/kernel/cpu.h

@@ -11,15 +11,12 @@
 
 #include <setjmp.h>
 
+#include <int.h>
 #include <smp.h>
 #include <timer.h>
 #include <arch/cpu.h>
 
-
-// define PAUSE, if not done in arch/cpu.h
-#ifndef PAUSE
-#	define PAUSE()
-#endif
+#include <scheduler.h>
 
 
 struct kernel_args;
@@ -31,20 +28,43 @@ namespace BKernel {
 using BKernel::Thread;
 
 
+typedef enum cpu_topology_level {
+	CPU_TOPOLOGY_SMT,
+	CPU_TOPOLOGY_CORE,
+	CPU_TOPOLOGY_PACKAGE,
+	//
+	CPU_TOPOLOGY_LEVELS
+} cpu_topology_level;
+
+typedef struct cpu_topology_node {
+	cpu_topology_level	level;
+
+	int					id;
+
+	cpu_topology_node**	children;
+	int					children_count;
+} cpu_topology_node;
+
+
 /* CPU local data structure */
 
 typedef struct cpu_ent {
 	int				cpu_num;
 
 	// thread.c: used to force a reschedule at quantum expiration time
-	int				preempted;
+	bool			preempted;
 	timer			quantum_timer;
 
 	// keeping track of CPU activity
+	seqlock			active_time_lock;
 	bigtime_t		active_time;
+	bigtime_t		irq_time;
+	bigtime_t		interrupt_time;
 	bigtime_t		last_kernel_time;
 	bigtime_t		last_user_time;
 
+	int32			ici_counter;
+
 	// used in the kernel debugger
 	addr_t			fault_handler;
 	addr_t			fault_handler_stack_pointer;
@@ -53,16 +73,24 @@ typedef struct cpu_ent {
 	Thread*			running_thread;
 	Thread*			previous_thread;
 	bool			invoke_scheduler;
-	bool			invoke_scheduler_if_idle;
 	bool			disabled;
 
+	// CPU topology information
+	int				topology_id[CPU_TOPOLOGY_LEVELS];
+	int				cache_id[CPU_MAX_CACHE_LEVEL];
+
+	// IRQs assigned to this CPU
+	struct list		irqs;
+	spinlock		irqs_lock;
+
 	// arch-specific stuff
-	arch_cpu_info arch;
-} cpu_ent __attribute__((aligned(64)));
+	arch_cpu_info 	arch;
+} cpu_ent CACHE_LINE_ALIGN;
 
 
 //extern cpu_ent gCPU[MAX_BOOT_CPUS];
 extern cpu_ent gCPU[];
+extern uint32 gCPUCacheLevelCount;
 
 
 #ifdef __cplusplus
@@ -79,6 +107,25 @@ bigtime_t cpu_get_active_time(int32 cpu);
 cpu_ent *get_cpu_struct(void);
 extern inline cpu_ent *get_cpu_struct(void) { return &gCPU[smp_get_current_cpu()]; }
 
+status_t cpu_build_topology_tree(void);
+const cpu_topology_node* get_cpu_topology(void);
+
+void cpu_set_scheduler_mode(enum scheduler_mode mode);
+
+status_t increase_cpu_performance(int delta);
+status_t decrease_cpu_performance(int delta);
+
+void cpu_idle(void);
+void cpu_wait(int32* variable, int32 test);
+
+
+static inline void
+cpu_pause(void)
+{
+	arch_cpu_pause();
+}
+
+
 void _user_clear_caches(void *address, size_t length, uint32 flags);
 bool _user_cpu_enabled(int32 cpu);
 status_t _user_set_cpu_enabled(int32 cpu, bool enabled);

headers/private/kernel/int.h

@@ -12,6 +12,8 @@
 #include <KernelExport.h>
 #include <arch/int.h>
 
+#include <util/list.h>
+
 // private install_io_interrupt_handler() flags
 #define B_NO_LOCK_VECTOR	0x100
 #define B_NO_HANDLED_INFO	0x200
@@ -19,6 +21,28 @@
 struct kernel_args;
 
 
+enum interrupt_type {
+	INTERRUPT_TYPE_EXCEPTION,
+	INTERRUPT_TYPE_IRQ,
+	INTERRUPT_TYPE_LOCAL_IRQ,
+	INTERRUPT_TYPE_SYSCALL,
+	INTERRUPT_TYPE_ICI,
+	INTERRUPT_TYPE_UNKNOWN
+};
+
+struct irq_assignment {
+	list_link	link;
+
+	uint32		irq;
+	uint32		count;
+
+	int32		handlers_count;
+
+	int32		load;
+	int32		cpu;
+};
+
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -53,8 +77,12 @@ are_interrupts_enabled(void)
 #define restore_interrupts(status)	arch_int_restore_interrupts(status)
 
 
-status_t reserve_io_interrupt_vectors(long count, long startVector);
-status_t allocate_io_interrupt_vectors(long count, long *startVector);
+status_t reserve_io_interrupt_vectors(long count, long startVector,
+	enum interrupt_type type);
+status_t allocate_io_interrupt_vectors(long count, long *startVector,
+	enum interrupt_type type);
 void free_io_interrupt_vectors(long count, long startVector);
 
+void assign_io_interrupt_to_cpu(long vector, int32 cpu);
+
 #endif /* _KERNEL_INT_H */

headers/private/kernel/kscheduler.h

@@ -1,4 +1,5 @@
 /*
+ * Copyright 2013, Paweł Dziepak, [email protected].
  * Copyright 2008-2011, Ingo Weinhold, [email protected].
  * Copyright 2005-2010, Axel Dörfler, [email protected].
  * Distributed under the terms of the MIT License.
@@ -17,78 +18,69 @@ struct scheduling_analysis;
 struct SchedulerListener;
 
 
-struct scheduler_ops {
-	/*!	Enqueues the thread in the ready-to-run queue.
-		The caller must hold the scheduler lock (with disabled interrupts).
-	*/
-	void (*enqueue_in_run_queue)(Thread* thread);
-
-	/*!	Selects a thread from the ready-to-run queue and, if that's not the
-		calling thread, switches the current CPU's context to run the selected
-		thread.
-		If it's the same thread, the thread will just continue to run.
-		In either case, unless the thread is dead or is sleeping/waiting
-		indefinitely, the function will eventually return.
-		The caller must hold the scheduler lock (with disabled interrupts).
-	*/
-	void (*reschedule)(void);
-
-	/*!	Sets the given thread's priority.
-		The thread may be running or may be in the ready-to-run queue.
-		The caller must hold the scheduler lock (with disabled interrupts).
-	*/
-	void (*set_thread_priority)(Thread* thread, int32 priority);
-	bigtime_t (*estimate_max_scheduling_latency)(Thread* thread);
-
-	/*!	Called when the Thread structure is first created.
-		Per-thread housekeeping resources can be allocated.
-		Interrupts must be enabled.
-	*/
-	status_t (*on_thread_create)(Thread* thread, bool idleThread);
-
-	/*!	Called when a Thread structure is initialized and made ready for
-		use.
-		The per-thread housekeeping data structures are reset, if needed.
-		The caller must hold the scheduler lock (with disabled interrupts).
-	*/
-	void (*on_thread_init)(Thread* thread);
-
-	/*!	Called when a Thread structure is freed.
-		Frees up any per-thread resources allocated on the scheduler's part. The
-		function may be called even if on_thread_create() failed.
-		Interrupts must be enabled.
-	*/
-	void (*on_thread_destroy)(Thread* thread);
-
-	/*!	Called in the early boot process to start thread scheduling on the
-		current CPU.
-		The function is called once for each CPU.
-		Interrupts must be disabled, but the caller must not hold the scheduler
-		lock.
-	*/
-	void (*start)(void);
-};
-
-extern struct scheduler_ops* gScheduler;
-extern spinlock gSchedulerLock;
-
-#define scheduler_enqueue_in_run_queue(thread)	\
-				gScheduler->enqueue_in_run_queue(thread)
-#define scheduler_set_thread_priority(thread, priority)	\
-				gScheduler->set_thread_priority(thread, priority)
-#define scheduler_reschedule()	gScheduler->reschedule()
-#define scheduler_start()		gScheduler->start()
-#define scheduler_on_thread_create(thread, idleThread) \
-				gScheduler->on_thread_create(thread, idleThread)
-#define scheduler_on_thread_init(thread) \
-				gScheduler->on_thread_init(thread)
-#define scheduler_on_thread_destroy(thread) \
-				gScheduler->on_thread_destroy(thread)
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+/*!	Enqueues the thread in the ready-to-run queue.
+	The caller must hold the enqueued thread \c scheduler_lock.
+*/
+void scheduler_enqueue_in_run_queue(Thread* thread);
+
+void scheduler_reschedule_ici(void);
+
+/*!	Selects a thread from the ready-to-run queue and, if that's not the
+	calling thread, switches the current CPU's context to run the selected
+	thread.
+	If it's the same thread, the thread will just continue to run.
+	In either case, unless the thread is dead or is sleeping/waiting
+	indefinitely, the function will eventually return.
+	The caller must hold the current thread \c scheduler_lock.
+*/
+void scheduler_reschedule(int32 next_state);
+
+/*!	Sets the given thread's priority.
+	The thread may be running or may be in the ready-to-run queue.
+*/
+int32 scheduler_set_thread_priority(Thread* thread, int32 priority);
+
+/*!	Called when the Thread structure is first created.
+	Per-thread housekeeping resources can be allocated.
+	Interrupts must be enabled.
+*/
+status_t scheduler_on_thread_create(Thread* thread, bool idleThread);
+
+/*!	Called when a Thread structure is initialized and made ready for
+	use.
+	The per-thread housekeeping data structures are reset, if needed.
+*/
+void scheduler_on_thread_init(Thread* thread);
+
+/*!	Called when a Thread structure is freed.
+	Frees up any per-thread resources allocated on the scheduler's part. The
+	function may be called even if on_thread_create() failed.
+	Interrupts must be enabled.
+*/
+void scheduler_on_thread_destroy(Thread* thread);
+
+/*!	Called in the early boot process to start thread scheduling on the
+	current CPU.
+	The function is called once for each CPU.
+*/
+void scheduler_start(void);
+
+/*! Sets scheduler operation mode.
+ */
+status_t scheduler_set_operation_mode(scheduler_mode mode);
+
+/*! Dumps scheduler specific thread information.
+*/
+void scheduler_dump_thread_data(Thread* thread);
+
+void scheduler_new_thread_entry(Thread* thread);
+
+void scheduler_set_cpu_enabled(int32 cpu, bool enabled);
+
 void scheduler_add_listener(struct SchedulerListener* listener);
 void scheduler_remove_listener(struct SchedulerListener* listener);
 
@@ -99,6 +91,9 @@ bigtime_t _user_estimate_max_scheduling_latency(thread_id thread);
 status_t _user_analyze_scheduling(bigtime_t from, bigtime_t until, void* buffer,
 	size_t size, struct scheduling_analysis* analysis);
 
+status_t _user_set_scheduler_mode(int32 mode);
+int32 _user_get_scheduler_mode(void);
+
 #ifdef __cplusplus
 }
 #endif
@@ -111,7 +106,7 @@ static inline void
 scheduler_reschedule_if_necessary_locked()
 {
 	if (gCPU[smp_get_current_cpu()].invoke_scheduler)
-		scheduler_reschedule();
+		scheduler_reschedule(B_THREAD_READY);
 }
 
 
@@ -123,11 +118,14 @@ scheduler_reschedule_if_necessary()
 {
 	if (are_interrupts_enabled()) {
 		cpu_status state = disable_interrupts();
-		acquire_spinlock(&gSchedulerLock);
+
+		Thread* thread = get_cpu_struct()->running_thread;
+		acquire_spinlock(&thread->scheduler_lock);
 
 		scheduler_reschedule_if_necessary_locked();
 
-		release_spinlock(&gSchedulerLock);
+		release_spinlock(&thread->scheduler_lock);
+
 		restore_interrupts(state);
 	}
 }

headers/private/kernel/ksystem_info.h

@@ -21,7 +21,11 @@ status_t system_info_init(struct kernel_args *args);
 status_t system_notifications_init();
 const char* get_haiku_revision(void);
 
-status_t _user_get_system_info(system_info *userInfo, size_t size);
+status_t _user_get_system_info(system_info *userInfo);
+status_t _user_get_cpu_info(uint32 firstCPU, uint32 cpuCount, cpu_info* info);
+status_t _user_get_cpu_topology_info(cpu_topology_node_info* topologyInfos,
+				uint32* topologyInfoCount);
+
 status_t _user_get_system_info_etc(int32 id, void *buffer,
 			size_t bufferSize);
 

headers/private/kernel/listeners.h

@@ -34,7 +34,7 @@ struct SchedulerListener : DoublyLinkedListLinkImpl<SchedulerListener> {
 
 typedef DoublyLinkedList<SchedulerListener> SchedulerListenerList;
 extern SchedulerListenerList gSchedulerListeners;
-	// guarded by the thread spinlock
+extern spinlock gSchedulerListenersLock;
 
 
 template<typename Parameter1>

headers/private/kernel/load_tracking.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2013 Paweł Dziepak, [email protected].
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_LOAD_TRACKING_H
+#define _KERNEL_LOAD_TRACKING_H
+
+
+#include <OS.h>
+
+
+const int32 kMaxLoad = 1000;
+const bigtime_t kLoadMeasureInterval = 50000;
+const bigtime_t kIntervalInaccuracy = kLoadMeasureInterval / 4;
+
+
+static inline int32
+compute_load(bigtime_t& measureTime, bigtime_t& measureActiveTime, int32& load,
+	bigtime_t now)
+{
+	if (measureTime == 0) {
+		measureTime = now;
+		return -1;
+	}
+
+	bigtime_t deltaTime = now - measureTime;
+
+	if (deltaTime < kLoadMeasureInterval)
+		return -1;
+
+	int32 oldLoad = load;
+	ASSERT(oldLoad >= 0 && oldLoad <= kMaxLoad);
+
+	int32 newLoad = measureActiveTime * kMaxLoad;
+	newLoad /= max_c(deltaTime, 1);
+	newLoad = max_c(min_c(newLoad, kMaxLoad), 0);
+
+	measureActiveTime = 0;
+	measureTime = now;
+
+	deltaTime += kIntervalInaccuracy;
+	int n = deltaTime / kLoadMeasureInterval;
+	ASSERT(n > 0);
+
+	if (n > 10)
+		load = newLoad;
+	else {
+		newLoad *= (1 << n) - 1;
+		load = (load + newLoad) / (1 << n);
+		ASSERT(load >= 0 && load <= kMaxLoad);
+	}
+
+	return oldLoad;
+}
+
+
+#endif	// _KERNEL_LOAD_TRACKING_H

headers/private/kernel/lock.h

@@ -9,7 +9,10 @@
 #ifndef _KERNEL_LOCK_H
 #define _KERNEL_LOCK_H
 
+
 #include <OS.h>
+
+#include <arch/atomic.h>
 #include <debug.h>
 
 
@@ -18,6 +21,7 @@ struct mutex_waiter;
 typedef struct mutex {
 	const char*				name;
 	struct mutex_waiter*	waiters;
+	spinlock				lock;
 #if KDEBUG
 	thread_id				holder;
 #else
@@ -44,8 +48,9 @@ struct rw_lock_waiter;
 typedef struct rw_lock {
 	const char*				name;
 	struct rw_lock_waiter*	waiters;
+	spinlock				lock;
 	thread_id				holder;
-	vint32					count;
+	int32					count;
 	int32					owner_count;
 	int16					active_readers;
 								// Only > 0 while a writer is waiting: number
@@ -88,14 +93,17 @@ typedef struct rw_lock {
 
 // static initializers
 #if KDEBUG
-#	define MUTEX_INITIALIZER(name)			{ name, NULL, -1, 0 }
+#	define MUTEX_INITIALIZER(name) \
+	{ name, NULL, B_SPINLOCK_INITIALIZER, -1, 0 }
 #	define RECURSIVE_LOCK_INITIALIZER(name)	{ MUTEX_INITIALIZER(name), 0 }
 #else
-#	define MUTEX_INITIALIZER(name)			{ name, NULL, 0, 0, 0 }
+#	define MUTEX_INITIALIZER(name) \
+	{ name, NULL, B_SPINLOCK_INITIALIZER, 0, 0, 0 }
 #	define RECURSIVE_LOCK_INITIALIZER(name)	{ MUTEX_INITIALIZER(name), -1, 0 }
 #endif
 
-#define RW_LOCK_INITIALIZER(name)			{ name, NULL, -1, 0, 0, 0 }
+#define RW_LOCK_INITIALIZER(name) \
+	{ name, NULL, B_SPINLOCK_INITIALIZER, -1, 0, 0, 0 }
 
 
 #if KDEBUG
@@ -144,11 +152,11 @@ extern status_t mutex_switch_from_read_lock(rw_lock* from, mutex* to);
 extern status_t _rw_lock_read_lock(rw_lock* lock);
 extern status_t _rw_lock_read_lock_with_timeout(rw_lock* lock,
 	uint32 timeoutFlags, bigtime_t timeout);
-extern void _rw_lock_read_unlock(rw_lock* lock, bool schedulerLocked);
-extern void _rw_lock_write_unlock(rw_lock* lock, bool schedulerLocked);
+extern void _rw_lock_read_unlock(rw_lock* lock);
+extern void _rw_lock_write_unlock(rw_lock* lock);
 
-extern status_t _mutex_lock(mutex* lock, bool schedulerLocked);
-extern void _mutex_unlock(mutex* lock, bool schedulerLocked);
+extern status_t _mutex_lock(mutex* lock, void* locker);
+extern void _mutex_unlock(mutex* lock);
 extern status_t _mutex_trylock(mutex* lock);
 extern status_t _mutex_lock_with_timeout(mutex* lock, uint32 timeoutFlags,
 	bigtime_t timeout);
@@ -191,7 +199,7 @@ rw_lock_read_unlock(rw_lock* lock)
 #else
 	int32 oldCount = atomic_add(&lock->count, -1);
 	if (oldCount >= RW_LOCK_WRITER_COUNT_BASE)
-		_rw_lock_read_unlock(lock, false);
+		_rw_lock_read_unlock(lock);
 #endif
 }
 
@@ -199,7 +207,7 @@ rw_lock_read_unlock(rw_lock* lock)
 static inline void
 rw_lock_write_unlock(rw_lock* lock)
 {
-	_rw_lock_write_unlock(lock, false);
+	_rw_lock_write_unlock(lock);
 }
 
 
@@ -207,23 +215,10 @@ static inline status_t
 mutex_lock(mutex* lock)
 {
 #if KDEBUG
-	return _mutex_lock(lock, false);
+	return _mutex_lock(lock, NULL);
 #else
 	if (atomic_add(&lock->count, -1) < 0)
-		return _mutex_lock(lock, false);
-	return B_OK;
-#endif
-}
-
-
-static inline status_t
-mutex_lock_threads_locked(mutex* lock)
-{
-#if KDEBUG
-	return _mutex_lock(lock, true);
-#else
-	if (atomic_add(&lock->count, -1) < 0)
-		return _mutex_lock(lock, true);
+		return _mutex_lock(lock, NULL);
 	return B_OK;
 #endif
 }
@@ -261,7 +256,7 @@ mutex_unlock(mutex* lock)
 #if !KDEBUG
 	if (atomic_add(&lock->count, 1) < -1)
 #endif
-		_mutex_unlock(lock, false);
+		_mutex_unlock(lock);
 }
 
 

headers/private/kernel/smp.h

@@ -9,8 +9,15 @@
 #define KERNEL_SMP_H
 
 
+#include <arch/atomic.h>
+#include <boot/kernel_args.h>
+#include <kernel.h>
+
 #include <KernelExport.h>
 
+#include <string.h>
+
+
 struct kernel_args;
 
 
@@ -22,8 +29,7 @@ enum {
 	SMP_MSG_GLOBAL_INVALIDATE_PAGES,
 	SMP_MSG_CPU_HALT,
 	SMP_MSG_CALL_FUNCTION,
-	SMP_MSG_RESCHEDULE,
-	SMP_MSG_RESCHEDULE_IF_IDLE
+	SMP_MSG_RESCHEDULE
 };
 
 enum {
@@ -32,10 +38,28 @@ enum {
 	SMP_MSG_FLAG_FREE_ARG	= 0x2,
 };
 
-typedef uint32 cpu_mask_t;
-
 typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3);
 
+class CPUSet {
+public:
+	inline				CPUSet();
+
+	inline	void		ClearAll();
+	inline	void		SetAll();
+
+	inline	void		SetBit(int32 cpu);
+	inline	void		ClearBit(int32 cpu);
+
+	inline	bool		GetBit(int32 cpu) const;
+
+	inline	bool		IsEmpty() const;
+
+private:
+	static	const int	kArraySize = ROUNDUP(SMP_MAX_CPUS, 32) / 32;
+
+			uint32		fBitmap[kArraySize];
+};
+
 
 #ifdef __cplusplus
 extern "C" {
@@ -48,10 +72,10 @@ status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu);
 status_t smp_init_post_generic_syscalls(void);
 bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous);
 void smp_wake_up_non_boot_cpus(void);
-void smp_cpu_rendezvous(volatile uint32 *var, int current_cpu);
+void smp_cpu_rendezvous(uint32* var);
 void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3,
 		void *data_ptr, uint32 flags);
-void smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data,
+void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
 		addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
 void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
 		void *data_ptr, uint32 flags);
@@ -69,6 +93,63 @@ int smp_intercpu_int_handler(int32 cpu);
 #endif
 
 
+inline
+CPUSet::CPUSet()
+{
+	memset(fBitmap, 0, sizeof(fBitmap));
+}
+
+
+inline void
+CPUSet::ClearAll()
+{
+	memset(fBitmap, 0, sizeof(fBitmap));
+}
+
+
+inline void
+CPUSet::SetAll()
+{
+	memset(fBitmap, ~uint8(0), sizeof(fBitmap));
+}
+
+
+inline void
+CPUSet::SetBit(int32 cpu)
+{
+	int32* element = (int32*)&fBitmap[cpu % kArraySize];
+	atomic_or(element, 1u << (cpu / kArraySize));
+}
+
+
+inline void
+CPUSet::ClearBit(int32 cpu)
+{
+	int32* element = (int32*)&fBitmap[cpu % kArraySize];
+	atomic_and(element, ~uint32(1u << (cpu / kArraySize)));
+}
+
+
+inline bool
+CPUSet::GetBit(int32 cpu) const
+{
+	int32* element = (int32*)&fBitmap[cpu % kArraySize];
+	return ((uint32)atomic_get(element) & (1u << (cpu / kArraySize))) != 0;
+}
+
+
+inline bool
+CPUSet::IsEmpty() const
+{
+	for (int i = 0; i < kArraySize; i++) {
+		if (fBitmap[i] != 0)
+			return false;
+	}
+
+	return true;
+}
+
+
 // Unless spinlock debug features are enabled, try to inline
 // {acquire,release}_spinlock().
 #if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION
@@ -77,7 +158,7 @@ int smp_intercpu_int_handler(int32 cpu);
 static inline bool
 try_acquire_spinlock_inline(spinlock* lock)
 {
-	return atomic_or((int32*)lock, 1) == 0;
+	return atomic_get_and_set((int32*)lock, 1) == 0;
 }
 
 
@@ -93,7 +174,7 @@ acquire_spinlock_inline(spinlock* lock)
 static inline void
 release_spinlock_inline(spinlock* lock)
 {
-	atomic_and((int32*)lock, 0);
+	atomic_set((int32*)lock, 0);
 }
 
 
@@ -101,6 +182,108 @@ release_spinlock_inline(spinlock* lock)
 #define acquire_spinlock(lock)		acquire_spinlock_inline(lock)
 #define release_spinlock(lock)		release_spinlock_inline(lock)
 
+
+static inline bool
+try_acquire_write_spinlock_inline(rw_spinlock* lock)
+{
+	return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0;
+}
+
+
+static inline void
+acquire_write_spinlock_inline(rw_spinlock* lock)
+{
+	if (try_acquire_write_spinlock(lock))
+		return;
+	acquire_write_spinlock(lock);
+}
+
+
+static inline void
+release_write_spinlock_inline(rw_spinlock* lock)
+{
+	atomic_set(&lock->lock, 0);
+}
+
+
+static inline bool
+try_acquire_read_spinlock_inline(rw_spinlock* lock)
+{
+	uint32 previous = atomic_add(&lock->lock, 1);
+	return (previous & (1u << 31)) == 0;
+}
+
+
+static inline void
+acquire_read_spinlock_inline(rw_spinlock* lock)
+{
+	if (try_acquire_read_spinlock(lock))
+		return;
+	acquire_read_spinlock(lock);
+}
+
+
+static inline void
+release_read_spinlock_inline(rw_spinlock* lock)
+{
+	atomic_add(&lock->lock, -1);
+}
+
+
+#define try_acquire_read_spinlock(lock)	try_acquire_read_spinlock_inline(lock)
+#define acquire_read_spinlock(lock)		acquire_read_spinlock_inline(lock)
+#define release_read_spinlock(lock)		release_read_spinlock_inline(lock)
+#define try_acquire_write_spinlock(lock) \
+	try_acquire_write_spinlock(lock)
+#define acquire_write_spinlock(lock)	acquire_write_spinlock_inline(lock)
+#define release_write_spinlock(lock)	release_write_spinlock_inline(lock)
+
+
+static inline bool
+try_acquire_write_seqlock_inline(seqlock* lock) {
+	bool succeed = try_acquire_spinlock(&lock->lock);
+	if (succeed)
+		atomic_add((int32*)&lock->count, 1);
+	return succeed;
+}
+
+
+static inline void
+acquire_write_seqlock_inline(seqlock* lock) {
+	acquire_spinlock(&lock->lock);
+	atomic_add((int32*)&lock->count, 1);
+}
+
+
+static inline void
+release_write_seqlock_inline(seqlock* lock) {
+	atomic_add((int32*)&lock->count, 1);
+	release_spinlock(&lock->lock);
+}
+
+
+static inline uint32
+acquire_read_seqlock_inline(seqlock* lock) {
+	return atomic_get((int32*)&lock->count);
+}
+
+
+static inline bool
+release_read_seqlock_inline(seqlock* lock, uint32 count) {
+	uint32 current = atomic_get((int32*)&lock->count);
+
+	return count % 2 == 0 && current == count;
+}
+
+
+#define try_acquire_write_seqlock(lock)	try_acquire_write_seqlock_inline(lock)
+#define acquire_write_seqlock(lock)		acquire_write_seqlock_inline(lock)
+#define release_write_seqlock(lock)		release_write_seqlock_inline(lock)
+#define acquire_read_seqlock(lock)		acquire_read_seqlock_inline(lock)
+#define release_read_seqlock(lock, count)	\
+	release_read_seqlock_inline(lock, count)
+
+
 #endif	// !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION
 
 

headers/private/kernel/team.h

@@ -46,7 +46,7 @@ thread_id load_image_etc(int32 argCount, const char* const* args,
 	const char* const* env, int32 priority, team_id parentID, uint32 flags);
 
 void team_set_job_control_state(Team* team, job_control_state newState,
-			Signal* signal, bool threadsLocked);
+			Signal* signal);
 void team_set_controlling_tty(int32 index);
 int32 team_get_controlling_tty();
 status_t team_set_foreground_process_group(int32 ttyIndex, pid_t processGroup);

headers/private/kernel/thread.h

@@ -11,12 +11,13 @@
 
 
 #include <OS.h>
-#include <thread_types.h>
-#include <arch/thread.h>
 
+#include <arch/atomic.h>
+#include <arch/thread.h>
 // For the thread blocking inline functions only.
 #include <kscheduler.h>
 #include <ksignal.h>
+#include <thread_types.h>
 
 
 struct arch_fork_arg;
@@ -69,15 +70,13 @@ public:
 using BKernel::ThreadCreationAttributes;
 
 
+extern spinlock gThreadCreationLock;
+
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-void thread_enqueue(Thread *t, struct thread_queue *q);
-Thread *thread_lookat_queue(struct thread_queue *q);
-Thread *thread_dequeue(struct thread_queue *q);
-Thread *thread_dequeue_id(struct thread_queue *q, thread_id id);
-
 void thread_at_kernel_entry(bigtime_t now);
 	// called when the thread enters the kernel on behalf of the thread
 void thread_at_kernel_exit(void);
@@ -86,9 +85,11 @@ void thread_reset_for_exec(void);
 
 status_t thread_init(struct kernel_args *args);
 status_t thread_preboot_init_percpu(struct kernel_args *args, int32 cpuNum);
-void thread_yield(bool force);
+void thread_yield(void);
 void thread_exit(void);
 
+void thread_map(void (*function)(Thread* thread, void* data), void* data);
+
 int32 thread_max_threads(void);
 int32 thread_used_threads(void);
 
@@ -135,8 +136,7 @@ status_t deselect_thread(int32 object, struct select_info *info, bool kernel);
 
 status_t thread_block();
 status_t thread_block_with_timeout(uint32 timeoutFlags, bigtime_t timeout);
-status_t thread_block_with_timeout_locked(uint32 timeoutFlags,
-			bigtime_t timeout);
+void thread_unblock(Thread* thread, status_t status);
 
 // used in syscalls.c
 status_t _user_set_thread_priority(thread_id thread, int32 newPriority);
@@ -212,7 +212,7 @@ thread_is_interrupted(Thread* thread, uint32 flags)
 static inline bool
 thread_is_blocked(Thread* thread)
 {
-	return thread->wait.status == 1;
+	return atomic_get(&thread->wait.status) == 1;
 }
 
 
@@ -234,22 +234,22 @@ thread_is_blocked(Thread* thread)
 	If a client lock other than the scheduler lock is used, this function must
 	be called with that lock being held. Afterwards that lock should be dropped
 	and the function that actually blocks the thread shall be invoked
-	(thread_block[_locked]() or thread_block_with_timeout[_locked]()). In
-	between these two steps no functionality that uses the thread blocking API
-	for this thread shall be used.
+	(thread_block[_locked]() or thread_block_with_timeout()). In between these
+	two steps no functionality that uses the thread blocking API for this thread
+	shall be used.
 
 	When the caller determines that the condition for unblocking the thread
 	occurred, it calls thread_unblock_locked() to unblock the thread. At that
 	time one of locks that are held when calling thread_prepare_to_block() must
 	be held. Usually that would be the client lock. In two cases it generally
 	isn't, however, since the unblocking code doesn't know about the client
-	lock: 1. When thread_block_with_timeout[_locked]() had been used and the
-	timeout occurs. 2. When thread_prepare_to_block() had been called with one
-	or both of the \c B_CAN_INTERRUPT or \c B_KILL_CAN_INTERRUPT flags specified
-	and someone calls thread_interrupt() that is supposed to wake up the thread.
+	lock: 1. When thread_block_with_timeout() had been used and the timeout
+	occurs. 2. When thread_prepare_to_block() had been called with one or both
+	of the \c B_CAN_INTERRUPT or \c B_KILL_CAN_INTERRUPT flags specified and
+	someone calls thread_interrupt() that is supposed to wake up the thread.
 	In either of these two cases only the scheduler lock is held by the
 	unblocking code. A timeout can only happen after
-	thread_block_with_timeout_locked() has been called, but an interruption is
+	thread_block_with_timeout() has been called, but an interruption is
 	possible at any time. The client code must deal with those situations.
 
 	Generally blocking and unblocking threads proceed in the following manner:
@@ -333,39 +333,6 @@ thread_prepare_to_block(Thread* thread, uint32 flags, uint32 type,
 }
 
 
-/*!	Blocks the current thread.
-
-	The thread is blocked until someone else unblock it. Must be called after a
-	call to thread_prepare_to_block(). If the thread has already been unblocked
-	after the previous call to thread_prepare_to_block(), this function will
-	return immediately. Cf. the documentation of thread_prepare_to_block() for
-	more details.
-
-	The caller must hold the scheduler lock.
-
-	\param thread The current thread.
-	\return The error code passed to the unblocking function. thread_interrupt()
-		uses \c B_INTERRUPTED. By convention \c B_OK means that the wait was
-		successful while another error code indicates a failure (what that means
-		depends on the client code).
-*/
-static inline status_t
-thread_block_locked(Thread* thread)
-{
-	if (thread->wait.status == 1) {
-		// check for signals, if interruptible
-		if (thread_is_interrupted(thread, thread->wait.flags)) {
-			thread->wait.status = B_INTERRUPTED;
-		} else {
-			thread->next_state = B_THREAD_WAITING;
-			scheduler_reschedule();
-		}
-	}
-
-	return thread->wait.status;
-}
-
-
 /*!	Unblocks the specified blocked thread.
 
 	If the thread is no longer waiting (e.g. because thread_unblock_locked() has
@@ -417,10 +384,12 @@ thread_unblock_locked(Thread* thread, status_t status)
 static inline status_t
 thread_interrupt(Thread* thread, bool kill)
 {
-	if ((thread->wait.flags & B_CAN_INTERRUPT) != 0
-		|| (kill && (thread->wait.flags & B_KILL_CAN_INTERRUPT) != 0)) {
-		thread_unblock_locked(thread, B_INTERRUPTED);
-		return B_OK;
+	if (thread_is_blocked(thread)) {
+		if ((thread->wait.flags & B_CAN_INTERRUPT) != 0
+			|| (kill && (thread->wait.flags & B_KILL_CAN_INTERRUPT) != 0)) {
+			thread_unblock_locked(thread, B_INTERRUPTED);
+			return B_OK;
+		}
 	}
 
 	return B_NOT_ALLOWED;

headers/private/kernel/thread_types.h

@@ -57,12 +57,15 @@ struct cpu_ent;
 struct image;					// defined in image.c
 struct io_context;
 struct realtime_sem_context;	// defined in realtime_sem.cpp
-struct scheduler_thread_data;
 struct select_info;
 struct user_thread;				// defined in libroot/user_thread.h
 struct VMAddressSpace;
 struct xsi_sem_context;			// defined in xsi_semaphore.cpp
 
+namespace Scheduler {
+	struct ThreadData;
+}
+
 namespace BKernel {
 	struct Team;
 	struct Thread;
@@ -242,10 +245,10 @@ struct Team : TeamThreadIteratorEntry<team_id>, KernelReferenceable,
 	struct job_control_entry* job_control_entry;
 
 	VMAddressSpace	*address_space;
-	Thread			*main_thread;	// protected by fLock and the scheduler
-									// lock (and the thread's lock), immutable
+	Thread			*main_thread;	// protected by fLock, immutable
 									// after first set
-	Thread			*thread_list;	// protected by fLock and the scheduler lock
+	Thread			*thread_list;	// protected by fLock, signal_lock and
+									// gThreadCreationLock
 	struct team_loading_info *loading_info;	// protected by fLock
 	struct list		image_list;		// protected by sImageMutex
 	struct list		watcher_list;
@@ -263,13 +266,13 @@ struct Team : TeamThreadIteratorEntry<team_id>, KernelReferenceable,
 
 	struct team_debug_info debug_info;
 
-	// protected by scheduler lock
+	// protected by time_lock
 	bigtime_t		dead_threads_kernel_time;
 	bigtime_t		dead_threads_user_time;
 	bigtime_t		cpu_clock_offset;
+	spinlock		time_lock;
 
-	// user group information; protected by fLock, the *_uid/*_gid fields also
-	// by the scheduler lock
+	// user group information; protected by fLock
 	uid_t			saved_set_uid;
 	uid_t			real_uid;
 	uid_t			effective_uid;
@@ -290,6 +293,8 @@ struct Team : TeamThreadIteratorEntry<team_id>, KernelReferenceable,
 		bool		initialized;	// true when the state has been initialized
 	} exit;
 
+	spinlock		signal_lock;
+
 public:
 								~Team();
 
@@ -397,7 +402,7 @@ private:
 
 			BKernel::QueuedSignalsCounter* fQueuedSignalsCounter;
 			BKernel::PendingSignals	fPendingSignals;
-									// protected by scheduler lock
+									// protected by signal_lock
 			struct sigaction 	fSignalActions[MAX_SIGNAL_NUMBER];
 									// indexed signal - 1, protected by fLock
 
@@ -405,7 +410,7 @@ private:
 			TeamTimeUserTimerList fCPUTimeUserTimers;
 									// protected by scheduler lock
 			TeamUserTimeUserTimerList fUserTimeUserTimers;
-			vint32				fUserDefinedTimerCount;	// accessed atomically
+			int32				fUserDefinedTimerCount;	// accessed atomically
 };
 
 
@@ -416,20 +421,17 @@ struct Thread : TeamThreadIteratorEntry<thread_id>, KernelReferenceable {
 	int64			serial_number;	// immutable after adding thread to hash
 	Thread			*hash_next;		// protected by thread hash lock
 	Thread			*team_next;		// protected by team lock and fLock
-	Thread			*queue_next;	// protected by scheduler lock
-	timer			alarm;			// protected by scheduler lock
 	char			name[B_OS_NAME_LENGTH];	// protected by fLock
 	int32			priority;		// protected by scheduler lock
-	int32			next_priority;	// protected by scheduler lock
 	int32			io_priority;	// protected by fLock
 	int32			state;			// protected by scheduler lock
-	int32			next_state;		// protected by scheduler lock
 	struct cpu_ent	*cpu;			// protected by scheduler lock
 	struct cpu_ent	*previous_cpu;	// protected by scheduler lock
 	int32			pinned_to_cpu;	// only accessed by this thread or in the
 									// scheduler, when thread is not running
+	spinlock		scheduler_lock;
 
-	sigset_t		sig_block_mask;	// protected by scheduler lock,
+	sigset_t		sig_block_mask;	// protected by team->signal_lock,
 									// only modified by the thread itself
 	sigset_t		sigsuspend_original_unblocked_mask;
 		// non-0 after a return from _user_sigsuspend(), containing the inverted
@@ -442,8 +444,8 @@ struct Thread : TeamThreadIteratorEntry<thread_id>, KernelReferenceable {
 
 	bool			in_kernel;		// protected by time_lock, only written by
 									// this thread
-	bool			was_yielded;	// protected by scheduler lock
-	struct scheduler_thread_data* scheduler_data; // protected by scheduler lock
+	bool			has_yielded;	// protected by scheduler lock
+	Scheduler::ThreadData*	scheduler_data; // protected by scheduler lock
 
 	struct user_thread*	user_thread;	// write-protected by fLock, only
 										// modified by the thread itself and
@@ -481,7 +483,8 @@ struct Thread : TeamThreadIteratorEntry<thread_id>, KernelReferenceable {
 		/* this field may only stay in debug builds in the future */
 
 	BKernel::Team	*team;	// protected by team lock, thread lock, scheduler
-							// lock
+							// lock, team_lock
+	rw_spinlock		team_lock;
 
 	struct {
 		sem_id		sem;		// immutable after thread creation
@@ -514,7 +517,7 @@ struct Thread : TeamThreadIteratorEntry<thread_id>, KernelReferenceable {
 	bigtime_t		user_time;			// protected by time_lock
 	bigtime_t		kernel_time;		// protected by time_lock
 	bigtime_t		last_time;			// protected by time_lock
-	bigtime_t		cpu_clock_offset;	// protected by scheduler lock
+	bigtime_t		cpu_clock_offset;	// protected by time_lock
 
 	void			(*post_interrupt_callback)(void*);
 	void*			post_interrupt_data;
@@ -604,11 +607,11 @@ private:
 			mutex				fLock;
 
 			BKernel::PendingSignals	fPendingSignals;
-									// protected by scheduler lock
+									// protected by team->signal_lock
 
 			UserTimerList		fUserTimers;			// protected by fLock
 			ThreadTimeUserTimerList fCPUTimeUserTimers;
-									// protected by scheduler lock
+									// protected by time_lock
 };
 
 
@@ -747,7 +750,7 @@ Thread::DequeuePendingSignal(sigset_t nonBlocked, Signal& buffer)
 
 /*!	Returns the thread's current total CPU time (kernel + user + offset).
 
-	The caller must hold the scheduler lock.
+	The caller must hold \c time_lock.
 
 	\param ignoreCurrentRun If \c true and the thread is currently running,
 		don't add the time since the last time \c last_time was updated. Should
@@ -762,7 +765,7 @@ Thread::CPUTime(bool ignoreCurrentRun) const
 
 	// If currently running, also add the time since the last check, unless
 	// requested otherwise.
-	if (!ignoreCurrentRun && cpu != NULL)
+	if (!ignoreCurrentRun && last_time != 0)
 		time += system_time() - last_time;
 
 	return time;
@@ -780,12 +783,6 @@ using BKernel::ProcessGroup;
 using BKernel::ProcessGroupList;
 
 
-struct thread_queue {
-	Thread*	head;
-	Thread*	tail;
-};
-
-
 #endif	// !_ASSEMBLER
 
 

headers/private/kernel/timer.h

@@ -23,13 +23,8 @@ struct kernel_args;
 #define B_TIMER_USE_TIMER_STRUCT_TIMES	0x4000
 	// For add_timer(): Use the timer::schedule_time (absolute time) and
 	// timer::period values instead of the period parameter.
-#define B_TIMER_ACQUIRE_SCHEDULER_LOCK	0x8000
-	// The timer hook is invoked with the scheduler lock held. When invoking
-	// cancel_timer() with the scheduler lock held, too, this helps to avoid
-	// race conditions.
 #define B_TIMER_FLAGS	\
-	(B_TIMER_USE_TIMER_STRUCT_TIMES | B_TIMER_ACQUIRE_SCHEDULER_LOCK	\
-	| B_TIMER_REAL_TIME_BASE)
+	(B_TIMER_USE_TIMER_STRUCT_TIMES | B_TIMER_REAL_TIME_BASE)
 
 /* Timer info structure */
 struct timer_info {

headers/private/kernel/user_debugger.h

@@ -68,7 +68,7 @@ struct team_debug_info {
 	thread_id	causing_thread;
 		// thread that caused the debugger to be attached; -1 for manual
 		// debugger attachment (or no debugger installed)
-	vint32		image_event;
+	int32		image_event;
 		// counter incremented whenever an image is created/deleted
 
 	struct ConditionVariable* debugger_changed_condition;

headers/private/kernel/util/AutoLock.h

@@ -160,6 +160,144 @@ private:
 typedef AutoLocker<spinlock, InterruptsSpinLocking> InterruptsSpinLocker;
 
 
+class ReadSpinLocking {
+public:
+	inline bool Lock(rw_spinlock* lockable)
+	{
+		acquire_read_spinlock(lockable);
+		return true;
+	}
+
+	inline void Unlock(rw_spinlock* lockable)
+	{
+		release_read_spinlock(lockable);
+	}
+};
+
+typedef AutoLocker<rw_spinlock, ReadSpinLocking> ReadSpinLocker;
+
+
+class InterruptsReadSpinLocking {
+public:
+	InterruptsReadSpinLocking()
+		:
+		fState(0)
+	{
+	}
+
+	inline bool Lock(rw_spinlock* lockable)
+	{
+		fState = disable_interrupts();
+		acquire_read_spinlock(lockable);
+		return true;
+	}
+
+	inline void Unlock(rw_spinlock* lockable)
+	{
+		release_read_spinlock(lockable);
+		restore_interrupts(fState);
+	}
+
+private:
+	int	fState;
+};
+
+typedef AutoLocker<rw_spinlock, InterruptsReadSpinLocking>
+	InterruptsReadSpinLocker;
+
+
+class WriteSpinLocking {
+public:
+	inline bool Lock(rw_spinlock* lockable)
+	{
+		acquire_write_spinlock(lockable);
+		return true;
+	}
+
+	inline void Unlock(rw_spinlock* lockable)
+	{
+		release_write_spinlock(lockable);
+	}
+};
+
+typedef AutoLocker<rw_spinlock, WriteSpinLocking> WriteSpinLocker;
+
+
+class InterruptsWriteSpinLocking {
+public:
+	InterruptsWriteSpinLocking()
+		:
+		fState(0)
+	{
+	}
+
+	inline bool Lock(rw_spinlock* lockable)
+	{
+		fState = disable_interrupts();
+		acquire_write_spinlock(lockable);
+		return true;
+	}
+
+	inline void Unlock(rw_spinlock* lockable)
+	{
+		release_write_spinlock(lockable);
+		restore_interrupts(fState);
+	}
+
+private:
+	int	fState;
+};
+
+typedef AutoLocker<rw_spinlock, InterruptsWriteSpinLocking>
+	InterruptsWriteSpinLocker;
+
+
+class WriteSequentialLocking {
+public:
+	inline bool Lock(seqlock* lockable)
+	{
+		acquire_write_seqlock(lockable);
+		return true;
+	}
+
+	inline void Unlock(seqlock* lockable)
+	{
+		release_write_seqlock(lockable);
+	}
+};
+
+typedef AutoLocker<seqlock, WriteSequentialLocking> WriteSequentialLocker;
+
+
+class InterruptsWriteSequentialLocking {
+public:
+	InterruptsWriteSequentialLocking()
+		:
+		fState(0)
+	{
+	}
+
+	inline bool Lock(seqlock* lockable)
+	{
+		fState = disable_interrupts();
+		acquire_write_seqlock(lockable);
+		return true;
+	}
+
+	inline void Unlock(seqlock* lockable)
+	{
+		release_write_seqlock(lockable);
+		restore_interrupts(fState);
+	}
+
+private:
+	int	fState;
+};
+
+typedef AutoLocker<seqlock, InterruptsWriteSequentialLocking>
+	InterruptsWriteSequentialLocker;
+
+
 class ThreadCPUPinLocking {
 public:
 	inline bool Lock(Thread* thread)
@@ -191,6 +329,12 @@ using BPrivate::WriteLocker;
 using BPrivate::InterruptsLocker;
 using BPrivate::SpinLocker;
 using BPrivate::InterruptsSpinLocker;
+using BPrivate::ReadSpinLocker;
+using BPrivate::InterruptsReadSpinLocker;
+using BPrivate::WriteSpinLocker;
+using BPrivate::InterruptsWriteSpinLocker;
+using BPrivate::WriteSequentialLocker;
+using BPrivate::InterruptsWriteSequentialLocker;
 using BPrivate::ThreadCPUPinner;
 using BPrivate::TeamLocker;
 using BPrivate::ThreadLocker;

headers/private/kernel/util/BitUtils.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2013 Haiku, Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Paweł Dziepak, pdziepak@quarnos.org
+ */
+#ifndef KERNEL_UTIL_BITUTIL_H
+#define KERNEL_UTIL_BITUTIL_H
+
+
+#include <SupportDefs.h>
+
+
+// http://graphics.stanford.edu/~seander/bithacks.html
+static inline uint32
+next_power_of_2(uint32 v)
+{
+	v--;
+	v |= v >> 1;
+	v |= v >> 2;
+	v |= v >> 4;
+	v |= v >> 8;
+	v |= v >> 16;
+	v++;
+
+	return v;
+}
+
+
+// http://graphics.stanford.edu/~seander/bithacks.html
+static inline uint32
+count_set_bits(uint32 v)
+{
+	v = v - ((v >> 1) & 0x55555555);
+	v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
+	return (((v + (v >> 4)) & 0xF0F0F0F) * 0x1010101) >> 24;
+}
+
+
+static inline uint32
+log2(uint32 v)
+{
+	static const int MultiplyDeBruijnBitPosition[32] = {
+		0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30,
+		8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31
+	};
+
+	v |= v >> 1;
+	v |= v >> 2;
+	v |= v >> 4;
+	v |= v >> 8;
+	v |= v >> 16;
+
+	return MultiplyDeBruijnBitPosition[(uint32)(v * 0x07C4ACDDU) >> 27];
+}
+
+
+#endif	// KERNEL_UTIL_RANDOM_H
+

headers/private/kernel/util/Bitmap.h

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2013 Haiku, Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Paweł Dziepak, pdziepak@quarnos.org
+ */
+#ifndef KERNEL_UTIL_BITMAP_H
+#define KERNEL_UTIL_BITMAP_H
+
+
+#include <debug.h>
+
+#include <SupportDefs.h>
+
+
+class Bitmap {
+public:
+						Bitmap(int bitCount);
+						~Bitmap();
+
+	inline	status_t	GetInitStatus();
+
+	inline	bool		Get(int index) const;
+	inline	void		Set(int index);
+	inline	void		Clear(int index);
+
+			int			GetHighestSet() const;
+
+private:
+			status_t	fInitStatus;
+
+			int			fElementsCount;
+			int			fSize;
+			addr_t*		fBits;
+
+	static	const int	kBitsPerElement;
+};
+
+
+status_t
+Bitmap::GetInitStatus()
+{
+	return fInitStatus;
+}
+
+
+bool
+Bitmap::Get(int index) const
+{
+	ASSERT(index < fSize);
+
+	const int kArrayElement = index / kBitsPerElement;
+	const addr_t kBitMask = addr_t(1) << (index % kBitsPerElement);
+	return fBits[kArrayElement] & kBitMask;
+}
+
+
+void
+Bitmap::Set(int index)
+{
+	ASSERT(index < fSize);
+
+	const int kArrayElement = index / kBitsPerElement;
+	const addr_t kBitMask = addr_t(1) << (index % kBitsPerElement);
+	fBits[kArrayElement] |= kBitMask;
+}
+
+
+void
+Bitmap::Clear(int index)
+{
+	ASSERT(index < fSize);
+
+	const int kArrayElement = index / kBitsPerElement;
+	const addr_t kBitMask = addr_t(1) << (index % kBitsPerElement);
+	fBits[kArrayElement] &= ~addr_t(kBitMask);
+}
+
+
+#endif	// KERNEL_UTIL_BITMAP_H
+

headers/private/kernel/util/Heap.h

@@ -0,0 +1,357 @@
+/*
+ * Copyright 2013 Haiku, Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Paweł Dziepak, pdziepak@quarnos.org
+ */
+#ifndef KERNEL_UTIL_HEAP_H
+#define KERNEL_UTIL_HEAP_H
+
+
+#include <debug.h>
+
+#include <SupportDefs.h>
+
+
+template<typename Element, typename Key>
+struct HeapLink {
+						HeapLink();
+
+			int			fIndex;
+			Key			fKey;
+};
+
+template<typename Element, typename Key>
+class HeapLinkImpl {
+private:
+	typedef HeapLink<Element, Key> Link;
+
+public:
+	inline	Link*		GetHeapLink();
+
+private:
+			Link		fHeapLink;
+};
+
+template<typename Element, typename Key>
+class HeapStandardGetLink {
+private:
+	typedef HeapLink<Element, Key> Link;
+
+public:
+	inline	Link*		operator()(Element* element) const;
+};
+
+template<typename Element, typename Key,
+	HeapLink<Element, Key> Element::*LinkMember>
+class HeapMemberGetLink {
+private:
+	typedef HeapLink<Element, Key> Link;
+
+public:
+	inline	Link*		operator()(Element* element) const;
+};
+
+template<typename Key>
+class HeapLesserCompare {
+public:
+	inline	bool		operator()(Key a, Key b);
+};
+
+template<typename Key>
+class HeapGreaterCompare {
+public:
+	inline	bool		operator()(Key a, Key b);
+};
+
+#define HEAP_TEMPLATE_LIST	\
+	template<typename Element, typename Key, typename Compare, typename GetLink>
+#define HEAP_CLASS_NAME	Heap<Element, Key, Compare, GetLink>
+
+template<typename Element, typename Key,
+	typename Compare = HeapLesserCompare<Key>,
+	typename GetLink = HeapStandardGetLink<Element, Key> >
+class Heap {
+public:
+						Heap();
+						Heap(int initialSize);
+						~Heap();
+
+	inline	Element*	PeekRoot() const;
+
+	static	const Key&	GetKey(Element* element);
+
+	inline	void		ModifyKey(Element* element, Key newKey);
+
+	inline	void		RemoveRoot();
+	inline	status_t	Insert(Element* element, Key key);
+
+private:
+			status_t	_GrowHeap(int minimalSize = 0);
+
+			void		_MoveUp(HeapLink<Element, Key>* link);
+			void		_MoveDown(HeapLink<Element, Key>* link);
+
+			Element**	fElements;
+			int			fLastElement;
+			int			fSize;
+
+	static	Compare		sCompare;
+	static	GetLink		sGetLink;
+};
+
+
+#if KDEBUG
+template<typename Element, typename Key>
+HeapLink<Element, Key>::HeapLink()
+	:
+	fIndex(-1)
+{
+}
+#else
+template<typename Element, typename Key>
+HeapLink<Element, Key>::HeapLink()
+{
+}
+#endif
+
+
+template<typename Element, typename Key>
+HeapLink<Element, Key>*
+HeapLinkImpl<Element, Key>::GetHeapLink()
+{
+	return &fHeapLink;
+}
+
+
+template<typename Element, typename Key>
+HeapLink<Element, Key>*
+HeapStandardGetLink<Element, Key>::operator()(Element* element) const
+{
+	return element->GetHeapLink();
+}
+
+
+template<typename Element, typename Key,
+	HeapLink<Element, Key> Element::*LinkMember>
+HeapLink<Element, Key>*
+HeapMemberGetLink<Element, Key, LinkMember>::operator()(Element* element) const
+{
+	return &(element->*LinkMember);
+}
+
+
+template<typename Key>
+bool
+HeapLesserCompare<Key>::operator()(Key a, Key b)
+{
+	return a < b;
+}
+
+
+template<typename Key>
+bool
+HeapGreaterCompare<Key>::operator()(Key a, Key b)
+{
+	return a > b;
+}
+
+
+HEAP_TEMPLATE_LIST
+HEAP_CLASS_NAME::Heap()
+	:
+	fElements(NULL),
+	fLastElement(0),
+	fSize(0)
+{
+}
+
+
+HEAP_TEMPLATE_LIST
+HEAP_CLASS_NAME::Heap(int initialSize)
+	:
+	fElements(NULL),
+	fLastElement(0),
+	fSize(0)
+{
+	_GrowHeap(initialSize);
+}
+
+
+HEAP_TEMPLATE_LIST
+HEAP_CLASS_NAME::~Heap()
+{
+	free(fElements);
+}
+
+
+HEAP_TEMPLATE_LIST
+Element*
+HEAP_CLASS_NAME::PeekRoot() const
+{
+	if (fLastElement > 0)
+		return fElements[0];
+	return NULL;
+}
+
+
+HEAP_TEMPLATE_LIST
+const Key&
+HEAP_CLASS_NAME::GetKey(Element* element)
+{
+	return sGetLink(element)->fKey;
+}
+
+
+HEAP_TEMPLATE_LIST
+void
+HEAP_CLASS_NAME::ModifyKey(Element* element, Key newKey)
+{
+	HeapLink<Element, Key>* link = sGetLink(element);
+
+	ASSERT(link->fIndex >= 0 && link->fIndex < fLastElement);
+	Key oldKey = link->fKey;
+	link->fKey = newKey;
+
+	if (sCompare(newKey, oldKey))
+		_MoveUp(link);
+	else if (sCompare(oldKey, newKey))
+		_MoveDown(link);
+}
+
+
+HEAP_TEMPLATE_LIST
+void
+HEAP_CLASS_NAME::RemoveRoot()
+{
+	ASSERT(fLastElement > 0);
+
+#if KDEBUG
+	Element* element = PeekRoot();
+	HeapLink<Element, Key>* link = sGetLink(element);
+	ASSERT(link->fIndex != -1);
+	link->fIndex = -1;
+#endif
+
+	fLastElement--;
+	if (fLastElement > 0) {
+		Element* lastElement = fElements[fLastElement];
+		fElements[0] = lastElement;
+		sGetLink(lastElement)->fIndex = 0;
+		_MoveDown(sGetLink(lastElement));
+	}
+}
+
+
+HEAP_TEMPLATE_LIST
+status_t
+HEAP_CLASS_NAME::Insert(Element* element, Key key)
+{
+	if (fLastElement == fSize) {
+		status_t result = _GrowHeap();
+		if (result != B_OK)
+			return result;
+	}
+
+	ASSERT(fLastElement != fSize);
+
+	HeapLink<Element, Key>* link = sGetLink(element);
+
+	ASSERT(link->fIndex == -1);
+
+	fElements[fLastElement] = element;
+	link->fIndex = fLastElement++;
+	link->fKey = key;
+	_MoveUp(link);
+
+	return B_OK;
+}
+
+
+HEAP_TEMPLATE_LIST
+status_t
+HEAP_CLASS_NAME::_GrowHeap(int minimalSize)
+{
+	int newSize = max_c(max_c(fSize * 2, 4), minimalSize);
+
+	size_t arraySize = newSize * sizeof(Element*);
+	Element** newBuffer
+		= reinterpret_cast<Element**>(realloc(fElements, arraySize));
+	if (newBuffer == NULL)
+		return B_NO_MEMORY;
+
+	fElements = newBuffer;
+	fSize = newSize;
+
+	return B_OK;
+}
+
+
+HEAP_TEMPLATE_LIST
+void
+HEAP_CLASS_NAME::_MoveUp(HeapLink<Element, Key>* link)
+{
+	while (true) {
+		int parent = (link->fIndex - 1) / 2;
+		if (link->fIndex > 0
+			&& sCompare(link->fKey, sGetLink(fElements[parent])->fKey)) {
+
+			sGetLink(fElements[parent])->fIndex = link->fIndex;
+
+			Element* element = fElements[link->fIndex];
+			fElements[link->fIndex] = fElements[parent];
+			fElements[parent] = element;
+
+			link->fIndex = parent;
+		} else
+			break;
+	}
+}
+
+
+HEAP_TEMPLATE_LIST
+void
+HEAP_CLASS_NAME::_MoveDown(HeapLink<Element, Key>* link)
+{
+	int current;
+
+	while (true) {
+		current = link->fIndex;
+
+		int child = 2 * link->fIndex + 1;
+		if (child < fLastElement
+			&& sCompare(sGetLink(fElements[child])->fKey, link->fKey)) {
+			current = child;
+		}
+
+		child = 2 * link->fIndex + 2;
+		if (child < fLastElement
+			&& sCompare(sGetLink(fElements[child])->fKey,
+				sGetLink(fElements[current])->fKey)) {
+			current = child;
+		}
+
+		if (link->fIndex == current)
+			break;
+
+		sGetLink(fElements[current])->fIndex = link->fIndex;
+
+		Element* element = fElements[link->fIndex];
+		fElements[link->fIndex] = fElements[current];
+		fElements[current] = element;
+
+		link->fIndex = current;
+	}
+}
+
+
+HEAP_TEMPLATE_LIST
+Compare HEAP_CLASS_NAME::sCompare;
+
+HEAP_TEMPLATE_LIST
+GetLink HEAP_CLASS_NAME::sGetLink;
+
+
+#endif	// KERNEL_UTIL_HEAP_H
+

headers/private/kernel/util/MinMaxHeap.h

@@ -0,0 +1,503 @@
+/*
+ * Copyright 2013 Haiku, Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Paweł Dziepak, pdziepak@quarnos.org
+ */
+#ifndef KERNEL_UTIL_MIN_MAX_HEAP_H
+#define KERNEL_UTIL_MIN_MAX_HEAP_H
+
+
+#include <debug.h>
+
+#include <SupportDefs.h>
+
+
+template<typename Element, typename Key>
+struct MinMaxHeapLink {
+						MinMaxHeapLink();
+
+			bool		fMinTree;
+			int			fIndex;
+			Key			fKey;
+};
+
+template<typename Element, typename Key>
+class MinMaxHeapLinkImpl {
+private:
+	typedef MinMaxHeapLink<Element, Key> Link;
+
+public:
+	inline	Link*		GetMinMaxHeapLink();
+
+private:
+			Link		fMinMaxHeapLink;
+};
+
+template<typename Element, typename Key>
+class MinMaxHeapStandardGetLink {
+private:
+	typedef MinMaxHeapLink<Element, Key> Link;
+
+public:
+	inline	Link*		operator()(Element* element) const;
+};
+
+template<typename Element, typename Key,
+	MinMaxHeapLink<Element, Key> Element::*LinkMember>
+class MinMaxHeapMemberGetLink {
+private:
+	typedef MinMaxHeapLink<Element, Key> Link;
+
+public:
+	inline	Link*		operator()(Element* element) const;
+};
+
+template<typename Key>
+class MinMaxHeapCompare {
+public:
+	inline	bool		operator()(Key a, Key b);
+};
+
+#define MIN_MAX_HEAP_TEMPLATE_LIST	\
+	template<typename Element, typename Key, typename Compare, typename GetLink>
+#define MIN_MAX_HEAP_CLASS_NAME	MinMaxHeap<Element, Key, Compare, GetLink>
+
+template<typename Element, typename Key,
+	typename Compare = MinMaxHeapCompare<Key>,
+	typename GetLink = MinMaxHeapStandardGetLink<Element, Key> >
+class MinMaxHeap {
+public:
+						MinMaxHeap();
+						MinMaxHeap(int initialSize);
+						~MinMaxHeap();
+
+	inline	Element*	PeekMinimum() const;
+	inline	Element*	PeekMaximum() const;
+
+	static	const Key&	GetKey(Element* element);
+
+	inline	void		ModifyKey(Element* element, Key newKey);
+
+	inline	void		RemoveMinimum();
+	inline	void		RemoveMaximum();
+
+	inline	status_t	Insert(Element* element, Key key);
+
+private:
+			status_t	_GrowHeap(int minimalSize = 0);
+
+			void		_MoveUp(MinMaxHeapLink<Element, Key>* link);
+			void		_MoveDown(MinMaxHeapLink<Element, Key>* link);
+			bool		_ChangeTree(MinMaxHeapLink<Element, Key>* link);
+
+			void		_RemoveLast(bool minTree);
+
+			Element**	fMinElements;
+			int			fMinLastElement;
+
+			Element**	fMaxElements;
+			int			fMaxLastElement;
+
+			int			fSize;
+
+	static	Compare		sCompare;
+	static	GetLink		sGetLink;
+};
+
+
+#if KDEBUG
+template<typename Element, typename Key>
+MinMaxHeapLink<Element, Key>::MinMaxHeapLink()
+	:
+	fIndex(-1)
+{
+}
+#else
+template<typename Element, typename Key>
+MinMaxHeapLink<Element, Key>::MinMaxHeapLink()
+{
+}
+#endif
+
+
+template<typename Element, typename Key>
+MinMaxHeapLink<Element, Key>*
+MinMaxHeapLinkImpl<Element, Key>::GetMinMaxHeapLink()
+{
+	return &fMinMaxHeapLink;
+}
+
+
+template<typename Element, typename Key>
+MinMaxHeapLink<Element, Key>*
+MinMaxHeapStandardGetLink<Element, Key>::operator()(Element* element) const
+{
+	return element->GetMinMaxHeapLink();
+}
+
+
+template<typename Element, typename Key,
+	MinMaxHeapLink<Element, Key> Element::*LinkMember>
+MinMaxHeapLink<Element, Key>*
+MinMaxHeapMemberGetLink<Element, Key, LinkMember>::operator()(
+	Element* element) const
+{
+	return &(element->*LinkMember);
+}
+
+
+template<typename Key>
+bool
+MinMaxHeapCompare<Key>::operator()(Key a, Key b)
+{
+	return a < b;
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+MIN_MAX_HEAP_CLASS_NAME::MinMaxHeap()
+	:
+	fMinElements(NULL),
+	fMinLastElement(0),
+	fMaxElements(NULL),
+	fMaxLastElement(0),
+	fSize(0)
+{
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+MIN_MAX_HEAP_CLASS_NAME::MinMaxHeap(int initialSize)
+	:
+	fMinElements(NULL),
+	fMinLastElement(0),
+	fMaxElements(NULL),
+	fMaxLastElement(0),
+	fSize(0)
+{
+	_GrowHeap(initialSize);
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+MIN_MAX_HEAP_CLASS_NAME::~MinMaxHeap()
+{
+	free(fMinElements);
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+Element*
+MIN_MAX_HEAP_CLASS_NAME::PeekMinimum() const
+{
+	if (fMinLastElement > 0)
+		return fMinElements[0];
+	else if (fMaxLastElement > 0) {
+		ASSERT(fMaxLastElement == 1);
+		return fMaxElements[0];
+	}
+
+	return NULL;
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+Element*
+MIN_MAX_HEAP_CLASS_NAME::PeekMaximum() const
+{
+	if (fMaxLastElement > 0)
+		return fMaxElements[0];
+	else if (fMinLastElement > 0) {
+		ASSERT(fMinLastElement == 1);
+		return fMinElements[0];
+	}
+
+	return NULL;
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+const Key&
+MIN_MAX_HEAP_CLASS_NAME::GetKey(Element* element)
+{
+	return sGetLink(element)->fKey;
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+void
+MIN_MAX_HEAP_CLASS_NAME::ModifyKey(Element* element, Key newKey)
+{
+	MinMaxHeapLink<Element, Key>* link = sGetLink(element);
+
+	Key oldKey = link->fKey;
+	link->fKey = newKey;
+
+	if (!sCompare(newKey, oldKey) && !sCompare(oldKey, newKey))
+		return;
+
+	if (sCompare(newKey, oldKey) ^ !link->fMinTree)
+		_MoveUp(link);
+	else
+		_MoveDown(link);
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+void
+MIN_MAX_HEAP_CLASS_NAME::RemoveMinimum()
+{
+	if (fMinLastElement == 0) {
+		ASSERT(fMaxLastElement == 1);
+		RemoveMaximum();
+		return;
+	}
+
+#if KDEBUG
+	Element* element = PeekMinimum();
+	MinMaxHeapLink<Element, Key>* link = sGetLink(element);
+	ASSERT(link->fIndex != -1);
+	link->fIndex = -1;
+#endif
+
+	_RemoveLast(true);
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+void
+MIN_MAX_HEAP_CLASS_NAME::RemoveMaximum()
+{
+	if (fMaxLastElement == 0) {
+		ASSERT(fMinLastElement == 1);
+		RemoveMinimum();
+		return;
+	}
+
+#if KDEBUG
+	Element* element = PeekMaximum();
+	MinMaxHeapLink<Element, Key>* link = sGetLink(element);
+	ASSERT(link->fIndex != -1);
+	link->fIndex = -1;
+#endif
+
+	_RemoveLast(false);
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+status_t
+MIN_MAX_HEAP_CLASS_NAME::Insert(Element* element, Key key)
+{
+	if (min_c(fMinLastElement, fMaxLastElement) == fSize) {
+		ASSERT(max_c(fMinLastElement, fMaxLastElement) == fSize);
+		status_t result = _GrowHeap();
+		if (result != B_OK)
+			return result;
+	}
+
+	ASSERT(fMinLastElement < fSize || fMaxLastElement < fSize);
+
+	MinMaxHeapLink<Element, Key>* link = sGetLink(element);
+
+	ASSERT(link->fIndex == -1);
+
+	link->fMinTree = fMinLastElement < fMaxLastElement;
+
+	int& lastElement = link->fMinTree ? fMinLastElement : fMaxLastElement;
+	Element** tree = link->fMinTree ? fMinElements : fMaxElements;
+
+	tree[lastElement] = element;
+	link->fIndex = lastElement++;
+	link->fKey = key;
+
+	if (!_ChangeTree(link))
+		_MoveUp(link);
+
+	return B_OK;
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+status_t
+MIN_MAX_HEAP_CLASS_NAME::_GrowHeap(int minimalSize)
+{
+	minimalSize = minimalSize % 2 == 0 ? minimalSize : minimalSize + 1;
+	int newSize = max_c(max_c(fSize * 4, 4), minimalSize);
+
+	size_t arraySize = newSize * sizeof(Element*);
+	Element** newBuffer
+		= reinterpret_cast<Element**>(realloc(fMinElements, arraySize));
+	if (newBuffer == NULL)
+		return B_NO_MEMORY;
+	fMinElements = newBuffer;
+
+	newBuffer += newSize / 2;
+	if (fMaxLastElement > 0)
+		memcpy(newBuffer, fMinElements + fSize, fSize * sizeof(Element*));
+	fMaxElements = newBuffer;
+
+	fSize = newSize / 2;
+	return B_OK;
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+void
+MIN_MAX_HEAP_CLASS_NAME::_MoveUp(MinMaxHeapLink<Element, Key>* link)
+{
+	Element** tree = link->fMinTree ? fMinElements : fMaxElements;
+	while (true) {
+		if (link->fIndex <= 0)
+			break;
+
+		int parent = (link->fIndex - 1) / 2;
+		bool isSmaller = sCompare(link->fKey, sGetLink(tree[parent])->fKey);
+		if (isSmaller ^ !link->fMinTree) {
+			ASSERT(sGetLink(tree[parent])->fIndex == parent);
+			sGetLink(tree[parent])->fIndex = link->fIndex;
+
+			Element* element = tree[link->fIndex];
+			tree[link->fIndex] = tree[parent];
+			tree[parent] = element;
+
+			link->fIndex = parent;
+		} else
+			break;
+	}
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+void
+MIN_MAX_HEAP_CLASS_NAME::_MoveDown(MinMaxHeapLink<Element, Key>* link)
+{
+	int current;
+
+	int lastElement = link->fMinTree ? fMinLastElement : fMaxLastElement;
+	Element** tree = link->fMinTree ? fMinElements : fMaxElements;
+	while (true) {
+		current = link->fIndex;
+
+		int child = 2 * link->fIndex + 1;
+		if (child < lastElement) {
+			bool isSmaller = sCompare(sGetLink(tree[child])->fKey, link->fKey);
+			if (isSmaller ^ !link->fMinTree)
+				current = child;
+		}
+
+		child = 2 * link->fIndex + 2;
+		if (child < lastElement) {
+			bool isSmaller = sCompare(sGetLink(tree[child])->fKey,
+					sGetLink(tree[current])->fKey);
+			if (isSmaller ^ !link->fMinTree)
+				current = child;
+		}
+
+		if (link->fIndex == current)
+			break;
+
+		ASSERT(sGetLink(tree[current])->fIndex == current);
+		sGetLink(tree[current])->fIndex = link->fIndex;
+
+		Element* element = tree[link->fIndex];
+		tree[link->fIndex] = tree[current];
+		tree[current] = element;
+
+		link->fIndex = current;
+	}
+
+	if (2 * link->fIndex + 1 >= lastElement)
+		_ChangeTree(link);
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+bool
+MIN_MAX_HEAP_CLASS_NAME::_ChangeTree(MinMaxHeapLink<Element, Key>* link)
+{
+	int otherLastElement = link->fMinTree ? fMaxLastElement : fMinLastElement;
+
+	Element** currentTree = link->fMinTree ? fMinElements : fMaxElements;
+	Element** otherTree = link->fMinTree ? fMaxElements : fMinElements;
+
+	if (otherLastElement <= 0) {
+		ASSERT(link->fMinTree ? fMinLastElement : fMaxLastElement == 1);
+		return false;
+	}
+
+	ASSERT((link->fIndex - 1) / 2 < otherLastElement);
+
+	Element* predecessor;
+	if (2 * link->fIndex + 1 < otherLastElement) {
+		predecessor = otherTree[2 * link->fIndex + 1];
+		ASSERT(sGetLink(predecessor)->fIndex == 2 * link->fIndex + 1);
+	} else if (link->fIndex < otherLastElement) {
+		predecessor = otherTree[link->fIndex];
+		ASSERT(sGetLink(predecessor)->fIndex == link->fIndex);
+	} else {
+		predecessor = otherTree[(link->fIndex - 1) / 2];
+		ASSERT(sGetLink(predecessor)->fIndex == (link->fIndex - 1) / 2);
+	}
+	MinMaxHeapLink<Element, Key>* predecessorLink = sGetLink(predecessor);
+
+	bool isSmaller = sCompare(predecessorLink->fKey, link->fKey);
+	if (isSmaller ^ !link->fMinTree) {
+		Element* element = currentTree[link->fIndex];
+		currentTree[link->fIndex] = otherTree[predecessorLink->fIndex];
+		otherTree[predecessorLink->fIndex] = element;
+
+		int index = link->fIndex;
+		link->fIndex = predecessorLink->fIndex;
+		predecessorLink->fIndex = index;
+
+		predecessorLink->fMinTree = !predecessorLink->fMinTree;
+		link->fMinTree = !link->fMinTree;
+
+		_MoveUp(link);
+		return true;
+	}
+
+	return false;
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+void
+MIN_MAX_HEAP_CLASS_NAME::_RemoveLast(bool minTree)
+{
+	bool deleteMin = fMaxLastElement < fMinLastElement;
+
+	Element** tree = deleteMin ? fMinElements : fMaxElements;
+	int& lastElement = deleteMin ? fMinLastElement : fMaxLastElement;
+
+	ASSERT(lastElement > 0);
+	lastElement--;
+	if (lastElement == 0 && deleteMin == minTree)
+		return;
+
+	Element* element = tree[lastElement];
+
+	if (minTree)
+		fMinElements[0] = element;
+	else
+		fMaxElements[0] = element;
+
+	MinMaxHeapLink<Element, Key>* link = sGetLink(element);
+	link->fIndex = 0;
+	link->fMinTree = minTree;
+	_MoveDown(link);
+}
+
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+Compare MIN_MAX_HEAP_CLASS_NAME::sCompare;
+
+MIN_MAX_HEAP_TEMPLATE_LIST
+GetLink MIN_MAX_HEAP_CLASS_NAME::sGetLink;
+
+
+#endif	// KERNEL_UTIL_MIN_MAX_HEAP_H
+

headers/private/kernel/util/atomic.h

@@ -10,6 +10,8 @@
 
 #include <SupportDefs.h>
 
+#include <debug.h>
+
 
 #ifdef __cplusplus
 
@@ -18,22 +20,34 @@ atomic_pointer_test_and_set(PointerType** _pointer, const PointerType* set,
 	const PointerType* test)
 {
 #if LONG_MAX == INT_MAX
-	return (PointerType*)atomic_test_and_set((vint32*)_pointer, (int32)set,
+	return (PointerType*)atomic_test_and_set((int32*)_pointer, (int32)set,
 		(int32)test);
 #else
-	return (PointerType*)atomic_test_and_set64((vint64*)_pointer, (int64)set,
+	return (PointerType*)atomic_test_and_set64((int64*)_pointer, (int64)set,
 		(int64)test);
 #endif
 }
 
 
 template<typename PointerType> PointerType*
-atomic_pointer_set(PointerType** _pointer, const PointerType* set)
+atomic_pointer_get_and_set(PointerType** _pointer, const PointerType* set)
 {
 #if LONG_MAX == INT_MAX
-	return (PointerType*)atomic_set((vint32*)_pointer, (int32)set);
+	return (PointerType*)atomic_get_and_set((int32*)_pointer, (int32)set);
 #else
-	return (PointerType*)atomic_set64((vint64*)_pointer, (int64)set);
+	return (PointerType*)atomic_get_and_set64((int64*)_pointer, (int64)set);
+#endif
+}
+
+
+template<typename PointerType> void
+atomic_pointer_set(PointerType** _pointer, const PointerType* set)
+{
+	ASSERT((addr_t(_pointer) & (sizeof(PointerType*) - 1)) == 0);
+#if LONG_MAX == INT_MAX
+	atomic_set((int32*)_pointer, (int32)set);
+#else
+	atomic_set64((int64*)_pointer, (int64)set);
 #endif
 }
 
@@ -41,10 +55,11 @@ atomic_pointer_set(PointerType** _pointer, const PointerType* set)
 template<typename PointerType> PointerType*
 atomic_pointer_get(PointerType** _pointer)
 {
+	ASSERT((addr_t(_pointer) & (sizeof(PointerType*) - 1)) == 0);
 #if LONG_MAX == INT_MAX
-	return (PointerType*)atomic_get((vint32*)_pointer);
+	return (PointerType*)atomic_get((int32*)_pointer);
 #else
-	return (PointerType*)atomic_get64((vint64*)_pointer);
+	return (PointerType*)atomic_get64((int64*)_pointer);
 #endif
 }
 

headers/private/kernel/vm/vm.h

@@ -142,7 +142,7 @@ status_t vm_get_physical_page_debug(phys_addr_t paddr, addr_t* vaddr,
 			void** _handle);
 status_t vm_put_physical_page_debug(addr_t vaddr, void* handle);
 
-void vm_get_info(struct system_memory_info *info);
+void vm_get_info(system_info *info);
 uint32 vm_num_page_faults(void);
 off_t vm_available_memory(void);
 off_t vm_available_not_needed_memory(void);

headers/private/kernel/vm/vm_types.h

@@ -136,7 +136,7 @@ public:
 #endif
 
 #if DEBUG_PAGE_ACCESS
-	vint32					accessing_thread;
+	int32					accessing_thread;
 #endif
 
 #if VM_PAGE_ALLOCATION_TRACKING_AVAILABLE

headers/private/kernel/wait_for_objects.h

@@ -17,12 +17,12 @@ struct select_sync;
 typedef struct select_info {
 	struct select_info*	next;				// next in the object's list
 	struct select_sync*	sync;
-	vint32				events;
+	int32				events;
 	uint16				selected_events;
 } select_info;
 
 typedef struct select_sync {
-	vint32				ref_count;
+	int32				ref_count;
 	sem_id				sem;
 	uint32				count;
 	struct select_info*	set;