kernel/x86: Inline atomic functions and memory barriers

headers/private/kernel/arch/atomic.h

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2014, Paweł Dziepak, [email protected].
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_ARCH_ATOMIC_H
+#define _KERNEL_ARCH_ATOMIC_H
+
+
+#include <SupportDefs.h>
+
+#include <KernelExport.h>
+
+
+#ifdef __x86_64__
+#	include <arch/x86/64/atomic.h>
+#elif __INTEL__
+#	include <arch/x86/32/atomic.h>
+#endif
+
+
+#endif	// _KERNEL_ARCH_ATOMIC_H
+

headers/private/kernel/arch/cpu.h

@@ -43,10 +43,6 @@ status_t arch_cpu_user_memset(void *s, char c, size_t count,
 
 void arch_cpu_sync_icache(void *address, size_t length);
 
-void arch_cpu_memory_read_barrier(void);
-void arch_cpu_memory_write_barrier(void);
-void arch_cpu_memory_read_write_barrier(void);
-
 
 #ifdef __cplusplus
 }

headers/private/kernel/arch/x86/32/atomic.h

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2014, Paweł Dziepak, [email protected].
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_ARCH_X86_32_ATOMIC_H
+#define _KERNEL_ARCH_X86_32_ATOMIC_H
+
+
+static inline void
+memory_read_barrier_inline(void)
+{
+	asm volatile("lock; addl $0, (%%esp)" : : : "memory");
+}
+
+
+static inline void
+memory_write_barrier_inline(void)
+{
+	asm volatile("lock; addl $0, (%%esp)" : : : "memory");
+}
+
+
+static inline void
+memory_full_barrier_inline(void)
+{
+	asm volatile("lock; addl $0, (%%esp)" : : : "memory");
+}
+
+
+#define memory_read_barrier		memory_read_barrier_inline
+#define memory_write_barrier	memory_write_barrier_inline
+#define memory_full_barrier		memory_full_barrier_inline
+
+
+static inline void
+atomic_set_inline(int32* value, int32 newValue)
+{
+	memory_write_barrier();
+	*(volatile int32*)value = newValue;
+}
+
+
+static inline int32
+atomic_get_and_set_inline(int32* value, int32 newValue)
+{
+	asm volatile("xchgl %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_test_and_set_inline(int32* value, int32 newValue, int32 testAgainst)
+{
+	asm volatile("lock; cmpxchgl %2, (%3)"
+		: "=a" (newValue)
+		: "0" (testAgainst), "r" (newValue), "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_add_inline(int32* value, int32 newValue)
+{
+	asm volatile("lock; xaddl %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_get_inline(int32* value)
+{
+	int32 newValue = *(volatile int32*)value;
+	memory_read_barrier();
+	return newValue;
+}
+
+
+#define atomic_set				atomic_set_inline
+#define atomic_get_and_set		atomic_get_and_set_inline
+#ifndef atomic_test_and_set
+#	define atomic_test_and_set	atomic_test_and_set_inline
+#endif
+#ifndef atomic_add
+#	define atomic_add			atomic_add_inline
+#endif
+#define atomic_get				atomic_get_inline
+
+
+#endif	// _KERNEL_ARCH_X86_32_ATOMIC_H
+

headers/private/kernel/arch/x86/64/atomic.h

@@ -0,0 +1,153 @@
+/*
+ * Copyright 2014, Paweł Dziepak, [email protected].
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_ARCH_X86_64_ATOMIC_H
+#define _KERNEL_ARCH_X86_64_ATOMIC_H
+
+
+static inline void
+memory_read_barrier_inline(void)
+{
+	asm volatile("lfence" : : : "memory");
+}
+
+
+static inline void
+memory_write_barrier_inline(void)
+{
+	asm volatile("sfence" : : : "memory");
+}
+
+
+static inline void
+memory_full_barrier_inline(void)
+{
+	asm volatile("mfence" : : : "memory");
+}
+
+
+#define memory_read_barrier		memory_read_barrier_inline
+#define memory_write_barrier	memory_write_barrier_inline
+#define memory_full_barrier		memory_full_barrier_inline
+
+
+static inline void
+atomic_set_inline(int32* value, int32 newValue)
+{
+	memory_write_barrier();
+	*(volatile int32*)value = newValue;
+}
+
+
+static inline int32
+atomic_get_and_set_inline(int32* value, int32 newValue)
+{
+	asm volatile("xchg %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_test_and_set_inline(int32* value, int32 newValue, int32 testAgainst)
+{
+	asm volatile("lock; cmpxchgl %2, (%3)"
+		: "=a" (newValue)
+		: "0" (testAgainst), "r" (newValue), "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_add_inline(int32* value, int32 newValue)
+{
+	asm volatile("lock; xaddl %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_get_inline(int32* value)
+{
+	int32 newValue = *(volatile int32*)value;
+	memory_read_barrier();
+	return newValue;
+}
+
+
+static inline void
+atomic_set64_inline(int64* value, int64 newValue)
+{
+	memory_write_barrier();
+	*(volatile int64*)value = newValue;
+}
+
+
+static inline int64
+atomic_get_and_set64_inline(int64* value, int64 newValue)
+{
+	asm volatile("xchgq %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int64
+atomic_test_and_set64_inline(int64* value, int64 newValue, int64 testAgainst)
+{
+	asm volatile("lock; cmpxchgq %2, (%3)"
+		: "=a" (newValue)
+		: "0" (testAgainst), "r" (newValue), "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int64
+atomic_add64_inline(int64* value, int64 newValue)
+{
+	asm volatile("lock; xaddq %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int64
+atomic_get64_inline(int64* value)
+{
+	int64 newValue = *(volatile int64*)value;
+	memory_read_barrier();
+	return newValue;
+}
+
+
+#define atomic_set				atomic_set_inline
+#define atomic_get_and_set		atomic_get_and_set_inline
+#ifndef atomic_test_and_set
+#	define atomic_test_and_set	atomic_test_and_set_inline
+#endif
+#ifndef atomic_add
+#	define atomic_add			atomic_add_inline
+#endif
+#define atomic_get				atomic_get_inline
+
+#define atomic_set64			atomic_set64_inline
+#define atomic_get_and_set64	atomic_get_and_set64_inline
+#define atomic_test_and_set64	atomic_test_and_set64_inline
+#define atomic_add64			atomic_add64_inline
+#define atomic_get64			atomic_get64_inline
+
+
+#endif	// _KERNEL_ARCH_X86_64_ATOMIC_H
+

headers/private/kernel/lock.h

@@ -9,7 +9,10 @@
 #ifndef _KERNEL_LOCK_H
 #define _KERNEL_LOCK_H
 
+
 #include <OS.h>
+
+#include <arch/atomic.h>
 #include <debug.h>
 
 

headers/private/kernel/smp.h

@@ -9,6 +9,7 @@
 #define KERNEL_SMP_H
 
 
+#include <arch/atomic.h>
 #include <boot/kernel_args.h>
 #include <kernel.h>
 
@@ -185,7 +186,7 @@ release_spinlock_inline(spinlock* lock)
 static inline bool
 try_acquire_write_spinlock_inline(rw_spinlock* lock)
 {
-	return atomic_test_and_set(&lock->lock, 1 << 31, 0) == 0;
+	return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0;
 }
 
 
@@ -209,9 +210,9 @@ static inline bool
 try_acquire_read_spinlock_inline(rw_spinlock* lock)
 {
 	uint32 previous = atomic_add(&lock->lock, 1);
-	if ((previous & (1 << 31)) == 0)
+	if ((previous & (1u << 31)) == 0)
 		return true;
-	atomic_test_and_set(&lock->lock, 1 << 31, previous);
+	atomic_test_and_set(&lock->lock, 1u << 31, previous);
 	return false;
 }
 

headers/private/kernel/thread.h

@@ -11,12 +11,13 @@
 
 
 #include <OS.h>
-#include <thread_types.h>
-#include <arch/thread.h>
 
+#include <arch/atomic.h>
+#include <arch/thread.h>
 // For the thread blocking inline functions only.
 #include <kscheduler.h>
 #include <ksignal.h>
+#include <thread_types.h>
 
 
 struct arch_fork_arg;