From 2e2c9bd3d058dac3609103ca67011eaee14e9333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dziepak?= Date: Mon, 25 Aug 2014 14:55:55 +0200 Subject: [PATCH] os/support: implement atomic_*() using GCC builtin helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If GCC knows what these functions are actually doing the resulting code can be optimized better what is especially noticeable in case of invocations of atomic_{or,and}() that ignore the result. Obviously, everything is inlined what also improves performance. Signed-off-by: Paweł Dziepak --- headers/os/support/SupportDefs.h | 160 ++++++++++++++++---- headers/private/kernel/arch/x86/32/atomic.h | 14 +- headers/private/kernel/arch/x86/64/atomic.h | 118 --------------- src/build/libroot/atomic.cpp | 6 + 4 files changed, 147 insertions(+), 151 deletions(-) diff --git a/headers/os/support/SupportDefs.h b/headers/os/support/SupportDefs.h index 1f413d378b..b8633345da 100644 --- a/headers/os/support/SupportDefs.h +++ b/headers/os/support/SupportDefs.h @@ -195,23 +195,6 @@ extern const char *B_EMPTY_STRING; extern "C" { #endif -/* Atomic functions; previous value is returned */ -extern void atomic_set(int32* value, int32 newValue); -extern int32 atomic_get_and_set(int32* value, int32 newValue); -extern int32 atomic_test_and_set(int32 *value, int32 newValue, int32 testAgainst); -extern int32 atomic_add(int32 *value, int32 addValue); -extern int32 atomic_and(int32 *value, int32 andValue); -extern int32 atomic_or(int32 *value, int32 orValue); -extern int32 atomic_get(int32 *value); - -extern void atomic_set64(int64* value, int64 newValue); -extern int64 atomic_get_and_set64(int64* value, int64 newValue); -extern int64 atomic_test_and_set64(int64 *value, int64 newValue, int64 testAgainst); -extern int64 atomic_add64(int64 *value, int64 addValue); -extern int64 atomic_and64(int64 *value, int64 andValue); -extern int64 atomic_or64(int64 *value, int64 orValue); -extern int64 atomic_get64(int64 *value); - /* Other stuff */ extern void* get_stack_frame(void); @@ -232,18 +215,141 @@ extern void* get_stack_frame(void); /* Use the built-in atomic functions, if requested and available. */ -#if defined(B_USE_BUILTIN_ATOMIC_FUNCTIONS) && __GNUC__ >= 4 +#if defined(B_USE_BUILTIN_ATOMIC_FUNCTIONS) && __GNUC__ > 4 \ + || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7) -#define atomic_test_and_set(valuePointer, newValue, testAgainst) \ - __sync_val_compare_and_swap(valuePointer, testAgainst, newValue) -#define atomic_add(valuePointer, addValue) \ - __sync_fetch_and_add(valuePointer, addValue) -#define atomic_and(valuePointer, andValue) \ - __sync_fetch_and_and(valuePointer, andValue) -#define atomic_or(valuePointer, orValue) \ - __sync_fetch_and_or(valuePointer, orValue) -#endif // B_USE_BUILTIN_ATOMIC_FUNCTIONS && __GNUC__ >= 4 +static inline void +atomic_set(int32* value, int32 newValue) +{ + __atomic_store_n(value, newValue, __ATOMIC_RELEASE); +} + + +static inline int32 +atomic_get_and_set(int32* value, int32 newValue) +{ + return __atomic_exchange_n(value, newValue, __ATOMIC_SEQ_CST); +} + + +static inline int32 +atomic_test_and_set(int32* value, int32 newValue, int32 testAgainst) +{ + __atomic_compare_exchange_n(value, &testAgainst, newValue, 1, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + return testAgainst; +} + + +static inline int32 +atomic_add(int32* value, int32 addValue) +{ + return __atomic_fetch_add(value, addValue, __ATOMIC_SEQ_CST); +} + + +static inline int32 +atomic_and(int32* value, int32 andValue) +{ + return __atomic_fetch_and(value, andValue, __ATOMIC_SEQ_CST); +} + + +static inline int32 +atomic_or(int32* value, int32 orValue) +{ + return __atomic_fetch_or(value, orValue, __ATOMIC_SEQ_CST); +} + + +static inline int32 +atomic_get(int32* value) +{ + return __atomic_load_n(value, __ATOMIC_ACQUIRE); +} + + +static inline void +atomic_set64(int64* value, int64 newValue) +{ + __atomic_store_n(value, newValue, __ATOMIC_RELEASE); +} + + +static inline int64 +atomic_get_and_set64(int64* value, int64 newValue) +{ + return __atomic_exchange_n(value, newValue, __ATOMIC_SEQ_CST); +} + + +static inline int64 +atomic_test_and_set64(int64* value, int64 newValue, int64 testAgainst) +{ + __atomic_compare_exchange_n(value, &testAgainst, newValue, 1, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + return testAgainst; +} + + +static inline int64 +atomic_add64(int64* value, int64 addValue) +{ + return __atomic_fetch_add(value, addValue, __ATOMIC_SEQ_CST); +} + + +static inline int64 +atomic_and64(int64* value, int64 andValue) +{ + return __atomic_fetch_and(value, andValue, __ATOMIC_SEQ_CST); +} + + +static inline int64 +atomic_or64(int64* value, int64 orValue) +{ + return __atomic_fetch_or(value, orValue, __ATOMIC_SEQ_CST); +} + + +static inline int64 +atomic_get64(int64* value) +{ + return __atomic_load_n(value, __ATOMIC_ACQUIRE); +} + + +#else // B_USE_BUILTIN_ATOMIC_FUNCTIONS && __GNUC__ > 4 + // || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7) + +#ifdef __cplusplus +extern "C" { +#endif + +/* Atomic functions; previous value is returned */ +extern void atomic_set(int32* value, int32 newValue); +extern int32 atomic_get_and_set(int32* value, int32 newValue); +extern int32 atomic_test_and_set(int32 *value, int32 newValue, int32 testAgainst); +extern int32 atomic_add(int32 *value, int32 addValue); +extern int32 atomic_and(int32 *value, int32 andValue); +extern int32 atomic_or(int32 *value, int32 orValue); +extern int32 atomic_get(int32 *value); + +extern void atomic_set64(int64* value, int64 newValue); +extern int64 atomic_get_and_set64(int64* value, int64 newValue); +extern int64 atomic_test_and_set64(int64 *value, int64 newValue, int64 testAgainst); +extern int64 atomic_add64(int64 *value, int64 addValue); +extern int64 atomic_and64(int64 *value, int64 andValue); +extern int64 atomic_or64(int64 *value, int64 orValue); +extern int64 atomic_get64(int64 *value); + +#ifdef __cplusplus +} +#endif + +#endif #endif /* _SUPPORT_DEFS_H */ diff --git a/headers/private/kernel/arch/x86/32/atomic.h b/headers/private/kernel/arch/x86/32/atomic.h index adec9c05c3..eb2996c70d 100644 --- a/headers/private/kernel/arch/x86/32/atomic.h +++ b/headers/private/kernel/arch/x86/32/atomic.h @@ -32,6 +32,9 @@ memory_full_barrier_inline(void) #define memory_full_barrier memory_full_barrier_inline +#if __GNUC__ < 4 + + static inline void atomic_set_inline(int32* value, int32 newValue) { @@ -84,14 +87,13 @@ atomic_get_inline(int32* value) #define atomic_set atomic_set_inline #define atomic_get_and_set atomic_get_and_set_inline -#ifndef atomic_test_and_set -# define atomic_test_and_set atomic_test_and_set_inline -#endif -#ifndef atomic_add -# define atomic_add atomic_add_inline -#endif +#define atomic_test_and_set atomic_test_and_set_inline +#define atomic_add atomic_add_inline #define atomic_get atomic_get_inline +#endif // dark ages + + #endif // _KERNEL_ARCH_X86_32_ATOMIC_H diff --git a/headers/private/kernel/arch/x86/64/atomic.h b/headers/private/kernel/arch/x86/64/atomic.h index fd567319f6..ed3245e3a0 100644 --- a/headers/private/kernel/arch/x86/64/atomic.h +++ b/headers/private/kernel/arch/x86/64/atomic.h @@ -31,123 +31,5 @@ memory_full_barrier_inline(void) #define memory_write_barrier memory_write_barrier_inline #define memory_full_barrier memory_full_barrier_inline - -static inline void -atomic_set_inline(int32* value, int32 newValue) -{ - memory_write_barrier(); - *(volatile int32*)value = newValue; -} - - -static inline int32 -atomic_get_and_set_inline(int32* value, int32 newValue) -{ - asm volatile("xchg %0, (%1)" - : "+r" (newValue) - : "r" (value) - : "memory"); - return newValue; -} - - -static inline int32 -atomic_test_and_set_inline(int32* value, int32 newValue, int32 testAgainst) -{ - asm volatile("lock; cmpxchgl %2, (%3)" - : "=a" (newValue) - : "0" (testAgainst), "r" (newValue), "r" (value) - : "memory"); - return newValue; -} - - -static inline int32 -atomic_add_inline(int32* value, int32 newValue) -{ - asm volatile("lock; xaddl %0, (%1)" - : "+r" (newValue) - : "r" (value) - : "memory"); - return newValue; -} - - -static inline int32 -atomic_get_inline(int32* value) -{ - int32 newValue = *(volatile int32*)value; - memory_read_barrier(); - return newValue; -} - - -static inline void -atomic_set64_inline(int64* value, int64 newValue) -{ - memory_write_barrier(); - *(volatile int64*)value = newValue; -} - - -static inline int64 -atomic_get_and_set64_inline(int64* value, int64 newValue) -{ - asm volatile("xchgq %0, (%1)" - : "+r" (newValue) - : "r" (value) - : "memory"); - return newValue; -} - - -static inline int64 -atomic_test_and_set64_inline(int64* value, int64 newValue, int64 testAgainst) -{ - asm volatile("lock; cmpxchgq %2, (%3)" - : "=a" (newValue) - : "0" (testAgainst), "r" (newValue), "r" (value) - : "memory"); - return newValue; -} - - -static inline int64 -atomic_add64_inline(int64* value, int64 newValue) -{ - asm volatile("lock; xaddq %0, (%1)" - : "+r" (newValue) - : "r" (value) - : "memory"); - return newValue; -} - - -static inline int64 -atomic_get64_inline(int64* value) -{ - int64 newValue = *(volatile int64*)value; - memory_read_barrier(); - return newValue; -} - - -#define atomic_set atomic_set_inline -#define atomic_get_and_set atomic_get_and_set_inline -#ifndef atomic_test_and_set -# define atomic_test_and_set atomic_test_and_set_inline -#endif -#ifndef atomic_add -# define atomic_add atomic_add_inline -#endif -#define atomic_get atomic_get_inline - -#define atomic_set64 atomic_set64_inline -#define atomic_get_and_set64 atomic_get_and_set64_inline -#define atomic_test_and_set64 atomic_test_and_set64_inline -#define atomic_add64 atomic_add64_inline -#define atomic_get64 atomic_get64_inline - - #endif // _KERNEL_ARCH_X86_64_ATOMIC_H diff --git a/src/build/libroot/atomic.cpp b/src/build/libroot/atomic.cpp index 38aed02692..0e1fed4911 100644 --- a/src/build/libroot/atomic.cpp +++ b/src/build/libroot/atomic.cpp @@ -6,6 +6,9 @@ #include +#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) + + void atomic_set(int32 *value, int32 newValue) { @@ -120,3 +123,6 @@ atomic_get64(int64 *value) return *value; } + +#endif // __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) +