Subversion Repositories Kolibri OS

Rev

Blame | Last modification | View Log | RSS feed

  1. /*-
  2.  * Copyright (c) 2011 Ed Schouten <ed@FreeBSD.org>
  3.  *                    David Chisnall <theraven@FreeBSD.org>
  4.  * All rights reserved.
  5.  *
  6.  * Redistribution and use in source and binary forms, with or without
  7.  * modification, are permitted provided that the following conditions
  8.  * are met:
  9.  * 1. Redistributions of source code must retain the above copyright
  10.  *    notice, this list of conditions and the following disclaimer.
  11.  * 2. Redistributions in binary form must reproduce the above copyright
  12.  *    notice, this list of conditions and the following disclaimer in the
  13.  *    documentation and/or other materials provided with the distribution.
  14.  *
  15.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16.  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17.  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18.  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19.  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20.  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21.  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22.  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23.  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24.  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25.  * SUCH DAMAGE.
  26.  *
  27.  * $FreeBSD$
  28.  */
  29.  
  30. #ifndef _STDATOMIC_H_
  31. #define _STDATOMIC_H_
  32.  
  33. #include <sys/cdefs.h>
  34. #include <sys/_types.h>
  35.  
  36. #if __has_extension(c_atomic) || __has_extension(cxx_atomic)
  37. #define __CLANG_ATOMICS
  38. #elif __GNUC_PREREQ__(4, 7)
  39. #define __GNUC_ATOMICS
  40. #elif defined(__GNUC__)
  41. #define __SYNC_ATOMICS
  42. #else
  43. #error "stdatomic.h does not support your compiler"
  44. #endif
  45.  
  46. /*
  47.  * 7.17.1 Atomic lock-free macros.
  48.  */
  49.  
  50. #ifdef __GCC_ATOMIC_BOOL_LOCK_FREE
  51. #define ATOMIC_BOOL_LOCK_FREE           __GCC_ATOMIC_BOOL_LOCK_FREE
  52. #endif
  53. #ifdef __GCC_ATOMIC_CHAR_LOCK_FREE
  54. #define ATOMIC_CHAR_LOCK_FREE           __GCC_ATOMIC_CHAR_LOCK_FREE
  55. #endif
  56. #ifdef __GCC_ATOMIC_CHAR16_T_LOCK_FREE
  57. #define ATOMIC_CHAR16_T_LOCK_FREE       __GCC_ATOMIC_CHAR16_T_LOCK_FREE
  58. #endif
  59. #ifdef __GCC_ATOMIC_CHAR32_T_LOCK_FREE
  60. #define ATOMIC_CHAR32_T_LOCK_FREE       __GCC_ATOMIC_CHAR32_T_LOCK_FREE
  61. #endif
  62. #ifdef __GCC_ATOMIC_WCHAR_T_LOCK_FREE
  63. #define ATOMIC_WCHAR_T_LOCK_FREE        __GCC_ATOMIC_WCHAR_T_LOCK_FREE
  64. #endif
  65. #ifdef __GCC_ATOMIC_SHORT_LOCK_FREE
  66. #define ATOMIC_SHORT_LOCK_FREE          __GCC_ATOMIC_SHORT_LOCK_FREE
  67. #endif
  68. #ifdef __GCC_ATOMIC_INT_LOCK_FREE
  69. #define ATOMIC_INT_LOCK_FREE            __GCC_ATOMIC_INT_LOCK_FREE
  70. #endif
  71. #ifdef __GCC_ATOMIC_LONG_LOCK_FREE
  72. #define ATOMIC_LONG_LOCK_FREE           __GCC_ATOMIC_LONG_LOCK_FREE
  73. #endif
  74. #ifdef __GCC_ATOMIC_LLONG_LOCK_FREE
  75. #define ATOMIC_LLONG_LOCK_FREE          __GCC_ATOMIC_LLONG_LOCK_FREE
  76. #endif
  77. #ifdef __GCC_ATOMIC_POINTER_LOCK_FREE
  78. #define ATOMIC_POINTER_LOCK_FREE        __GCC_ATOMIC_POINTER_LOCK_FREE
  79. #endif
  80.  
  81. /*
  82.  * 7.17.2 Initialization.
  83.  */
  84.  
  85. #if defined(__CLANG_ATOMICS)
  86. #define ATOMIC_VAR_INIT(value)          (value)
  87. #define atomic_init(obj, value)         __c11_atomic_init(obj, value)
  88. #else
  89. #define ATOMIC_VAR_INIT(value)          { .__val = (value) }
  90. #define atomic_init(obj, value)         ((void)((obj)->__val = (value)))
  91. #endif
  92.  
  93. /*
  94.  * Clang and recent GCC both provide predefined macros for the memory
  95.  * orderings.  If we are using a compiler that doesn't define them, use the
  96.  * clang values - these will be ignored in the fallback path.
  97.  */
  98.  
  99. #ifndef __ATOMIC_RELAXED
  100. #define __ATOMIC_RELAXED                0
  101. #endif
  102. #ifndef __ATOMIC_CONSUME
  103. #define __ATOMIC_CONSUME                1
  104. #endif
  105. #ifndef __ATOMIC_ACQUIRE
  106. #define __ATOMIC_ACQUIRE                2
  107. #endif
  108. #ifndef __ATOMIC_RELEASE
  109. #define __ATOMIC_RELEASE                3
  110. #endif
  111. #ifndef __ATOMIC_ACQ_REL
  112. #define __ATOMIC_ACQ_REL                4
  113. #endif
  114. #ifndef __ATOMIC_SEQ_CST
  115. #define __ATOMIC_SEQ_CST                5
  116. #endif
  117.  
  118. /*
  119.  * 7.17.3 Order and consistency.
  120.  *
  121.  * The memory_order_* constants that denote the barrier behaviour of the
  122.  * atomic operations.
  123.  */
  124.  
  125. typedef enum {
  126.         memory_order_relaxed = __ATOMIC_RELAXED,
  127.         memory_order_consume = __ATOMIC_CONSUME,
  128.         memory_order_acquire = __ATOMIC_ACQUIRE,
  129.         memory_order_release = __ATOMIC_RELEASE,
  130.         memory_order_acq_rel = __ATOMIC_ACQ_REL,
  131.         memory_order_seq_cst = __ATOMIC_SEQ_CST
  132. } memory_order;
  133.  
  134. /*
  135.  * 7.17.4 Fences.
  136.  */
  137.  
  138. static __inline void
  139. atomic_thread_fence(memory_order __order __unused)
  140. {
  141.  
  142. #ifdef __CLANG_ATOMICS
  143.         __c11_atomic_thread_fence(__order);
  144. #elif defined(__GNUC_ATOMICS)
  145.         __atomic_thread_fence(__order);
  146. #else
  147.         __sync_synchronize();
  148. #endif
  149. }
  150.  
  151. static __inline void
  152. atomic_signal_fence(memory_order __order __unused)
  153. {
  154.  
  155. #ifdef __CLANG_ATOMICS
  156.         __c11_atomic_signal_fence(__order);
  157. #elif defined(__GNUC_ATOMICS)
  158.         __atomic_signal_fence(__order);
  159. #else
  160.         __asm volatile ("" ::: "memory");
  161. #endif
  162. }
  163.  
  164. /*
  165.  * 7.17.5 Lock-free property.
  166.  */
  167.  
  168. #if defined(_KERNEL)
  169. /* Atomics in kernelspace are always lock-free. */
  170. #define atomic_is_lock_free(obj) \
  171.         ((void)(obj), (_Bool)1)
  172. #elif defined(__CLANG_ATOMICS)
  173. #define atomic_is_lock_free(obj) \
  174.         __atomic_is_lock_free(sizeof(*(obj)), obj)
  175. #elif defined(__GNUC_ATOMICS)
  176. #define atomic_is_lock_free(obj) \
  177.         __atomic_is_lock_free(sizeof((obj)->__val), &(obj)->__val)
  178. #else
  179. #define atomic_is_lock_free(obj) \
  180.         ((void)(obj), sizeof((obj)->__val) <= sizeof(void *))
  181. #endif
  182.  
  183. /*
  184.  * 7.17.6 Atomic integer types.
  185.  */
  186.  
  187. typedef _Atomic(_Bool)                  atomic_bool;
  188. typedef _Atomic(char)                   atomic_char;
  189. typedef _Atomic(signed char)            atomic_schar;
  190. typedef _Atomic(unsigned char)          atomic_uchar;
  191. typedef _Atomic(short)                  atomic_short;
  192. typedef _Atomic(unsigned short)         atomic_ushort;
  193. typedef _Atomic(int)                    atomic_int;
  194. typedef _Atomic(unsigned int)           atomic_uint;
  195. typedef _Atomic(long)                   atomic_long;
  196. typedef _Atomic(unsigned long)          atomic_ulong;
  197. typedef _Atomic(long long)              atomic_llong;
  198. typedef _Atomic(unsigned long long)     atomic_ullong;
  199. #if 0
  200. typedef _Atomic(__char16_t)             atomic_char16_t;
  201. typedef _Atomic(__char32_t)             atomic_char32_t;
  202. #endif
  203. typedef _Atomic(wchar_t)                atomic_wchar_t;
  204. typedef _Atomic(int_least8_t)           atomic_int_least8_t;
  205. typedef _Atomic(uint_least8_t)          atomic_uint_least8_t;
  206. typedef _Atomic(int_least16_t)          atomic_int_least16_t;
  207. typedef _Atomic(uint_least16_t)         atomic_uint_least16_t;
  208. typedef _Atomic(int_least32_t)          atomic_int_least32_t;
  209. typedef _Atomic(uint_least32_t)         atomic_uint_least32_t;
  210. typedef _Atomic(int_least64_t)          atomic_int_least64_t;
  211. typedef _Atomic(uint_least64_t)         atomic_uint_least64_t;
  212. typedef _Atomic(int_fast8_t)            atomic_int_fast8_t;
  213. typedef _Atomic(uint_fast8_t)           atomic_uint_fast8_t;
  214. typedef _Atomic(int_fast16_t)           atomic_int_fast16_t;
  215. typedef _Atomic(uint_fast16_t)          atomic_uint_fast16_t;
  216. typedef _Atomic(int_fast32_t)           atomic_int_fast32_t;
  217. typedef _Atomic(uint_fast32_t)          atomic_uint_fast32_t;
  218. typedef _Atomic(int_fast64_t)           atomic_int_fast64_t;
  219. typedef _Atomic(uint_fast64_t)          atomic_uint_fast64_t;
  220. typedef _Atomic(intptr_t)               atomic_intptr_t;
  221. typedef _Atomic(uintptr_t)              atomic_uintptr_t;
  222. typedef _Atomic(size_t)                 atomic_size_t;
  223. typedef _Atomic(ptrdiff_t)              atomic_ptrdiff_t;
  224. typedef _Atomic(intmax_t)               atomic_intmax_t;
  225. typedef _Atomic(uintmax_t)              atomic_uintmax_t;
  226.  
  227. /*
  228.  * 7.17.7 Operations on atomic types.
  229.  */
  230.  
  231. /*
  232.  * Compiler-specific operations.
  233.  */
  234.  
  235. #if defined(__CLANG_ATOMICS)
  236. #define atomic_compare_exchange_strong_explicit(object, expected,       \
  237.     desired, success, failure)                                          \
  238.         __c11_atomic_compare_exchange_strong(object, expected, desired, \
  239.             success, failure)
  240. #define atomic_compare_exchange_weak_explicit(object, expected,         \
  241.     desired, success, failure)                                          \
  242.         __c11_atomic_compare_exchange_weak(object, expected, desired,   \
  243.             success, failure)
  244. #define atomic_exchange_explicit(object, desired, order)                \
  245.         __c11_atomic_exchange(object, desired, order)
  246. #define atomic_fetch_add_explicit(object, operand, order)               \
  247.         __c11_atomic_fetch_add(object, operand, order)
  248. #define atomic_fetch_and_explicit(object, operand, order)               \
  249.         __c11_atomic_fetch_and(object, operand, order)
  250. #define atomic_fetch_or_explicit(object, operand, order)                \
  251.         __c11_atomic_fetch_or(object, operand, order)
  252. #define atomic_fetch_sub_explicit(object, operand, order)               \
  253.         __c11_atomic_fetch_sub(object, operand, order)
  254. #define atomic_fetch_xor_explicit(object, operand, order)               \
  255.         __c11_atomic_fetch_xor(object, operand, order)
  256. #define atomic_load_explicit(object, order)                             \
  257.         __c11_atomic_load(object, order)
  258. #define atomic_store_explicit(object, desired, order)                   \
  259.         __c11_atomic_store(object, desired, order)
  260. #elif defined(__GNUC_ATOMICS)
  261. #define atomic_compare_exchange_strong_explicit(object, expected,       \
  262.     desired, success, failure)                                          \
  263.         __atomic_compare_exchange_n(&(object)->__val, expected,         \
  264.             desired, 0, success, failure)
  265. #define atomic_compare_exchange_weak_explicit(object, expected,         \
  266.     desired, success, failure)                                          \
  267.         __atomic_compare_exchange_n(&(object)->__val, expected,         \
  268.             desired, 1, success, failure)
  269. #define atomic_exchange_explicit(object, desired, order)                \
  270.         __atomic_exchange_n(&(object)->__val, desired, order)
  271. #define atomic_fetch_add_explicit(object, operand, order)               \
  272.         __atomic_fetch_add(&(object)->__val, operand, order)
  273. #define atomic_fetch_and_explicit(object, operand, order)               \
  274.         __atomic_fetch_and(&(object)->__val, operand, order)
  275. #define atomic_fetch_or_explicit(object, operand, order)                \
  276.         __atomic_fetch_or(&(object)->__val, operand, order)
  277. #define atomic_fetch_sub_explicit(object, operand, order)               \
  278.         __atomic_fetch_sub(&(object)->__val, operand, order)
  279. #define atomic_fetch_xor_explicit(object, operand, order)               \
  280.         __atomic_fetch_xor(&(object)->__val, operand, order)
  281. #define atomic_load_explicit(object, order)                             \
  282.         __atomic_load_n(&(object)->__val, order)
  283. #define atomic_store_explicit(object, desired, order)                   \
  284.         __atomic_store_n(&(object)->__val, desired, order)
  285. #else
  286. #define __atomic_apply_stride(object, operand) \
  287.         (((__typeof__((object)->__val))0) + (operand))
  288. #define atomic_compare_exchange_strong_explicit(object, expected,       \
  289.     desired, success, failure)  __extension__ ({                        \
  290.         __typeof__(expected) __ep = (expected);                         \
  291.         __typeof__(*__ep) __e = *__ep;                                  \
  292.         (void)(success); (void)(failure);                               \
  293.         (_Bool)((*__ep = __sync_val_compare_and_swap(&(object)->__val,  \
  294.             __e, desired)) == __e);                                     \
  295. })
  296. #define atomic_compare_exchange_weak_explicit(object, expected,         \
  297.     desired, success, failure)                                          \
  298.         atomic_compare_exchange_strong_explicit(object, expected,       \
  299.                 desired, success, failure)
  300. #if __has_builtin(__sync_swap)
  301. /* Clang provides a full-barrier atomic exchange - use it if available. */
  302. #define atomic_exchange_explicit(object, desired, order)                \
  303.         ((void)(order), __sync_swap(&(object)->__val, desired))
  304. #else
  305. /*
  306.  * __sync_lock_test_and_set() is only an acquire barrier in theory (although in
  307.  * practice it is usually a full barrier) so we need an explicit barrier before
  308.  * it.
  309.  */
  310. #define atomic_exchange_explicit(object, desired, order)                \
  311. __extension__ ({                                                        \
  312.         __typeof__(object) __o = (object);                              \
  313.         __typeof__(desired) __d = (desired);                            \
  314.         (void)(order);                                                  \
  315.         __sync_synchronize();                                           \
  316.         __sync_lock_test_and_set(&(__o)->__val, __d);                   \
  317. })
  318. #endif
  319. #define atomic_fetch_add_explicit(object, operand, order)               \
  320.         ((void)(order), __sync_fetch_and_add(&(object)->__val,          \
  321.             __atomic_apply_stride(object, operand)))
  322. #define atomic_fetch_and_explicit(object, operand, order)               \
  323.         ((void)(order), __sync_fetch_and_and(&(object)->__val, operand))
  324. #define atomic_fetch_or_explicit(object, operand, order)                \
  325.         ((void)(order), __sync_fetch_and_or(&(object)->__val, operand))
  326. #define atomic_fetch_sub_explicit(object, operand, order)               \
  327.         ((void)(order), __sync_fetch_and_sub(&(object)->__val,          \
  328.             __atomic_apply_stride(object, operand)))
  329. #define atomic_fetch_xor_explicit(object, operand, order)               \
  330.         ((void)(order), __sync_fetch_and_xor(&(object)->__val, operand))
  331. #define atomic_load_explicit(object, order)                             \
  332.         ((void)(order), __sync_fetch_and_add(&(object)->__val, 0))
  333. #define atomic_store_explicit(object, desired, order)                   \
  334.         ((void)atomic_exchange_explicit(object, desired, order))
  335. #endif
  336.  
  337. /*
  338.  * Convenience functions.
  339.  *
  340.  * Don't provide these in kernel space. In kernel space, we should be
  341.  * disciplined enough to always provide explicit barriers.
  342.  */
  343.  
  344. #ifndef _KERNEL
  345. #define atomic_compare_exchange_strong(object, expected, desired)       \
  346.         atomic_compare_exchange_strong_explicit(object, expected,       \
  347.             desired, memory_order_seq_cst, memory_order_seq_cst)
  348. #define atomic_compare_exchange_weak(object, expected, desired)         \
  349.         atomic_compare_exchange_weak_explicit(object, expected,         \
  350.             desired, memory_order_seq_cst, memory_order_seq_cst)
  351. #define atomic_exchange(object, desired)                                \
  352.         atomic_exchange_explicit(object, desired, memory_order_seq_cst)
  353. #define atomic_fetch_add(object, operand)                               \
  354.         atomic_fetch_add_explicit(object, operand, memory_order_seq_cst)
  355. #define atomic_fetch_and(object, operand)                               \
  356.         atomic_fetch_and_explicit(object, operand, memory_order_seq_cst)
  357. #define atomic_fetch_or(object, operand)                                \
  358.         atomic_fetch_or_explicit(object, operand, memory_order_seq_cst)
  359. #define atomic_fetch_sub(object, operand)                               \
  360.         atomic_fetch_sub_explicit(object, operand, memory_order_seq_cst)
  361. #define atomic_fetch_xor(object, operand)                               \
  362.         atomic_fetch_xor_explicit(object, operand, memory_order_seq_cst)
  363. #define atomic_load(object)                                             \
  364.         atomic_load_explicit(object, memory_order_seq_cst)
  365. #define atomic_store(object, desired)                                   \
  366.         atomic_store_explicit(object, desired, memory_order_seq_cst)
  367. #endif /* !_KERNEL */
  368.  
  369. /*
  370.  * 7.17.8 Atomic flag type and operations.
  371.  *
  372.  * XXX: Assume atomic_bool can be used as an atomic_flag. Is there some
  373.  * kind of compiler built-in type we could use?
  374.  */
  375.  
  376. typedef struct {
  377.         atomic_bool     __flag;
  378. } atomic_flag;
  379.  
  380. #define ATOMIC_FLAG_INIT                { ATOMIC_VAR_INIT(0) }
  381.  
  382. static __inline _Bool
  383. atomic_flag_test_and_set_explicit(volatile atomic_flag *__object,
  384.     memory_order __order)
  385. {
  386.         return (atomic_exchange_explicit(&__object->__flag, 1, __order));
  387. }
  388.  
  389. static __inline void
  390. atomic_flag_clear_explicit(volatile atomic_flag *__object, memory_order __order)
  391. {
  392.  
  393.         atomic_store_explicit(&__object->__flag, 0, __order);
  394. }
  395.  
  396. #ifndef _KERNEL
  397. static __inline _Bool
  398. atomic_flag_test_and_set(volatile atomic_flag *__object)
  399. {
  400.  
  401.         return (atomic_flag_test_and_set_explicit(__object,
  402.             memory_order_seq_cst));
  403. }
  404.  
  405. static __inline void
  406. atomic_flag_clear(volatile atomic_flag *__object)
  407. {
  408.  
  409.         atomic_flag_clear_explicit(__object, memory_order_seq_cst);
  410. }
  411. #endif /* !_KERNEL */
  412.  
  413. #endif /* !_STDATOMIC_H_ */
  414.