Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * atomics.h
4 : * Atomic operations.
5 : *
6 : * Hardware and compiler dependent functions for manipulating memory
7 : * atomically and dealing with cache coherency. Used to implement locking
8 : * facilities and lockless algorithms/data structures.
9 : *
10 : * To bring up postgres on a platform/compiler at the very least
11 : * implementations for the following operations should be provided:
12 : * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier()
13 : * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32()
14 : * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag()
15 : * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate.
16 : *
17 : * There exist generic, hardware independent, implementations for several
18 : * compilers which might be sufficient, although possibly not optimal, for a
19 : * new platform. If no such generic implementation is available spinlocks (or
20 : * even OS provided semaphores) will be used to implement the API.
21 : *
22 : * Implement _u64 atomics if and only if your platform can use them
23 : * efficiently (and obviously correctly).
24 : *
25 : * Use higher level functionality (lwlocks, spinlocks, heavyweight locks)
26 : * whenever possible. Writing correct code using these facilities is hard.
27 : *
28 : * For an introduction to using memory barriers within the PostgreSQL backend,
29 : * see src/backend/storage/lmgr/README.barrier
30 : *
31 : * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
32 : * Portions Copyright (c) 1994, Regents of the University of California
33 : *
34 : * src/include/port/atomics.h
35 : *
36 : *-------------------------------------------------------------------------
37 : */
38 : #ifndef ATOMICS_H
39 : #define ATOMICS_H
40 :
41 : #ifdef FRONTEND
42 : #error "atomics.h may not be included from frontend code"
43 : #endif
44 :
45 : #define INSIDE_ATOMICS_H
46 :
47 : #include <limits.h>
48 :
49 : /*
50 : * First a set of architecture specific files is included.
51 : *
52 : * These files can provide the full set of atomics or can do pretty much
53 : * nothing if all the compilers commonly used on these platforms provide
54 : * usable generics.
55 : *
56 : * Don't add an inline assembly of the actual atomic operations if all the
57 : * common implementations of your platform provide intrinsics. Intrinsics are
58 : * much easier to understand and potentially support more architectures.
59 : *
60 : * It will often make sense to define memory barrier semantics here, since
61 : * e.g. generic compiler intrinsics for x86 memory barriers can't know that
62 : * postgres doesn't need x86 read/write barriers do anything more than a
63 : * compiler barrier.
64 : *
65 : */
66 : #if defined(__arm__) || defined(__arm) || \
67 : defined(__aarch64__) || defined(__aarch64)
68 : #include "port/atomics/arch-arm.h"
69 : #elif defined(__i386__) || defined(__i386) || defined(__x86_64__)
70 : #include "port/atomics/arch-x86.h"
71 : #elif defined(__ia64__) || defined(__ia64)
72 : #include "port/atomics/arch-ia64.h"
73 : #elif defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__)
74 : #include "port/atomics/arch-ppc.h"
75 : #elif defined(__hppa) || defined(__hppa__)
76 : #include "port/atomics/arch-hppa.h"
77 : #endif
78 :
79 : /*
80 : * Compiler specific, but architecture independent implementations.
81 : *
82 : * Provide architecture independent implementations of the atomic
83 : * facilities. At the very least compiler barriers should be provided, but a
84 : * full implementation of
85 : * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier()
86 : * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32()
87 : * using compiler intrinsics are a good idea.
88 : */
89 : /*
90 : * Given a gcc-compatible xlc compiler, prefer the xlc implementation. The
91 : * ppc64le "IBM XL C/C++ for Linux, V13.1.2" implements both interfaces, but
92 : * __sync_lock_test_and_set() of one-byte types elicits SIGSEGV.
93 : */
94 : #if defined(__IBMC__) || defined(__IBMCPP__)
95 : #include "port/atomics/generic-xlc.h"
96 : /* gcc or compatible, including clang and icc */
97 : #elif defined(__GNUC__) || defined(__INTEL_COMPILER)
98 : #include "port/atomics/generic-gcc.h"
99 : #elif defined(_MSC_VER)
100 : #include "port/atomics/generic-msvc.h"
101 : #elif defined(__hpux) && defined(__ia64) && !defined(__GNUC__)
102 : #include "port/atomics/generic-acc.h"
103 : #elif defined(__SUNPRO_C) && !defined(__GNUC__)
104 : #include "port/atomics/generic-sunpro.h"
105 : #else
106 : /*
107 : * Unsupported compiler, we'll likely use slower fallbacks... At least
108 : * compiler barriers should really be provided.
109 : */
110 : #endif
111 :
112 : /*
113 : * Provide a full fallback of the pg_*_barrier(), pg_atomic**_flag and
114 : * pg_atomic_* APIs for platforms without sufficient spinlock and/or atomics
115 : * support. In the case of spinlock backed atomics the emulation is expected
116 : * to be efficient, although less so than native atomics support.
117 : */
118 : #include "port/atomics/fallback.h"
119 :
120 : /*
121 : * Provide additional operations using supported infrastructure. These are
122 : * expected to be efficient if the underlying atomic operations are efficient.
123 : */
124 : #include "port/atomics/generic.h"
125 :
126 :
127 : /*
128 : * pg_compiler_barrier - prevent the compiler from moving code across
129 : *
130 : * A compiler barrier need not (and preferably should not) emit any actual
131 : * machine code, but must act as an optimization fence: the compiler must not
132 : * reorder loads or stores to main memory around the barrier. However, the
133 : * CPU may still reorder loads or stores at runtime, if the architecture's
134 : * memory model permits this.
135 : */
136 : #define pg_compiler_barrier() pg_compiler_barrier_impl()
137 :
138 : /*
139 : * pg_memory_barrier - prevent the CPU from reordering memory access
140 : *
141 : * A memory barrier must act as a compiler barrier, and in addition must
142 : * guarantee that all loads and stores issued prior to the barrier are
143 : * completed before any loads or stores issued after the barrier. Unless
144 : * loads and stores are totally ordered (which is not the case on most
145 : * architectures) this requires issuing some sort of memory fencing
146 : * instruction.
147 : */
148 : #define pg_memory_barrier() pg_memory_barrier_impl()
149 :
150 : /*
151 : * pg_(read|write)_barrier - prevent the CPU from reordering memory access
152 : *
153 : * A read barrier must act as a compiler barrier, and in addition must
154 : * guarantee that any loads issued prior to the barrier are completed before
155 : * any loads issued after the barrier. Similarly, a write barrier acts
156 : * as a compiler barrier, and also orders stores. Read and write barriers
157 : * are thus weaker than a full memory barrier, but stronger than a compiler
158 : * barrier. In practice, on machines with strong memory ordering, read and
159 : * write barriers may require nothing more than a compiler barrier.
160 : */
161 : #define pg_read_barrier() pg_read_barrier_impl()
162 : #define pg_write_barrier() pg_write_barrier_impl()
163 :
164 : /*
165 : * Spinloop delay - Allow CPU to relax in busy loops
166 : */
167 : #define pg_spin_delay() pg_spin_delay_impl()
168 :
169 : /*
170 : * pg_atomic_init_flag - initialize atomic flag.
171 : *
172 : * No barrier semantics.
173 : */
174 : static inline void
175 1 : pg_atomic_init_flag(volatile pg_atomic_flag *ptr)
176 : {
177 : AssertPointerAlignment(ptr, sizeof(*ptr));
178 :
179 1 : pg_atomic_init_flag_impl(ptr);
180 1 : }
181 :
182 : /*
183 : * pg_atomic_test_and_set_flag - TAS()
184 : *
185 : * Returns true if the flag has successfully been set, false otherwise.
186 : *
187 : * Acquire (including read barrier) semantics.
188 : */
189 : static inline bool
190 3 : pg_atomic_test_set_flag(volatile pg_atomic_flag *ptr)
191 : {
192 : AssertPointerAlignment(ptr, sizeof(*ptr));
193 :
194 3 : return pg_atomic_test_set_flag_impl(ptr);
195 : }
196 :
197 : /*
198 : * pg_atomic_unlocked_test_flag - Check if the lock is free
199 : *
200 : * Returns true if the flag currently is not set, false otherwise.
201 : *
202 : * No barrier semantics.
203 : */
204 : static inline bool
205 3 : pg_atomic_unlocked_test_flag(volatile pg_atomic_flag *ptr)
206 : {
207 : AssertPointerAlignment(ptr, sizeof(*ptr));
208 :
209 3 : return pg_atomic_unlocked_test_flag_impl(ptr);
210 : }
211 :
212 : /*
213 : * pg_atomic_clear_flag - release lock set by TAS()
214 : *
215 : * Release (including write barrier) semantics.
216 : */
217 : static inline void
218 2 : pg_atomic_clear_flag(volatile pg_atomic_flag *ptr)
219 : {
220 : AssertPointerAlignment(ptr, sizeof(*ptr));
221 :
222 2 : pg_atomic_clear_flag_impl(ptr);
223 2 : }
224 :
225 :
226 : /*
227 : * pg_atomic_init_u32 - initialize atomic variable
228 : *
229 : * Has to be done before any concurrent usage..
230 : *
231 : * No barrier semantics.
232 : */
233 : static inline void
234 203363 : pg_atomic_init_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
235 : {
236 203363 : AssertPointerAlignment(ptr, 4);
237 :
238 203363 : pg_atomic_init_u32_impl(ptr, val);
239 203363 : }
240 :
241 : /*
242 : * pg_atomic_read_u32 - unlocked read from atomic variable.
243 : *
244 : * The read is guaranteed to return a value as it has been written by this or
245 : * another process at some point in the past. There's however no cache
246 : * coherency interaction guaranteeing the value hasn't since been written to
247 : * again.
248 : *
249 : * No barrier semantics.
250 : */
251 : static inline uint32
252 24105873 : pg_atomic_read_u32(volatile pg_atomic_uint32 *ptr)
253 : {
254 24105873 : AssertPointerAlignment(ptr, 4);
255 24105873 : return pg_atomic_read_u32_impl(ptr);
256 : }
257 :
258 : /*
259 : * pg_atomic_write_u32 - write to atomic variable.
260 : *
261 : * The write is guaranteed to succeed as a whole, i.e. it's not possible to
262 : * observe a partial write for any reader. Note that this correctly interacts
263 : * with pg_atomic_compare_exchange_u32, in contrast to
264 : * pg_atomic_unlocked_write_u32().
265 : *
266 : * No barrier semantics.
267 : */
268 : static inline void
269 322366 : pg_atomic_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
270 : {
271 322366 : AssertPointerAlignment(ptr, 4);
272 :
273 322366 : pg_atomic_write_u32_impl(ptr, val);
274 322366 : }
275 :
276 : /*
277 : * pg_atomic_unlocked_write_u32 - unlocked write to atomic variable.
278 : *
279 : * The write is guaranteed to succeed as a whole, i.e. it's not possible to
280 : * observe a partial write for any reader. But note that writing this way is
281 : * not guaranteed to correctly interact with read-modify-write operations like
282 : * pg_atomic_compare_exchange_u32. This should only be used in cases where
283 : * minor performance regressions due to atomics emulation are unacceptable.
284 : *
285 : * No barrier semantics.
286 : */
287 : static inline void
288 71730 : pg_atomic_unlocked_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val)
289 : {
290 71730 : AssertPointerAlignment(ptr, 4);
291 :
292 71730 : pg_atomic_unlocked_write_u32_impl(ptr, val);
293 71730 : }
294 :
295 : /*
296 : * pg_atomic_exchange_u32 - exchange newval with current value
297 : *
298 : * Returns the old value of 'ptr' before the swap.
299 : *
300 : * Full barrier semantics.
301 : */
302 : static inline uint32
303 397 : pg_atomic_exchange_u32(volatile pg_atomic_uint32 *ptr, uint32 newval)
304 : {
305 397 : AssertPointerAlignment(ptr, 4);
306 :
307 397 : return pg_atomic_exchange_u32_impl(ptr, newval);
308 : }
309 :
310 : /*
311 : * pg_atomic_compare_exchange_u32 - CAS operation
312 : *
313 : * Atomically compare the current value of ptr with *expected and store newval
314 : * iff ptr and *expected have the same value. The current value of *ptr will
315 : * always be stored in *expected.
316 : *
317 : * Return true if values have been exchanged, false otherwise.
318 : *
319 : * Full barrier semantics.
320 : */
321 : static inline bool
322 21814680 : pg_atomic_compare_exchange_u32(volatile pg_atomic_uint32 *ptr,
323 : uint32 *expected, uint32 newval)
324 : {
325 21814680 : AssertPointerAlignment(ptr, 4);
326 21814680 : AssertPointerAlignment(expected, 4);
327 :
328 21814680 : return pg_atomic_compare_exchange_u32_impl(ptr, expected, newval);
329 : }
330 :
331 : /*
332 : * pg_atomic_fetch_add_u32 - atomically add to variable
333 : *
334 : * Returns the value of ptr before the arithmetic operation.
335 : *
336 : * Full barrier semantics.
337 : */
338 : static inline uint32
339 16656 : pg_atomic_fetch_add_u32(volatile pg_atomic_uint32 *ptr, int32 add_)
340 : {
341 16656 : AssertPointerAlignment(ptr, 4);
342 16656 : return pg_atomic_fetch_add_u32_impl(ptr, add_);
343 : }
344 :
345 : /*
346 : * pg_atomic_fetch_sub_u32 - atomically subtract from variable
347 : *
348 : * Returns the value of ptr before the arithmetic operation. Note that sub_
349 : * may not be INT_MIN due to platform limitations.
350 : *
351 : * Full barrier semantics.
352 : */
353 : static inline uint32
354 2 : pg_atomic_fetch_sub_u32(volatile pg_atomic_uint32 *ptr, int32 sub_)
355 : {
356 2 : AssertPointerAlignment(ptr, 4);
357 2 : Assert(sub_ != INT_MIN);
358 2 : return pg_atomic_fetch_sub_u32_impl(ptr, sub_);
359 : }
360 :
361 : /*
362 : * pg_atomic_fetch_and_u32 - atomically bit-and and_ with variable
363 : *
364 : * Returns the value of ptr before the arithmetic operation.
365 : *
366 : * Full barrier semantics.
367 : */
368 : static inline uint32
369 1394658 : pg_atomic_fetch_and_u32(volatile pg_atomic_uint32 *ptr, uint32 and_)
370 : {
371 1394658 : AssertPointerAlignment(ptr, 4);
372 1394658 : return pg_atomic_fetch_and_u32_impl(ptr, and_);
373 : }
374 :
375 : /*
376 : * pg_atomic_fetch_or_u32 - atomically bit-or or_ with variable
377 : *
378 : * Returns the value of ptr before the arithmetic operation.
379 : *
380 : * Full barrier semantics.
381 : */
382 : static inline uint32
383 1720462 : pg_atomic_fetch_or_u32(volatile pg_atomic_uint32 *ptr, uint32 or_)
384 : {
385 1720462 : AssertPointerAlignment(ptr, 4);
386 1720462 : return pg_atomic_fetch_or_u32_impl(ptr, or_);
387 : }
388 :
389 : /*
390 : * pg_atomic_add_fetch_u32 - atomically add to variable
391 : *
392 : * Returns the value of ptr after the arithmetic operation.
393 : *
394 : * Full barrier semantics.
395 : */
396 : static inline uint32
397 47 : pg_atomic_add_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 add_)
398 : {
399 47 : AssertPointerAlignment(ptr, 4);
400 47 : return pg_atomic_add_fetch_u32_impl(ptr, add_);
401 : }
402 :
403 : /*
404 : * pg_atomic_sub_fetch_u32 - atomically subtract from variable
405 : *
406 : * Returns the value of ptr after the arithmetic operation. Note that sub_ may
407 : * not be INT_MIN due to platform limitations.
408 : *
409 : * Full barrier semantics.
410 : */
411 : static inline uint32
412 14180330 : pg_atomic_sub_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 sub_)
413 : {
414 14180330 : AssertPointerAlignment(ptr, 4);
415 14180330 : Assert(sub_ != INT_MIN);
416 14180330 : return pg_atomic_sub_fetch_u32_impl(ptr, sub_);
417 : }
418 :
419 : /* ----
420 : * The 64 bit operations have the same semantics as their 32bit counterparts
421 : * if they are available. Check the corresponding 32bit function for
422 : * documentation.
423 : * ----
424 : */
425 : static inline void
426 14 : pg_atomic_init_u64(volatile pg_atomic_uint64 *ptr, uint64 val)
427 : {
428 : /*
429 : * Can't necessarily enforce alignment - and don't need it - when using
430 : * the spinlock based fallback implementation. Therefore only assert when
431 : * not using it.
432 : */
433 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
434 14 : AssertPointerAlignment(ptr, 8);
435 : #endif
436 14 : pg_atomic_init_u64_impl(ptr, val);
437 14 : }
438 :
439 : static inline uint64
440 4 : pg_atomic_read_u64(volatile pg_atomic_uint64 *ptr)
441 : {
442 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
443 4 : AssertPointerAlignment(ptr, 8);
444 : #endif
445 4 : return pg_atomic_read_u64_impl(ptr);
446 : }
447 :
448 : static inline void
449 4 : pg_atomic_write_u64(volatile pg_atomic_uint64 *ptr, uint64 val)
450 : {
451 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
452 4 : AssertPointerAlignment(ptr, 8);
453 : #endif
454 4 : pg_atomic_write_u64_impl(ptr, val);
455 4 : }
456 :
457 : static inline uint64
458 2 : pg_atomic_exchange_u64(volatile pg_atomic_uint64 *ptr, uint64 newval)
459 : {
460 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
461 2 : AssertPointerAlignment(ptr, 8);
462 : #endif
463 2 : return pg_atomic_exchange_u64_impl(ptr, newval);
464 : }
465 :
466 : static inline bool
467 3 : pg_atomic_compare_exchange_u64(volatile pg_atomic_uint64 *ptr,
468 : uint64 *expected, uint64 newval)
469 : {
470 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
471 3 : AssertPointerAlignment(ptr, 8);
472 3 : AssertPointerAlignment(expected, 8);
473 : #endif
474 3 : return pg_atomic_compare_exchange_u64_impl(ptr, expected, newval);
475 : }
476 :
477 : static inline uint64
478 3165 : pg_atomic_fetch_add_u64(volatile pg_atomic_uint64 *ptr, int64 add_)
479 : {
480 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
481 3165 : AssertPointerAlignment(ptr, 8);
482 : #endif
483 3165 : return pg_atomic_fetch_add_u64_impl(ptr, add_);
484 : }
485 :
486 : static inline uint64
487 1 : pg_atomic_fetch_sub_u64(volatile pg_atomic_uint64 *ptr, int64 sub_)
488 : {
489 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
490 1 : AssertPointerAlignment(ptr, 8);
491 : #endif
492 1 : Assert(sub_ != PG_INT64_MIN);
493 1 : return pg_atomic_fetch_sub_u64_impl(ptr, sub_);
494 : }
495 :
496 : static inline uint64
497 3 : pg_atomic_fetch_and_u64(volatile pg_atomic_uint64 *ptr, uint64 and_)
498 : {
499 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
500 3 : AssertPointerAlignment(ptr, 8);
501 : #endif
502 3 : return pg_atomic_fetch_and_u64_impl(ptr, and_);
503 : }
504 :
505 : static inline uint64
506 2 : pg_atomic_fetch_or_u64(volatile pg_atomic_uint64 *ptr, uint64 or_)
507 : {
508 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
509 2 : AssertPointerAlignment(ptr, 8);
510 : #endif
511 2 : return pg_atomic_fetch_or_u64_impl(ptr, or_);
512 : }
513 :
514 : static inline uint64
515 1 : pg_atomic_add_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 add_)
516 : {
517 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
518 1 : AssertPointerAlignment(ptr, 8);
519 : #endif
520 1 : return pg_atomic_add_fetch_u64_impl(ptr, add_);
521 : }
522 :
523 : static inline uint64
524 1 : pg_atomic_sub_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 sub_)
525 : {
526 : #ifndef PG_HAVE_ATOMIC_U64_SIMULATION
527 1 : AssertPointerAlignment(ptr, 8);
528 : #endif
529 1 : Assert(sub_ != PG_INT64_MIN);
530 1 : return pg_atomic_sub_fetch_u64_impl(ptr, sub_);
531 : }
532 :
533 : #undef INSIDE_ATOMICS_H
534 :
535 : #endif /* ATOMICS_H */
|
