src/include/port/atomics/generic.h - cloudberry - Git at Google

 /*-------------------------------------------------------------------------
  *
  * generic.h
  *	  Implement higher level operations based on some lower level atomic
  *	  operations.
  *
  * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * src/include/port/atomics/generic.h
  *
  *-------------------------------------------------------------------------
  */

 /* intentionally no include guards, should only be included by atomics.h */
 #ifndef INSIDE_ATOMICS_H
 #	error "should be included via atomics.h"
 #endif

 /*
  * If read or write barriers are undefined, we upgrade them to full memory
  * barriers.
  */
 #if !defined(pg_read_barrier_impl)
 #	define pg_read_barrier_impl pg_memory_barrier_impl
 #endif
 #if !defined(pg_write_barrier_impl)
 #	define pg_write_barrier_impl pg_memory_barrier_impl
 #endif

 #ifndef PG_HAVE_SPIN_DELAY
 #define PG_HAVE_SPIN_DELAY
 #define pg_spin_delay_impl()	((void)0)
 #endif


 /* provide fallback */
 #if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) && defined(PG_HAVE_ATOMIC_U32_SUPPORT)
 #define PG_HAVE_ATOMIC_FLAG_SUPPORT
 typedef pg_atomic_uint32 pg_atomic_flag;
 #endif

 #ifndef PG_HAVE_ATOMIC_READ_U32
 #define PG_HAVE_ATOMIC_READ_U32
 static inline uint32
 pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr)
 {
 	return ptr->value;
 }
 #endif

 #ifndef PG_HAVE_ATOMIC_WRITE_U32
 #define PG_HAVE_ATOMIC_WRITE_U32
 static inline void
 pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val)
 {
 	ptr->value = val;
 }
 #endif

 #ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32
 #define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32
 static inline void
 pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val)
 {
 	ptr->value = val;
 }
 #endif

 /*
  * provide fallback for test_and_set using atomic_exchange if available
  */
 #if !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32)

 #define PG_HAVE_ATOMIC_INIT_FLAG
 static inline void
 pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr)
 {
 	pg_atomic_write_u32_impl(ptr, 0);
 }

 #define PG_HAVE_ATOMIC_TEST_SET_FLAG
 static inline bool
 pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr)
 {
 	return pg_atomic_exchange_u32_impl(ptr, 1) == 0;
 }

 #define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG
 static inline bool
 pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr)
 {
 	return pg_atomic_read_u32_impl(ptr) == 0;
 }


 #define PG_HAVE_ATOMIC_CLEAR_FLAG
 static inline void
 pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr)
 {
 	/* XXX: release semantics suffice? */
 	pg_memory_barrier_impl();
 	pg_atomic_write_u32_impl(ptr, 0);
 }

 /*
  * provide fallback for test_and_set using atomic_compare_exchange if
  * available.
  */
 #elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)

 #define PG_HAVE_ATOMIC_INIT_FLAG
 static inline void
 pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr)
 {
 	pg_atomic_write_u32_impl(ptr, 0);
 }

 #define PG_HAVE_ATOMIC_TEST_SET_FLAG
 static inline bool
 pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr)
 {
 	uint32 value = 0;
 	return pg_atomic_compare_exchange_u32_impl(ptr, &value, 1);
 }

 #define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG
 static inline bool
 pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr)
 {
 	return pg_atomic_read_u32_impl(ptr) == 0;
 }

 #define PG_HAVE_ATOMIC_CLEAR_FLAG
 static inline void
 pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr)
 {
 	/*
 	 * Use a memory barrier + plain write if we have a native memory
 	 * barrier. But don't do so if memory barriers use spinlocks - that'd lead
 	 * to circularity if flags are used to implement spinlocks.
 	 */
 #ifndef PG_HAVE_MEMORY_BARRIER_EMULATION
 	/* XXX: release semantics suffice? */
 	pg_memory_barrier_impl();
 	pg_atomic_write_u32_impl(ptr, 0);
 #else
 	uint32 value = 1;
 	pg_atomic_compare_exchange_u32_impl(ptr, &value, 0);
 #endif
 }

 #elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG)
 #	error "No pg_atomic_test_and_set provided"
 #endif /* !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) */


 #ifndef PG_HAVE_ATOMIC_INIT_U32
 #define PG_HAVE_ATOMIC_INIT_U32
 static inline void
 pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_)
 {
 	ptr->value = val_;
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
 #define PG_HAVE_ATOMIC_EXCHANGE_U32
 static inline uint32
 pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 xchg_)
 {
 	uint32 old;
 	old = ptr->value;			/* ok if read is not atomic */
 	while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, xchg_))
 		/* skip */;
 	return old;
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
 #define PG_HAVE_ATOMIC_FETCH_ADD_U32
 static inline uint32
 pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
 {
 	uint32 old;
 	old = ptr->value;			/* ok if read is not atomic */
 	while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old + add_))
 		/* skip */;
 	return old;
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
 #define PG_HAVE_ATOMIC_FETCH_SUB_U32
 static inline uint32
 pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_)
 {
 	return pg_atomic_fetch_add_u32_impl(ptr, -sub_);
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
 #define PG_HAVE_ATOMIC_FETCH_AND_U32
 static inline uint32
 pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_)
 {
 	uint32 old;
 	old = ptr->value;			/* ok if read is not atomic */
 	while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old & and_))
 		/* skip */;
 	return old;
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
 #define PG_HAVE_ATOMIC_FETCH_OR_U32
 static inline uint32
 pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_)
 {
 	uint32 old;
 	old = ptr->value;			/* ok if read is not atomic */
 	while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old | or_))
 		/* skip */;
 	return old;
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32)
 #define PG_HAVE_ATOMIC_ADD_FETCH_U32
 static inline uint32
 pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
 {
 	return pg_atomic_fetch_add_u32_impl(ptr, add_) + add_;
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U32)
 #define PG_HAVE_ATOMIC_SUB_FETCH_U32
 static inline uint32
 pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_)
 {
 	return pg_atomic_fetch_sub_u32_impl(ptr, sub_) - sub_;
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
 #define PG_HAVE_ATOMIC_EXCHANGE_U64
 static inline uint64
 pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 xchg_)
 {
 	uint64 old;
 	old = ptr->value;			/* ok if read is not atomic */
 	while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, xchg_))
 		/* skip */;
 	return old;
 }
 #endif

 #ifndef PG_HAVE_ATOMIC_WRITE_U64
 #define PG_HAVE_ATOMIC_WRITE_U64

 #if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \
 	!defined(PG_HAVE_ATOMIC_U64_SIMULATION)

 static inline void
 pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val)
 {
 	/*
 	 * On this platform aligned 64bit writes are guaranteed to be atomic,
 	 * except if using the fallback implementation, where can't guarantee the
 	 * required alignment.
 	 */
 	AssertPointerAlignment(ptr, 8);
 	ptr->value = val;
 }

 #else

 static inline void
 pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val)
 {
 	/*
 	 * 64 bit writes aren't safe on all platforms. In the generic
 	 * implementation implement them as an atomic exchange.
 	 */
 	pg_atomic_exchange_u64_impl(ptr, val);
 }

 #endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */
 #endif /* PG_HAVE_ATOMIC_WRITE_U64 */

 #ifndef PG_HAVE_ATOMIC_READ_U64
 #define PG_HAVE_ATOMIC_READ_U64

 #if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \
 	!defined(PG_HAVE_ATOMIC_U64_SIMULATION)

 static inline uint64
 pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr)
 {
 	/*
 	 * On this platform aligned 64-bit reads are guaranteed to be atomic.
 	 */
 	AssertPointerAlignment(ptr, 8);
 	return ptr->value;
 }

 #else

 static inline uint64
 pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr)
 {
 	uint64 old = 0;

 	/*
 	 * 64-bit reads aren't atomic on all platforms. In the generic
 	 * implementation implement them as a compare/exchange with 0. That'll
 	 * fail or succeed, but always return the old value. Possibly might store
 	 * a 0, but only if the previous value also was a 0 - i.e. harmless.
 	 */
 	pg_atomic_compare_exchange_u64_impl(ptr, &old, 0);

 	return old;
 }
 #endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */
 #endif /* PG_HAVE_ATOMIC_READ_U64 */

 #ifndef PG_HAVE_ATOMIC_INIT_U64
 #define PG_HAVE_ATOMIC_INIT_U64
 static inline void
 pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_)
 {
 	ptr->value = val_;
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
 #define PG_HAVE_ATOMIC_FETCH_ADD_U64
 static inline uint64
 pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
 {
 	uint64 old;
 	old = ptr->value;			/* ok if read is not atomic */
 	while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old + add_))
 		/* skip */;
 	return old;
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
 #define PG_HAVE_ATOMIC_FETCH_SUB_U64
 static inline uint64
 pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_)
 {
 	return pg_atomic_fetch_add_u64_impl(ptr, -sub_);
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
 #define PG_HAVE_ATOMIC_FETCH_AND_U64
 static inline uint64
 pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_)
 {
 	uint64 old;
 	old = ptr->value;			/* ok if read is not atomic */
 	while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old & and_))
 		/* skip */;
 	return old;
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
 #define PG_HAVE_ATOMIC_FETCH_OR_U64
 static inline uint64
 pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_)
 {
 	uint64 old;
 	old = ptr->value;			/* ok if read is not atomic */
 	while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old | or_))
 		/* skip */;
 	return old;
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64)
 #define PG_HAVE_ATOMIC_ADD_FETCH_U64
 static inline uint64
 pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
 {
 	return pg_atomic_fetch_add_u64_impl(ptr, add_) + add_;
 }
 #endif

 #if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U64)
 #define PG_HAVE_ATOMIC_SUB_FETCH_U64
 static inline uint64
 pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_)
 {
 	return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_;
 }
 #endif
	/*-------------------------------------------------------------------------
	*
	* generic.h
	* Implement higher level operations based on some lower level atomic
	* operations.
	*
	* Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
	* Portions Copyright (c) 1994, Regents of the University of California
	*
	* src/include/port/atomics/generic.h
	*
	*-------------------------------------------------------------------------
	*/

	/* intentionally no include guards, should only be included by atomics.h */
	#ifndef INSIDE_ATOMICS_H
	# error "should be included via atomics.h"
	#endif

	/*
	* If read or write barriers are undefined, we upgrade them to full memory
	* barriers.
	*/
	#if !defined(pg_read_barrier_impl)
	# define pg_read_barrier_impl pg_memory_barrier_impl
	#endif
	#if !defined(pg_write_barrier_impl)
	# define pg_write_barrier_impl pg_memory_barrier_impl
	#endif

	#ifndef PG_HAVE_SPIN_DELAY
	#define PG_HAVE_SPIN_DELAY
	#define pg_spin_delay_impl() ((void)0)
	#endif


	/* provide fallback */
	#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) && defined(PG_HAVE_ATOMIC_U32_SUPPORT)
	#define PG_HAVE_ATOMIC_FLAG_SUPPORT
	typedef pg_atomic_uint32 pg_atomic_flag;
	#endif

	#ifndef PG_HAVE_ATOMIC_READ_U32
	#define PG_HAVE_ATOMIC_READ_U32
	static inline uint32
	pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr)
	{
	return ptr->value;
	}
	#endif

	#ifndef PG_HAVE_ATOMIC_WRITE_U32
	#define PG_HAVE_ATOMIC_WRITE_U32
	static inline void
	pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val)
	{
	ptr->value = val;
	}
	#endif

	#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32
	#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32
	static inline void
	pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val)
	{
	ptr->value = val;
	}
	#endif

	/*
	* provide fallback for test_and_set using atomic_exchange if available
	*/
	#if !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32)

	#define PG_HAVE_ATOMIC_INIT_FLAG
	static inline void
	pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr)
	{
	pg_atomic_write_u32_impl(ptr, 0);
	}

	#define PG_HAVE_ATOMIC_TEST_SET_FLAG
	static inline bool
	pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr)
	{
	return pg_atomic_exchange_u32_impl(ptr, 1) == 0;
	}

	#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG
	static inline bool
	pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr)
	{
	return pg_atomic_read_u32_impl(ptr) == 0;
	}


	#define PG_HAVE_ATOMIC_CLEAR_FLAG
	static inline void
	pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr)
	{
	/* XXX: release semantics suffice? */
	pg_memory_barrier_impl();
	pg_atomic_write_u32_impl(ptr, 0);
	}

	/*
	* provide fallback for test_and_set using atomic_compare_exchange if
	* available.
	*/
	#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)

	#define PG_HAVE_ATOMIC_INIT_FLAG
	static inline void
	pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr)
	{
	pg_atomic_write_u32_impl(ptr, 0);
	}

	#define PG_HAVE_ATOMIC_TEST_SET_FLAG
	static inline bool
	pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr)
	{
	uint32 value = 0;
	return pg_atomic_compare_exchange_u32_impl(ptr, &value, 1);
	}

	#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG
	static inline bool
	pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr)
	{
	return pg_atomic_read_u32_impl(ptr) == 0;
	}

	#define PG_HAVE_ATOMIC_CLEAR_FLAG
	static inline void
	pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr)
	{
	/*
	* Use a memory barrier + plain write if we have a native memory
	* barrier. But don't do so if memory barriers use spinlocks - that'd lead
	* to circularity if flags are used to implement spinlocks.
	*/
	#ifndef PG_HAVE_MEMORY_BARRIER_EMULATION
	/* XXX: release semantics suffice? */
	pg_memory_barrier_impl();
	pg_atomic_write_u32_impl(ptr, 0);
	#else
	uint32 value = 1;
	pg_atomic_compare_exchange_u32_impl(ptr, &value, 0);
	#endif
	}

	#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG)
	# error "No pg_atomic_test_and_set provided"
	#endif /* !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) */


	#ifndef PG_HAVE_ATOMIC_INIT_U32
	#define PG_HAVE_ATOMIC_INIT_U32
	static inline void
	pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_)
	{
	ptr->value = val_;
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
	#define PG_HAVE_ATOMIC_EXCHANGE_U32
	static inline uint32
	pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 xchg_)
	{
	uint32 old;
	old = ptr->value; /* ok if read is not atomic */
	while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, xchg_))
	/* skip */;
	return old;
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
	#define PG_HAVE_ATOMIC_FETCH_ADD_U32
	static inline uint32
	pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
	{
	uint32 old;
	old = ptr->value; /* ok if read is not atomic */
	while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old + add_))
	/* skip */;
	return old;
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
	#define PG_HAVE_ATOMIC_FETCH_SUB_U32
	static inline uint32
	pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_)
	{
	return pg_atomic_fetch_add_u32_impl(ptr, -sub_);
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
	#define PG_HAVE_ATOMIC_FETCH_AND_U32
	static inline uint32
	pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_)
	{
	uint32 old;
	old = ptr->value; /* ok if read is not atomic */
	while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old & and_))
	/* skip */;
	return old;
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)
	#define PG_HAVE_ATOMIC_FETCH_OR_U32
	static inline uint32
	pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_)
	{
	uint32 old;
	old = ptr->value; /* ok if read is not atomic */
	while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old \| or_))
	/* skip */;
	return old;
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32)
	#define PG_HAVE_ATOMIC_ADD_FETCH_U32
	static inline uint32
	pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
	{
	return pg_atomic_fetch_add_u32_impl(ptr, add_) + add_;
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U32)
	#define PG_HAVE_ATOMIC_SUB_FETCH_U32
	static inline uint32
	pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_)
	{
	return pg_atomic_fetch_sub_u32_impl(ptr, sub_) - sub_;
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
	#define PG_HAVE_ATOMIC_EXCHANGE_U64
	static inline uint64
	pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 xchg_)
	{
	uint64 old;
	old = ptr->value; /* ok if read is not atomic */
	while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, xchg_))
	/* skip */;
	return old;
	}
	#endif

	#ifndef PG_HAVE_ATOMIC_WRITE_U64
	#define PG_HAVE_ATOMIC_WRITE_U64

	#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \
	!defined(PG_HAVE_ATOMIC_U64_SIMULATION)

	static inline void
	pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val)
	{
	/*
	* On this platform aligned 64bit writes are guaranteed to be atomic,
	* except if using the fallback implementation, where can't guarantee the
	* required alignment.
	*/
	AssertPointerAlignment(ptr, 8);
	ptr->value = val;
	}

	#else

	static inline void
	pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val)
	{
	/*
	* 64 bit writes aren't safe on all platforms. In the generic
	* implementation implement them as an atomic exchange.
	*/
	pg_atomic_exchange_u64_impl(ptr, val);
	}

	#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */
	#endif /* PG_HAVE_ATOMIC_WRITE_U64 */

	#ifndef PG_HAVE_ATOMIC_READ_U64
	#define PG_HAVE_ATOMIC_READ_U64

	#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \
	!defined(PG_HAVE_ATOMIC_U64_SIMULATION)

	static inline uint64
	pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr)
	{
	/*
	* On this platform aligned 64-bit reads are guaranteed to be atomic.
	*/
	AssertPointerAlignment(ptr, 8);
	return ptr->value;
	}

	#else

	static inline uint64
	pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr)
	{
	uint64 old = 0;

	/*
	* 64-bit reads aren't atomic on all platforms. In the generic
	* implementation implement them as a compare/exchange with 0. That'll
	* fail or succeed, but always return the old value. Possibly might store
	* a 0, but only if the previous value also was a 0 - i.e. harmless.
	*/
	pg_atomic_compare_exchange_u64_impl(ptr, &old, 0);

	return old;
	}
	#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */
	#endif /* PG_HAVE_ATOMIC_READ_U64 */

	#ifndef PG_HAVE_ATOMIC_INIT_U64
	#define PG_HAVE_ATOMIC_INIT_U64
	static inline void
	pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_)
	{
	ptr->value = val_;
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
	#define PG_HAVE_ATOMIC_FETCH_ADD_U64
	static inline uint64
	pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
	{
	uint64 old;
	old = ptr->value; /* ok if read is not atomic */
	while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old + add_))
	/* skip */;
	return old;
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
	#define PG_HAVE_ATOMIC_FETCH_SUB_U64
	static inline uint64
	pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_)
	{
	return pg_atomic_fetch_add_u64_impl(ptr, -sub_);
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
	#define PG_HAVE_ATOMIC_FETCH_AND_U64
	static inline uint64
	pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_)
	{
	uint64 old;
	old = ptr->value; /* ok if read is not atomic */
	while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old & and_))
	/* skip */;
	return old;
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
	#define PG_HAVE_ATOMIC_FETCH_OR_U64
	static inline uint64
	pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_)
	{
	uint64 old;
	old = ptr->value; /* ok if read is not atomic */
	while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old \| or_))
	/* skip */;
	return old;
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64)
	#define PG_HAVE_ATOMIC_ADD_FETCH_U64
	static inline uint64
	pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
	{
	return pg_atomic_fetch_add_u64_impl(ptr, add_) + add_;
	}
	#endif

	#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U64)
	#define PG_HAVE_ATOMIC_SUB_FETCH_U64
	static inline uint64
	pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_)
	{
	return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_;
	}
	#endif