|
|
|
@ -56,140 +56,128 @@
|
|
|
|
|
/* Unsigned */
|
|
|
|
|
ATOMIC_INLINE uint64_t atomic_add_and_fetch_uint64(uint64_t *p, uint64_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_add_and_fetch(p, x);
|
|
|
|
|
return __sync_add_and_fetch(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint64_t atomic_sub_and_fetch_uint64(uint64_t *p, uint64_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_sub_and_fetch(p, x);
|
|
|
|
|
return __sync_sub_and_fetch(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint64_t atomic_fetch_and_add_uint64(uint64_t *p, uint64_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_add(p, x);
|
|
|
|
|
return __sync_fetch_and_add(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint64_t atomic_fetch_and_sub_uint64(uint64_t *p, uint64_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_sub(p, x);
|
|
|
|
|
return __sync_fetch_and_sub(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint64_t atomic_cas_uint64(uint64_t *v, uint64_t old, uint64_t _new)
|
|
|
|
|
{
|
|
|
|
|
return __sync_val_compare_and_swap(v, old, _new);
|
|
|
|
|
return __sync_val_compare_and_swap(v, old, _new);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Signed */
|
|
|
|
|
ATOMIC_INLINE int64_t atomic_add_and_fetch_int64(int64_t *p, int64_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_add_and_fetch(p, x);
|
|
|
|
|
return __sync_add_and_fetch(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int64_t atomic_sub_and_fetch_int64(int64_t *p, int64_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_sub_and_fetch(p, x);
|
|
|
|
|
return __sync_sub_and_fetch(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int64_t atomic_fetch_and_add_int64(int64_t *p, int64_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_add(p, x);
|
|
|
|
|
return __sync_fetch_and_add(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int64_t atomic_fetch_and_sub_int64(int64_t *p, int64_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_sub(p, x);
|
|
|
|
|
return __sync_fetch_and_sub(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int64_t atomic_cas_int64(int64_t *v, int64_t old, int64_t _new)
|
|
|
|
|
{
|
|
|
|
|
return __sync_val_compare_and_swap(v, old, _new);
|
|
|
|
|
return __sync_val_compare_and_swap(v, old, _new);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# elif (defined(__amd64__) || defined(__x86_64__))
|
|
|
|
|
/* Unsigned */
|
|
|
|
|
ATOMIC_INLINE uint64_t atomic_fetch_and_add_uint64(uint64_t *p, uint64_t x)
|
|
|
|
|
{
|
|
|
|
|
asm volatile (
|
|
|
|
|
"lock; xaddq %0, %1;"
|
|
|
|
|
: "+r" (x), "=m" (*p) /* Outputs. */
|
|
|
|
|
: "m" (*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return x;
|
|
|
|
|
asm volatile("lock; xaddq %0, %1;"
|
|
|
|
|
: "+r"(x), "=m"(*p) /* Outputs. */
|
|
|
|
|
: "m"(*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint64_t atomic_fetch_and_sub_uint64(uint64_t *p, uint64_t x)
|
|
|
|
|
{
|
|
|
|
|
x = (uint64_t)(-(int64_t)x);
|
|
|
|
|
asm volatile (
|
|
|
|
|
"lock; xaddq %0, %1;"
|
|
|
|
|
: "+r" (x), "=m" (*p) /* Outputs. */
|
|
|
|
|
: "m" (*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return x;
|
|
|
|
|
x = (uint64_t)(-(int64_t)x);
|
|
|
|
|
asm volatile("lock; xaddq %0, %1;"
|
|
|
|
|
: "+r"(x), "=m"(*p) /* Outputs. */
|
|
|
|
|
: "m"(*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint64_t atomic_add_and_fetch_uint64(uint64_t *p, uint64_t x)
|
|
|
|
|
{
|
|
|
|
|
return atomic_fetch_and_add_uint64(p, x) + x;
|
|
|
|
|
return atomic_fetch_and_add_uint64(p, x) + x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint64_t atomic_sub_and_fetch_uint64(uint64_t *p, uint64_t x)
|
|
|
|
|
{
|
|
|
|
|
return atomic_fetch_and_sub_uint64(p, x) - x;
|
|
|
|
|
return atomic_fetch_and_sub_uint64(p, x) - x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint64_t atomic_cas_uint64(uint64_t *v, uint64_t old, uint64_t _new)
|
|
|
|
|
{
|
|
|
|
|
uint64_t ret;
|
|
|
|
|
asm volatile (
|
|
|
|
|
"lock; cmpxchgq %2,%1"
|
|
|
|
|
: "=a" (ret), "+m" (*v)
|
|
|
|
|
: "r" (_new), "0" (old)
|
|
|
|
|
: "memory");
|
|
|
|
|
return ret;
|
|
|
|
|
uint64_t ret;
|
|
|
|
|
asm volatile("lock; cmpxchgq %2,%1" : "=a"(ret), "+m"(*v) : "r"(_new), "0"(old) : "memory");
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Signed */
|
|
|
|
|
ATOMIC_INLINE int64_t atomic_fetch_and_add_int64(int64_t *p, int64_t x)
|
|
|
|
|
{
|
|
|
|
|
asm volatile (
|
|
|
|
|
"lock; xaddq %0, %1;"
|
|
|
|
|
: "+r" (x), "=m" (*p) /* Outputs. */
|
|
|
|
|
: "m" (*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return x;
|
|
|
|
|
asm volatile("lock; xaddq %0, %1;"
|
|
|
|
|
: "+r"(x), "=m"(*p) /* Outputs. */
|
|
|
|
|
: "m"(*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int64_t atomic_fetch_and_sub_int64(int64_t *p, int64_t x)
|
|
|
|
|
{
|
|
|
|
|
x = -x;
|
|
|
|
|
asm volatile (
|
|
|
|
|
"lock; xaddq %0, %1;"
|
|
|
|
|
: "+r" (x), "=m" (*p) /* Outputs. */
|
|
|
|
|
: "m" (*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return x;
|
|
|
|
|
x = -x;
|
|
|
|
|
asm volatile("lock; xaddq %0, %1;"
|
|
|
|
|
: "+r"(x), "=m"(*p) /* Outputs. */
|
|
|
|
|
: "m"(*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int64_t atomic_add_and_fetch_int64(int64_t *p, int64_t x)
|
|
|
|
|
{
|
|
|
|
|
return atomic_fetch_and_add_int64(p, x) + x;
|
|
|
|
|
return atomic_fetch_and_add_int64(p, x) + x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int64_t atomic_sub_and_fetch_int64(int64_t *p, int64_t x)
|
|
|
|
|
{
|
|
|
|
|
return atomic_fetch_and_sub_int64(p, x) - x;
|
|
|
|
|
return atomic_fetch_and_sub_int64(p, x) - x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int64_t atomic_cas_int64(int64_t *v, int64_t old, int64_t _new)
|
|
|
|
|
{
|
|
|
|
|
int64_t ret;
|
|
|
|
|
asm volatile (
|
|
|
|
|
"lock; cmpxchgq %2,%1"
|
|
|
|
|
: "=a" (ret), "+m" (*v)
|
|
|
|
|
: "r" (_new), "0" (old)
|
|
|
|
|
: "memory");
|
|
|
|
|
return ret;
|
|
|
|
|
int64_t ret;
|
|
|
|
|
asm volatile("lock; cmpxchgq %2,%1" : "=a"(ret), "+m"(*v) : "r"(_new), "0"(old) : "memory");
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
# else
|
|
|
|
|
# error "Missing implementation for 64-bit atomic operations"
|
|
|
|
@ -202,102 +190,90 @@ ATOMIC_INLINE int64_t atomic_cas_int64(int64_t *v, int64_t old, int64_t _new)
|
|
|
|
|
/* Unsigned */
|
|
|
|
|
ATOMIC_INLINE uint32_t atomic_add_and_fetch_uint32(uint32_t *p, uint32_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_add_and_fetch(p, x);
|
|
|
|
|
return __sync_add_and_fetch(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint32_t atomic_sub_and_fetch_uint32(uint32_t *p, uint32_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_sub_and_fetch(p, x);
|
|
|
|
|
return __sync_sub_and_fetch(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint32_t atomic_cas_uint32(uint32_t *v, uint32_t old, uint32_t _new)
|
|
|
|
|
{
|
|
|
|
|
return __sync_val_compare_and_swap(v, old, _new);
|
|
|
|
|
return __sync_val_compare_and_swap(v, old, _new);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Signed */
|
|
|
|
|
ATOMIC_INLINE int32_t atomic_add_and_fetch_int32(int32_t *p, int32_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_add_and_fetch(p, x);
|
|
|
|
|
return __sync_add_and_fetch(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int32_t atomic_sub_and_fetch_int32(int32_t *p, int32_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_sub_and_fetch(p, x);
|
|
|
|
|
return __sync_sub_and_fetch(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int32_t atomic_cas_int32(int32_t *v, int32_t old, int32_t _new)
|
|
|
|
|
{
|
|
|
|
|
return __sync_val_compare_and_swap(v, old, _new);
|
|
|
|
|
return __sync_val_compare_and_swap(v, old, _new);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
|
|
|
|
|
/* Unsigned */
|
|
|
|
|
ATOMIC_INLINE uint32_t atomic_add_and_fetch_uint32(uint32_t *p, uint32_t x)
|
|
|
|
|
{
|
|
|
|
|
uint32_t ret = x;
|
|
|
|
|
asm volatile (
|
|
|
|
|
"lock; xaddl %0, %1;"
|
|
|
|
|
: "+r" (ret), "=m" (*p) /* Outputs. */
|
|
|
|
|
: "m" (*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return ret + x;
|
|
|
|
|
uint32_t ret = x;
|
|
|
|
|
asm volatile("lock; xaddl %0, %1;"
|
|
|
|
|
: "+r"(ret), "=m"(*p) /* Outputs. */
|
|
|
|
|
: "m"(*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return ret + x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint32_t atomic_sub_and_fetch_uint32(uint32_t *p, uint32_t x)
|
|
|
|
|
{
|
|
|
|
|
uint32_t ret = (uint32_t)(-(int32_t)x);
|
|
|
|
|
asm volatile (
|
|
|
|
|
"lock; xaddl %0, %1;"
|
|
|
|
|
: "+r" (ret), "=m" (*p) /* Outputs. */
|
|
|
|
|
: "m" (*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return ret - x;
|
|
|
|
|
uint32_t ret = (uint32_t)(-(int32_t)x);
|
|
|
|
|
asm volatile("lock; xaddl %0, %1;"
|
|
|
|
|
: "+r"(ret), "=m"(*p) /* Outputs. */
|
|
|
|
|
: "m"(*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return ret - x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint32_t atomic_cas_uint32(uint32_t *v, uint32_t old, uint32_t _new)
|
|
|
|
|
{
|
|
|
|
|
uint32_t ret;
|
|
|
|
|
asm volatile (
|
|
|
|
|
"lock; cmpxchgl %2,%1"
|
|
|
|
|
: "=a" (ret), "+m" (*v)
|
|
|
|
|
: "r" (_new), "0" (old)
|
|
|
|
|
: "memory");
|
|
|
|
|
return ret;
|
|
|
|
|
uint32_t ret;
|
|
|
|
|
asm volatile("lock; cmpxchgl %2,%1" : "=a"(ret), "+m"(*v) : "r"(_new), "0"(old) : "memory");
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Signed */
|
|
|
|
|
ATOMIC_INLINE int32_t atomic_add_and_fetch_int32(int32_t *p, int32_t x)
|
|
|
|
|
{
|
|
|
|
|
int32_t ret = x;
|
|
|
|
|
asm volatile (
|
|
|
|
|
"lock; xaddl %0, %1;"
|
|
|
|
|
: "+r" (ret), "=m" (*p) /* Outputs. */
|
|
|
|
|
: "m" (*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return ret + x;
|
|
|
|
|
int32_t ret = x;
|
|
|
|
|
asm volatile("lock; xaddl %0, %1;"
|
|
|
|
|
: "+r"(ret), "=m"(*p) /* Outputs. */
|
|
|
|
|
: "m"(*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return ret + x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int32_t atomic_sub_and_fetch_int32(int32_t *p, int32_t x)
|
|
|
|
|
{
|
|
|
|
|
int32_t ret = -x;
|
|
|
|
|
asm volatile (
|
|
|
|
|
"lock; xaddl %0, %1;"
|
|
|
|
|
: "+r" (ret), "=m" (*p) /* Outputs. */
|
|
|
|
|
: "m" (*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return ret - x;
|
|
|
|
|
int32_t ret = -x;
|
|
|
|
|
asm volatile("lock; xaddl %0, %1;"
|
|
|
|
|
: "+r"(ret), "=m"(*p) /* Outputs. */
|
|
|
|
|
: "m"(*p) /* Inputs. */
|
|
|
|
|
);
|
|
|
|
|
return ret - x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int32_t atomic_cas_int32(int32_t *v, int32_t old, int32_t _new)
|
|
|
|
|
{
|
|
|
|
|
int32_t ret;
|
|
|
|
|
asm volatile (
|
|
|
|
|
"lock; cmpxchgl %2,%1"
|
|
|
|
|
: "=a" (ret), "+m" (*v)
|
|
|
|
|
: "r" (_new), "0" (old)
|
|
|
|
|
: "memory");
|
|
|
|
|
return ret;
|
|
|
|
|
int32_t ret;
|
|
|
|
|
asm volatile("lock; cmpxchgl %2,%1" : "=a"(ret), "+m"(*v) : "r"(_new), "0"(old) : "memory");
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#else
|
|
|
|
@ -308,33 +284,33 @@ ATOMIC_INLINE int32_t atomic_cas_int32(int32_t *v, int32_t old, int32_t _new)
|
|
|
|
|
/* Unsigned */
|
|
|
|
|
ATOMIC_INLINE uint32_t atomic_fetch_and_add_uint32(uint32_t *p, uint32_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_add(p, x);
|
|
|
|
|
return __sync_fetch_and_add(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint32_t atomic_fetch_and_or_uint32(uint32_t *p, uint32_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_or(p, x);
|
|
|
|
|
return __sync_fetch_and_or(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE uint32_t atomic_fetch_and_and_uint32(uint32_t *p, uint32_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_and(p, x);
|
|
|
|
|
return __sync_fetch_and_and(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Signed */
|
|
|
|
|
ATOMIC_INLINE int32_t atomic_fetch_and_add_int32(int32_t *p, int32_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_add(p, x);
|
|
|
|
|
return __sync_fetch_and_add(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int32_t atomic_fetch_and_or_int32(int32_t *p, int32_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_or(p, x);
|
|
|
|
|
return __sync_fetch_and_or(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ATOMIC_INLINE int32_t atomic_fetch_and_and_int32(int32_t *p, int32_t x)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_and(p, x);
|
|
|
|
|
return __sync_fetch_and_and(p, x);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#else
|
|
|
|
@ -347,21 +323,21 @@ ATOMIC_INLINE int32_t atomic_fetch_and_and_int32(int32_t *p, int32_t x)
|
|
|
|
|
/* Unsigned */
|
|
|
|
|
ATOMIC_INLINE uint8_t atomic_fetch_and_and_uint8(uint8_t *p, uint8_t b)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_and(p, b);
|
|
|
|
|
return __sync_fetch_and_and(p, b);
|
|
|
|
|
}
|
|
|
|
|
ATOMIC_INLINE uint8_t atomic_fetch_and_or_uint8(uint8_t *p, uint8_t b)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_or(p, b);
|
|
|
|
|
return __sync_fetch_and_or(p, b);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Signed */
|
|
|
|
|
ATOMIC_INLINE int8_t atomic_fetch_and_and_int8(int8_t *p, int8_t b)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_and(p, b);
|
|
|
|
|
return __sync_fetch_and_and(p, b);
|
|
|
|
|
}
|
|
|
|
|
ATOMIC_INLINE int8_t atomic_fetch_and_or_int8(int8_t *p, int8_t b)
|
|
|
|
|
{
|
|
|
|
|
return __sync_fetch_and_or(p, b);
|
|
|
|
|
return __sync_fetch_and_or(p, b);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#else
|
|
|
|
|