From ae5712d110b720768dcc30522171506a4b7f28a4 Mon Sep 17 00:00:00 2001 From: Olivier Dion Date: Tue, 22 Aug 2023 16:23:17 -0400 Subject: [PATCH 1/3] uatomic/x86: Remove redundant memory barriers When liburcu is configured to _not_ use atomic builtins, the implementation of atomic operations is done using inline assembler for each architecture. Because we control the emitted assembler, we know whether specific operations (e.g. lock; cmpxchg) already have an implicit memory barrier. In those cases, emitting an explicit cmm_smp_mb() before/after the operation is redundant and hurts performance. Remove those redundant barriers on x86. Change-Id: Ic1f6cfe9c2afe250946549cf6187f8fa88f5b009 Signed-off-by: Olivier Dion Signed-off-by: Mathieu Desnoyers --- include/urcu/uatomic/generic.h | 228 ++++------------ include/urcu/uatomic/x86.h | 470 +++++++++++++++++++++++++++++++++ 2 files changed, 516 insertions(+), 182 deletions(-) diff --git a/include/urcu/uatomic/generic.h b/include/urcu/uatomic/generic.h index 8f8c437..ed655bb 100644 --- a/include/urcu/uatomic/generic.h +++ b/include/urcu/uatomic/generic.h @@ -15,7 +15,6 @@ */ #include -#include #include #include @@ -27,125 +26,61 @@ extern "C" { #define uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v))) #endif -#define uatomic_load_store_return_op(op, addr, v, mo) \ - __extension__ \ - ({ \ - \ - switch (mo) { \ - case CMM_ACQUIRE: \ - case CMM_CONSUME: \ - case CMM_RELAXED: \ - break; \ - case CMM_RELEASE: \ - case CMM_ACQ_REL: \ - case CMM_SEQ_CST: \ - case CMM_SEQ_CST_FENCE: \ - cmm_smp_mb(); \ - break; \ - default: \ - abort(); \ - } \ - \ - __typeof__((*addr)) _value = op(addr, v); \ - \ - switch (mo) { \ - case CMM_CONSUME: \ - cmm_smp_read_barrier_depends(); \ - break; \ - case CMM_ACQUIRE: \ - case CMM_ACQ_REL: \ - case CMM_SEQ_CST: \ - case CMM_SEQ_CST_FENCE: \ - cmm_smp_mb(); \ - break; \ - case CMM_RELAXED: \ - case CMM_RELEASE: \ - break; \ - default: \ - abort(); \ - } \ - _value; \ +/* + * Can be defined for the architecture. + * + * What needs to be emitted _before_ the `operation' with memory ordering `mo'. + */ +#ifndef _cmm_compat_c11_smp_mb__before_mo +# define _cmm_compat_c11_smp_mb__before_mo(operation, mo) cmm_smp_mb() +#endif + +/* + * Can be defined for the architecture. + * + * What needs to be emitted _after_ the `operation' with memory ordering `mo'. + */ +#ifndef _cmm_compat_c11_smp_mb__after_mo +# define _cmm_compat_c11_smp_mb__after_mo(operation, mo) cmm_smp_mb() +#endif + +#define uatomic_load_store_return_op(op, addr, v, mo) \ + __extension__ \ + ({ \ + _cmm_compat_c11_smp_mb__before_mo(op, mo); \ + __typeof__((*addr)) _value = op(addr, v); \ + _cmm_compat_c11_smp_mb__after_mo(op, mo); \ + \ + _value; \ }) -#define uatomic_load_store_op(op, addr, v, mo) \ - do { \ - switch (mo) { \ - case CMM_ACQUIRE: \ - case CMM_CONSUME: \ - case CMM_RELAXED: \ - break; \ - case CMM_RELEASE: \ - case CMM_ACQ_REL: \ - case CMM_SEQ_CST: \ - case CMM_SEQ_CST_FENCE: \ - cmm_smp_mb(); \ - break; \ - default: \ - abort(); \ - } \ - \ - op(addr, v); \ - \ - switch (mo) { \ - case CMM_CONSUME: \ - cmm_smp_read_barrier_depends(); \ - break; \ - case CMM_ACQUIRE: \ - case CMM_ACQ_REL: \ - case CMM_SEQ_CST: \ - case CMM_SEQ_CST_FENCE: \ - cmm_smp_mb(); \ - break; \ - case CMM_RELAXED: \ - case CMM_RELEASE: \ - break; \ - default: \ - abort(); \ - } \ +#define uatomic_load_store_op(op, addr, v, mo) \ + do { \ + _cmm_compat_c11_smp_mb__before_mo(op, mo); \ + op(addr, v); \ + _cmm_compat_c11_smp_mb__after_mo(op, mo); \ } while (0) -#define uatomic_store(addr, v, mo) \ - do { \ - switch (mo) { \ - case CMM_RELAXED: \ - break; \ - case CMM_RELEASE: \ - case CMM_SEQ_CST: \ - case CMM_SEQ_CST_FENCE: \ - cmm_smp_mb(); \ - break; \ - default: \ - abort(); \ - } \ - \ - uatomic_set(addr, v); \ - \ - switch (mo) { \ - case CMM_RELAXED: \ - case CMM_RELEASE: \ - break; \ - case CMM_SEQ_CST: \ - case CMM_SEQ_CST_FENCE: \ - cmm_smp_mb(); \ - break; \ - default: \ - abort(); \ - } \ +#define uatomic_store(addr, v, mo) \ + do { \ + _cmm_compat_c11_smp_mb__before_mo(uatomic_set, mo); \ + uatomic_set(addr, v); \ + _cmm_compat_c11_smp_mb__after_mo(uatomic_set, mo); \ } while (0) -#define uatomic_and_mo(addr, v, mo) \ +#define uatomic_and_mo(addr, v, mo) \ uatomic_load_store_op(uatomic_and, addr, v, mo) -#define uatomic_or_mo(addr, v, mo) \ +#define uatomic_or_mo(addr, v, mo) \ uatomic_load_store_op(uatomic_or, addr, v, mo) -#define uatomic_add_mo(addr, v, mo) \ +#define uatomic_add_mo(addr, v, mo) \ uatomic_load_store_op(uatomic_add, addr, v, mo) -#define uatomic_sub_mo(addr, v, mo) \ +#define uatomic_sub_mo(addr, v, mo) \ uatomic_load_store_op(uatomic_sub, addr, v, mo) -#define uatomic_inc_mo(addr, mo) \ +#define uatomic_inc_mo(addr, mo) \ uatomic_load_store_op(uatomic_add, addr, 1, mo) #define uatomic_dec_mo(addr, mo) \ @@ -157,58 +92,14 @@ extern "C" { #define uatomic_cmpxchg_mo(addr, old, new, mos, mof) \ __extension__ \ ({ \ - switch (mos) { \ - case CMM_ACQUIRE: \ - case CMM_CONSUME: \ - case CMM_RELAXED: \ - break; \ - case CMM_RELEASE: \ - case CMM_ACQ_REL: \ - case CMM_SEQ_CST: \ - case CMM_SEQ_CST_FENCE: \ - cmm_smp_mb(); \ - break; \ - default: \ - abort(); \ - } \ - \ + _cmm_compat_c11_smp_mb__before_mo(uatomic_cmpxchg, mos); \ __typeof__(*(addr)) _value = uatomic_cmpxchg(addr, old, \ new); \ \ if (_value == (old)) { \ - switch (mos) { \ - case CMM_CONSUME: \ - cmm_smp_read_barrier_depends(); \ - break; \ - case CMM_ACQUIRE: \ - case CMM_ACQ_REL: \ - case CMM_SEQ_CST: \ - case CMM_SEQ_CST_FENCE: \ - cmm_smp_mb(); \ - break; \ - case CMM_RELAXED: \ - case CMM_RELEASE: \ - break; \ - default: \ - abort(); \ - } \ + _cmm_compat_c11_smp_mb__after_mo(uatomic_cmpxchg, mos); \ } else { \ - switch (mof) { \ - case CMM_CONSUME: \ - cmm_smp_read_barrier_depends(); \ - break; \ - case CMM_ACQUIRE: \ - case CMM_ACQ_REL: \ - case CMM_SEQ_CST: \ - case CMM_SEQ_CST_FENCE: \ - cmm_smp_mb(); \ - break; \ - case CMM_RELAXED: \ - case CMM_RELEASE: \ - break; \ - default: \ - abort(); \ - } \ + _cmm_compat_c11_smp_mb__after_mo(uatomic_cmpxchg, mof); \ } \ _value; \ }) @@ -222,7 +113,6 @@ extern "C" { #define uatomic_sub_return_mo(addr, v, mo) \ uatomic_load_store_return_op(uatomic_sub_return, addr, v) - #ifndef uatomic_read #define uatomic_read(addr) CMM_LOAD_SHARED(*(addr)) #endif @@ -230,35 +120,9 @@ extern "C" { #define uatomic_load(addr, mo) \ __extension__ \ ({ \ - switch (mo) { \ - case CMM_ACQUIRE: \ - case CMM_CONSUME: \ - case CMM_RELAXED: \ - break; \ - case CMM_SEQ_CST: \ - case CMM_SEQ_CST_FENCE: \ - cmm_smp_mb(); \ - break; \ - default: \ - abort(); \ - } \ - \ + _cmm_compat_c11_smp_mb__before_mo(uatomic_read, mo); \ __typeof__(*(addr)) _rcu_value = uatomic_read(addr); \ - \ - switch (mo) { \ - case CMM_RELAXED: \ - break; \ - case CMM_CONSUME: \ - cmm_smp_read_barrier_depends(); \ - break; \ - case CMM_ACQUIRE: \ - case CMM_SEQ_CST: \ - case CMM_SEQ_CST_FENCE: \ - cmm_smp_mb(); \ - break; \ - default: \ - abort(); \ - } \ + _cmm_compat_c11_smp_mb__after_mo(uatomic_read, mo); \ \ _rcu_value; \ }) diff --git a/include/urcu/uatomic/x86.h b/include/urcu/uatomic/x86.h index b5725e0..616eee9 100644 --- a/include/urcu/uatomic/x86.h +++ b/include/urcu/uatomic/x86.h @@ -8,6 +8,8 @@ #ifndef _URCU_ARCH_UATOMIC_X86_H #define _URCU_ARCH_UATOMIC_X86_H +#include /* For abort(3). */ + /* * Code inspired from libuatomic_ops-1.2, inherited in part from the * Boehm-Demers-Weiser conservative garbage collector. @@ -630,6 +632,474 @@ extern unsigned long _compat_uatomic_add_return(void *addr, #define cmm_smp_mb__before_uatomic_dec() cmm_barrier() #define cmm_smp_mb__after_uatomic_dec() cmm_barrier() +static inline void _cmm_compat_c11_smp_mb__before_uatomic_read_mo(enum cmm_memorder mo) +{ + /* + * A SMP barrier is not necessary for CMM_SEQ_CST because, only a + * previous store can be reordered with the load. However, emitting the + * memory barrier after the store is sufficient to prevent reordering + * between the two. This follows toolchains decision of emitting the + * memory fence on the stores instead of the loads. + * + * A compiler barrier is necessary because the underlying operation does + * not clobber the registers. + */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + cmm_barrier(); + break; + case CMM_ACQ_REL: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + default: + abort(); + break; + } +} + +static inline void _cmm_compat_c11_smp_mb__after_uatomic_read_mo(enum cmm_memorder mo) +{ + /* + * A SMP barrier is not necessary for CMM_SEQ_CST because following + * loads and stores cannot be reordered with the load. + * + * A SMP barrier is however necessary for CMM_SEQ_CST_FENCE to respect + * the memory model, since the underlying operation does not have a lock + * prefix. + * + * A compiler barrier is necessary because the underlying operation does + * not clobber the registers. + */ + switch (mo) { + case CMM_SEQ_CST_FENCE: + cmm_smp_mb(); + break; + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_SEQ_CST: + cmm_barrier(); + break; + case CMM_ACQ_REL: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + default: + abort(); + break; + } +} + +static inline void _cmm_compat_c11_smp_mb__before_uatomic_set_mo(enum cmm_memorder mo) +{ + /* + * A SMP barrier is not necessary for CMM_SEQ_CST because the store can + * only be reodered with later loads + * + * A compiler barrier is necessary because the underlying operation does + * not clobber the registers. + */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + cmm_barrier(); + break; + case CMM_ACQ_REL: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + default: + abort(); + break; + } +} + +static inline void _cmm_compat_c11_smp_mb__after_uatomic_set_mo(enum cmm_memorder mo) +{ + /* + * A SMP barrier is necessary for CMM_SEQ_CST because the store can be + * reorded with later loads. Since no memory barrier is being emitted + * before loads, one has to be emitted after the store. This follows + * toolchains decision of emitting the memory fence on the stores instead + * of the loads. + * + * A SMP barrier is necessary for CMM_SEQ_CST_FENCE to respect the + * memory model, since the underlying store does not have a lock prefix. + * + * A compiler barrier is necessary because the underlying operation does + * not clobber the registers. + */ + switch (mo) { + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + cmm_smp_mb(); + break; + case CMM_RELAXED: /* Fall-through */ + case CMM_RELEASE: + cmm_barrier(); + break; + case CMM_ACQ_REL: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + default: + abort(); + break; + } +} + +static inline void _cmm_compat_c11_smp_mb__before_uatomic_xchg_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_xchg has implicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__after_uatomic_xchg_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_xchg has implicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__before_uatomic_cmpxchg_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_cmpxchg has implicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__after_uatomic_cmpxchg_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_cmpxchg has implicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__before_uatomic_and_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_and has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__after_uatomic_and_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_and has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__before_uatomic_or_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_or has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__after_uatomic_or_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_or has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__before_uatomic_add_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_add has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__after_uatomic_add_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_add has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__before_uatomic_sub_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_sub has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__after_uatomic_sub_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_sub has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__before_uatomic_inc_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_inc has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__after_uatomic_inc_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_inc has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__before_uatomic_dec_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_dec has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__after_uatomic_dec_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_dec has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__before_uatomic_add_return_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_add_return has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__after_uatomic_add_return_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_add_return has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__before_uatomic_sub_return_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_sub_return has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +static inline void _cmm_compat_c11_smp_mb__after_uatomic_sub_return_mo(enum cmm_memorder mo) +{ + /* NOP. uatomic_sub_return has explicit lock prefix. */ + switch (mo) { + case CMM_RELAXED: /* Fall-through */ + case CMM_ACQUIRE: /* Fall-through */ + case CMM_CONSUME: /* Fall-through */ + case CMM_RELEASE: /* Fall-through */ + case CMM_ACQ_REL: /* Fall-through */ + case CMM_SEQ_CST: /* Fall-through */ + case CMM_SEQ_CST_FENCE: + break; + default: + abort(); + } +} + +#define _cmm_compat_c11_smp_mb__before_mo(operation, mo) \ + do { \ + _cmm_compat_c11_smp_mb__before_ ## operation ## _mo (mo); \ + } while (0) + +#define _cmm_compat_c11_smp_mb__after_mo(operation, mo) \ + do { \ + _cmm_compat_c11_smp_mb__after_ ## operation ## _mo (mo); \ + } while (0) + + #ifdef __cplusplus } #endif -- 2.34.1 From 02a024dc4cbadddb8700aca7b3f6cf27f69e8f3f Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 2 May 2024 10:34:12 -0400 Subject: [PATCH 2/3] Relicense src/compat-smp.h to MIT Relicense the code in src/compat-smp.h from LGPLv2.1 to MIT to match the copies in the following projects: - lttng-ust - libside - librseq This code is entirely authored by EfficiOS. This relicensing initially appeared in the lttng-ust project after the code was imported into liburcu: commit 4159f02937a2740abd7f5b113f376b198a86bc71 (test-struct-tls) Author: Mathieu Desnoyers Date: Tue Oct 25 12:32:12 2022 -0400 Relicense common/smp.c common/smp.h to MIT Signed-off-by: Mathieu Desnoyers Change-Id: Ib7edb6c18fd3c004503b9c023ba4e280241ede14 --- src/compat-smp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compat-smp.h b/src/compat-smp.h index 31fa979..2f204d7 100644 --- a/src/compat-smp.h +++ b/src/compat-smp.h @@ -1,5 +1,5 @@ /* - * SPDX-License-Identifier: LGPL-2.1-only + * SPDX-License-Identifier: MIT * * Copyright (C) 2011-2012 Mathieu Desnoyers * Copyright (C) 2019 Michael Jeanson -- 2.34.1 From 9922f33e29869ac23296a857fd6a9f813d24ad4a Mon Sep 17 00:00:00 2001 From: Benjamin Marzinski via lttng-dev Date: Wed, 1 May 2024 19:42:41 -0400 Subject: [PATCH 3/3] fix: handle EINTR correctly in get_cpu_mask_from_sysfs If the read() in get_cpu_mask_from_sysfs() fails with EINTR, the code is supposed to retry, but the while loop condition has (bytes_read > 0), which is false when read() fails with EINTR. The result is that the code exits the loop, having only read part of the string. Use (bytes_read != 0) in the while loop condition instead, since the (bytes_read < 0) case is already handled in the loop. Signed-off-by: Benjamin Marzinski Signed-off-by: Mathieu Desnoyers Change-Id: I565030d4625ae199cabc4c2ab5eb8ac49ea4dfcb --- src/compat-smp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compat-smp.h b/src/compat-smp.h index 2f204d7..5da8d6a 100644 --- a/src/compat-smp.h +++ b/src/compat-smp.h @@ -164,7 +164,7 @@ static inline int get_cpu_mask_from_sysfs(char *buf, size_t max_bytes, const cha total_bytes_read += bytes_read; assert(total_bytes_read <= max_bytes); - } while (max_bytes > total_bytes_read && bytes_read > 0); + } while (max_bytes > total_bytes_read && bytes_read != 0); /* * Make sure the mask read is a null terminated string. -- 2.34.1