From: Mathieu Desnoyers Date: Wed, 17 Aug 2011 10:07:35 +0000 (-0400) Subject: Merge branch 'master' into lfqueue-dev X-Git-Tag: v0.6.5~51 X-Git-Url: https://git.liburcu.org/?p=urcu.git;a=commitdiff_plain;h=e7385b904e9fb77e94a85a7c0d092cc6039dd948;hp=34b1b9db1cc283c556af5f35f70ea2e4713a440d Merge branch 'master' into lfqueue-dev --- diff --git a/tests/api.h b/tests/api.h index 40147b2..c5d716f 100644 --- a/tests/api.h +++ b/tests/api.h @@ -27,55 +27,13 @@ * to redistribute under later versions of GPL might not be available. */ +#include #include -#ifndef __always_inline -#define __always_inline inline -#endif - -#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)])) -#define BUILD_BUG_ON_ZERO(e) (sizeof(char[1 - 2 * !!(e)]) - 1) - -#ifdef __ASSEMBLY__ -# define stringify_in_c(...) __VA_ARGS__ -# define ASM_CONST(x) x -#else -/* This version of stringify will deal with commas... */ -# define __stringify_in_c(...) #__VA_ARGS__ -# define stringify_in_c(...) __stringify_in_c(__VA_ARGS__) " " -# define __ASM_CONST(x) x##UL -# define ASM_CONST(x) __ASM_CONST(x) -#endif - - -/* - * arch-i386.h: Expose x86 atomic instructions. 80486 and better only. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, but version 2 only due to inclusion - * of Linux-kernel code. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * - * Copyright (c) 2006 Paul E. McKenney, IBM. - * - * Much code taken from the Linux kernel. For such code, the option - * to redistribute under later versions of GPL might not be available. - */ - /* * Machine parameters. */ -/* #define CAA_CACHE_LINE_SIZE 64 */ #define ____cacheline_internodealigned_in_smp \ __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))) @@ -112,14 +70,6 @@ #include /* #include "atomic.h" */ -/* - * Default machine parameters. - */ - -#ifndef CAA_CACHE_LINE_SIZE -/* #define CAA_CACHE_LINE_SIZE 128 */ -#endif /* #ifndef CAA_CACHE_LINE_SIZE */ - /* * Exclusive locking primitives. */ @@ -337,107 +287,8 @@ long long get_microseconds(void) per_thread(name, __i_p_t_i) = v; \ } while (0) -/* - * CPU traversal primitives. - */ - -#ifndef NR_CPUS -#define NR_CPUS 16 -#endif /* #ifndef NR_CPUS */ - -#define for_each_possible_cpu(cpu) \ - for (cpu = 0; cpu < NR_CPUS; cpu++) -#define for_each_online_cpu(cpu) \ - for (cpu = 0; cpu < NR_CPUS; cpu++) - -/* - * Per-CPU variables. - */ - -#define DEFINE_PER_CPU(type, name) \ - struct { \ - __typeof__(type) v \ - __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))); \ - } __per_cpu_##name[NR_CPUS] -#define DECLARE_PER_CPU(type, name) extern DEFINE_PER_CPU(type, name) - DEFINE_PER_THREAD(int, smp_processor_id); -#define per_cpu(name, thread) __per_cpu_##name[thread].v -#define __get_cpu_var(name) per_cpu(name, smp_processor_id()) - -#define init_per_cpu(name, v) \ - do { \ - int __i_p_c_i; \ - for (__i_p_c_i = 0; __i_p_c_i < NR_CPUS; __i_p_c_i++) \ - per_cpu(name, __i_p_c_i) = v; \ - } while (0) - -/* - * CPU state checking (crowbarred). - */ - -#define idle_cpu(cpu) 0 -#define in_softirq() 1 -#define hardirq_count() 0 -#define PREEMPT_SHIFT 0 -#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) -#define PREEMPT_BITS 8 -#define SOFTIRQ_BITS 8 - -/* - * CPU hotplug. - */ - -struct notifier_block { - int (*notifier_call)(struct notifier_block *, unsigned long, void *); - struct notifier_block *next; - int priority; -}; - -#define CPU_ONLINE 0x0002 /* CPU (unsigned)v is up */ -#define CPU_UP_PREPARE 0x0003 /* CPU (unsigned)v coming up */ -#define CPU_UP_CANCELED 0x0004 /* CPU (unsigned)v NOT coming up */ -#define CPU_DOWN_PREPARE 0x0005 /* CPU (unsigned)v going down */ -#define CPU_DOWN_FAILED 0x0006 /* CPU (unsigned)v NOT going down */ -#define CPU_DEAD 0x0007 /* CPU (unsigned)v dead */ -#define CPU_DYING 0x0008 /* CPU (unsigned)v not running any task, - * not handling interrupts, soon dead */ -#define CPU_POST_DEAD 0x0009 /* CPU (unsigned)v dead, cpu_hotplug - * lock is dropped */ - -/* Used for CPU hotplug events occuring while tasks are frozen due to a suspend - * operation in progress - */ -#define CPU_TASKS_FROZEN 0x0010 - -#define CPU_ONLINE_FROZEN (CPU_ONLINE | CPU_TASKS_FROZEN) -#define CPU_UP_PREPARE_FROZEN (CPU_UP_PREPARE | CPU_TASKS_FROZEN) -#define CPU_UP_CANCELED_FROZEN (CPU_UP_CANCELED | CPU_TASKS_FROZEN) -#define CPU_DOWN_PREPARE_FROZEN (CPU_DOWN_PREPARE | CPU_TASKS_FROZEN) -#define CPU_DOWN_FAILED_FROZEN (CPU_DOWN_FAILED | CPU_TASKS_FROZEN) -#define CPU_DEAD_FROZEN (CPU_DEAD | CPU_TASKS_FROZEN) -#define CPU_DYING_FROZEN (CPU_DYING | CPU_TASKS_FROZEN) - -/* Hibernation and suspend events */ -#define PM_HIBERNATION_PREPARE 0x0001 /* Going to hibernate */ -#define PM_POST_HIBERNATION 0x0002 /* Hibernation finished */ -#define PM_SUSPEND_PREPARE 0x0003 /* Going to suspend the system */ -#define PM_POST_SUSPEND 0x0004 /* Suspend finished */ -#define PM_RESTORE_PREPARE 0x0005 /* Going to restore a saved image */ -#define PM_POST_RESTORE 0x0006 /* Restore failed */ - -#define NOTIFY_DONE 0x0000 /* Don't care */ -#define NOTIFY_OK 0x0001 /* Suits me */ -#define NOTIFY_STOP_MASK 0x8000 /* Don't call further */ -#define NOTIFY_BAD (NOTIFY_STOP_MASK|0x0002) - /* Bad/Veto action */ -/* - * Clean way to return from the notifier and stop further calls. - */ -#define NOTIFY_STOP (NOTIFY_OK|NOTIFY_STOP_MASK) - /* * Bug checks. */ diff --git a/urcu-qsbr.c b/urcu-qsbr.c index a239be0..8d8a9cf 100644 --- a/urcu-qsbr.c +++ b/urcu-qsbr.c @@ -150,26 +150,33 @@ static void update_counter_and_wait(void) */ for (;;) { wait_loops++; - if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) { - uatomic_dec(&gp_futex); + if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { + uatomic_set(&gp_futex, -1); + /* + * Write futex before write waiting (the other side + * reads them in the opposite order). + */ + cmm_smp_wmb(); + cds_list_for_each_entry(index, ®istry, node) { + _CMM_STORE_SHARED(index->waiting, 1); + } /* Write futex before read reader_gp */ cmm_smp_mb(); } - cds_list_for_each_entry_safe(index, tmp, ®istry, node) { if (!rcu_gp_ongoing(&index->ctr)) cds_list_move(&index->node, &qsreaders); } if (cds_list_empty(®istry)) { - if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) { + if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { /* Read reader_gp before write futex */ cmm_smp_mb(); uatomic_set(&gp_futex, 0); } break; } else { - if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) { + if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { wait_gp(); } else { #ifndef HAS_INCOHERENT_CACHES diff --git a/urcu/compiler.h b/urcu/compiler.h index 54904cc..489677b 100644 --- a/urcu/compiler.h +++ b/urcu/compiler.h @@ -61,4 +61,7 @@ (type *)((char *)__ptr - offsetof(type, member)); \ }) +#define CAA_BUILD_BUG_ON_ZERO(cond) (sizeof(struct { int:-!!(cond); })) +#define CAA_BUILD_BUG_ON(cond) ((void)BUILD_BUG_ON_ZERO(cond)) + #endif /* _URCU_COMPILER_H */ diff --git a/urcu/static/urcu-qsbr.h b/urcu/static/urcu-qsbr.h index 5b7adac..489abb0 100644 --- a/urcu/static/urcu-qsbr.h +++ b/urcu/static/urcu-qsbr.h @@ -124,6 +124,7 @@ struct rcu_reader { unsigned long ctr; /* Data used for registry */ struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE))); + int waiting; pthread_t tid; }; @@ -136,7 +137,11 @@ extern int32_t gp_futex; */ static inline void wake_up_gp(void) { - if (unlikely(uatomic_read(&gp_futex) == -1)) { + if (unlikely(_CMM_LOAD_SHARED(rcu_reader.waiting))) { + _CMM_STORE_SHARED(rcu_reader.waiting, 0); + cmm_smp_mb(); + if (uatomic_read(&gp_futex) != -1) + return; uatomic_set(&gp_futex, 0); futex_noasync(&gp_futex, FUTEX_WAKE, 1, NULL, NULL, 0);