urcu.c

   1 /*
   2  * urcu.c
   3  *
   4  * Userspace RCU library
   5  *
   6  * Copyright February 2009 - Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
   7  *
   8  * Distributed under GPLv2
   9  */
  10
  11 #include <stdio.h>
  12 #include <pthread.h>
  13 #include <signal.h>
  14 #include <assert.h>
  15 #include <stdlib.h>
  16 #include <string.h>
  17
  18 #include "urcu.h"
  19
  20 pthread_mutex_t urcu_mutex = PTHREAD_MUTEX_INITIALIZER;
  21
  22 /*
  23  * Global grace period counter.
  24  * Contains the current RCU_GP_CTR_BIT.
  25  * Also has a RCU_GP_CTR_BIT of 1, to accelerate the reader fast path.
  26  */
  27 long urcu_gp_ctr = RCU_GP_COUNT;
  28
  29 long __thread urcu_active_readers;
  30
  31 /* Thread IDs of registered readers */
  32 #define INIT_NUM_THREADS 4
  33
  34 struct reader_data {
  35         pthread_t tid;
  36         long *urcu_active_readers;
  37 };
  38
  39 #ifdef DEBUG_YIELD
  40 unsigned int yield_active;
  41 unsigned int __thread rand_yield;
  42 #endif
  43
  44 static struct reader_data *reader_data;
  45 static int num_readers, alloc_readers;
  46 static int sig_done;
  47
  48 void internal_urcu_lock(void)
  49 {
  50         int ret;
  51         ret = pthread_mutex_lock(&urcu_mutex);
  52         if (ret) {
  53                 perror("Error in pthread mutex lock");
  54                 exit(-1);
  55         }
  56 }
  57
  58 void internal_urcu_unlock(void)
  59 {
  60         int ret;
  61
  62         ret = pthread_mutex_unlock(&urcu_mutex);
  63         if (ret) {
  64                 perror("Error in pthread mutex unlock");
  65                 exit(-1);
  66         }
  67 }
  68
  69 /*
  70  * called with urcu_mutex held.
  71  */
  72 static void switch_next_urcu_qparity(void)
  73 {
  74         urcu_gp_ctr ^= RCU_GP_CTR_BIT;
  75 }
  76
  77 #ifdef DEBUG_FULL_MB
  78 static void force_mb_all_threads(void)
  79 {
  80         mb();
  81 }
  82 #else
  83 static void force_mb_all_threads(void)
  84 {
  85         struct reader_data *index;
  86         /*
  87          * Ask for each threads to execute a mb() so we can consider the
  88          * compiler barriers around rcu read lock as real memory barriers.
  89          */
  90         if (!reader_data)
  91                 return;
  92         debug_yield_write();
  93         sig_done = 0;
  94         debug_yield_write();
  95         mb();   /* write sig_done before sending the signals */
  96         debug_yield_write();
  97         for (index = reader_data; index < reader_data + num_readers; index++) {
  98                 pthread_kill(index->tid, SIGURCU);
  99                 debug_yield_write();
 100         }
 101         /*
 102          * Wait for sighandler (and thus mb()) to execute on every thread.
 103          * BUSY-LOOP.
 104          */
 105         while (sig_done < num_readers)
 106                 barrier();
 107         debug_yield_write();
 108         mb();   /* read sig_done before ending the barrier */
 109         debug_yield_write();
 110 }
 111 #endif
 112
 113 void wait_for_quiescent_state(void)
 114 {
 115         struct reader_data *index;
 116
 117         if (!reader_data)
 118                 return;
 119         /* Wait for each thread urcu_active_readers count to become 0.
 120          */
 121         for (index = reader_data; index < reader_data + num_readers; index++) {
 122                 /*
 123                  * BUSY-LOOP.
 124                  */
 125                 while (rcu_old_gp_ongoing(index->urcu_active_readers))
 126                         barrier();
 127         }
 128 }
 129
 130 void synchronize_rcu(void)
 131 {
 132         /* All threads should read qparity before accessing data structure
 133          * where new ptr points to. */
 134         /* Write new ptr before changing the qparity */
 135         force_mb_all_threads();
 136         debug_yield_write();
 137
 138         internal_urcu_lock();
 139         debug_yield_write();
 140
 141         switch_next_urcu_qparity();     /* 0 -> 1 */
 142         debug_yield_write();
 143
 144         /*
 145          * Must commit qparity update to memory before waiting for parity
 146          * 0 quiescent state. Failure to do so could result in the writer
 147          * waiting forever while new readers are always accessing data (no
 148          * progress).
 149          */
 150         mb();
 151
 152         /*
 153          * Wait for previous parity to be empty of readers.
 154          */
 155         wait_for_quiescent_state();     /* Wait readers in parity 0 */
 156         debug_yield_write();
 157
 158         /*
 159          * Must finish waiting for quiescent state for parity 0 before
 160          * committing qparity update to memory. Failure to do so could result in
 161          * the writer waiting forever while new readers are always accessing
 162          * data (no progress).
 163          */
 164         mb();
 165
 166         switch_next_urcu_qparity();     /* 1 -> 0 */
 167         debug_yield_write();
 168
 169         /*
 170          * Must commit qparity update to memory before waiting for parity
 171          * 1 quiescent state. Failure to do so could result in the writer
 172          * waiting forever while new readers are always accessing data (no
 173          * progress).
 174          */
 175         mb();
 176
 177         /*
 178          * Wait for previous parity to be empty of readers.
 179          */
 180         wait_for_quiescent_state();     /* Wait readers in parity 1 */
 181         debug_yield_write();
 182
 183         internal_urcu_unlock();
 184         debug_yield_write();
 185
 186         /* All threads should finish using the data referred to by old ptr
 187          * before decrementing their urcu_active_readers count */
 188         /* Finish waiting for reader threads before letting the old ptr being
 189          * freed. */
 190         force_mb_all_threads();
 191         debug_yield_write();
 192 }
 193
 194 void urcu_add_reader(pthread_t id)
 195 {
 196         struct reader_data *oldarray;
 197
 198         if (!reader_data) {
 199                 alloc_readers = INIT_NUM_THREADS;
 200                 num_readers = 0;
 201                 reader_data =
 202                         malloc(sizeof(struct reader_data) * alloc_readers);
 203         }
 204         if (alloc_readers < num_readers + 1) {
 205                 oldarray = reader_data;
 206                 reader_data = malloc(sizeof(struct reader_data)
 207                                 * (alloc_readers << 1));
 208                 memcpy(reader_data, oldarray,
 209                         sizeof(struct reader_data) * alloc_readers);
 210                 alloc_readers <<= 1;
 211                 free(oldarray);
 212         }
 213         reader_data[num_readers].tid = id;
 214         /* reference to the TLS of _this_ reader thread. */
 215         reader_data[num_readers].urcu_active_readers = &urcu_active_readers;
 216         num_readers++;
 217 }
 218
 219 /*
 220  * Never shrink (implementation limitation).
 221  * This is O(nb threads). Eventually use a hash table.
 222  */
 223 void urcu_remove_reader(pthread_t id)
 224 {
 225         struct reader_data *index;
 226
 227         assert(reader_data != NULL);
 228         for (index = reader_data; index < reader_data + num_readers; index++) {
 229                 if (pthread_equal(index->tid, id)) {
 230                         memcpy(index, &reader_data[num_readers - 1],
 231                                 sizeof(struct reader_data));
 232                         reader_data[num_readers - 1].tid = 0;
 233                         reader_data[num_readers - 1].urcu_active_readers = NULL;
 234                         num_readers--;
 235                         return;
 236                 }
 237         }
 238         /* Hrm not found, forgot to register ? */
 239         assert(0);
 240 }
 241
 242 void urcu_register_thread(void)
 243 {
 244         internal_urcu_lock();
 245         urcu_add_reader(pthread_self());
 246         internal_urcu_unlock();
 247 }
 248
 249 void urcu_unregister_thread(void)
 250 {
 251         internal_urcu_lock();
 252         urcu_remove_reader(pthread_self());
 253         internal_urcu_unlock();
 254 }
 255
 256 #ifndef DEBUG_FULL_MB
 257 void sigurcu_handler(int signo, siginfo_t *siginfo, void *context)
 258 {
 259         mb();
 260         atomic_inc(&sig_done);
 261 }
 262
 263 void __attribute__((constructor)) urcu_init(void)
 264 {
 265         struct sigaction act;
 266         int ret;
 267
 268         act.sa_sigaction = sigurcu_handler;
 269         ret = sigaction(SIGURCU, &act, NULL);
 270         if (ret) {
 271                 perror("Error in sigaction");
 272                 exit(-1);
 273         }
 274 }
 275
 276 void __attribute__((destructor)) urcu_exit(void)
 277 {
 278         struct sigaction act;
 279         int ret;
 280
 281         ret = sigaction(SIGURCU, NULL, &act);
 282         if (ret) {
 283                 perror("Error in sigaction");
 284                 exit(-1);
 285         }
 286         assert(act.sa_sigaction == sigurcu_handler);
 287         free(reader_data);
 288 }
 289 #endif