Commit | Line | Data |
---|---|---|
287a512f | 1 | /* |
ab5be9fa MJ |
2 | * Copyright (C) 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
3 | * Copyright (C) 2017 Jérémie Galarneau <jeremie.galarneau@efficios.com> | |
287a512f | 4 | * |
ab5be9fa | 5 | * SPDX-License-Identifier: LGPL-2.1-only |
287a512f | 6 | * |
287a512f JG |
7 | */ |
8 | ||
c9e313bc | 9 | #include "error.hpp" |
f40b76ae | 10 | #include "macros.hpp" |
28ab034a JG |
11 | #include "waiter.hpp" |
12 | ||
287a512f | 13 | #include <poll.h> |
28ab034a JG |
14 | #include <urcu/futex.h> |
15 | #include <urcu/uatomic.h> | |
287a512f | 16 | |
32670d71 JG |
17 | namespace { |
18 | /* Number of busy-loop attempts before waiting on futex. */ | |
19 | constexpr auto wait_attempt_count = 1000; | |
287a512f JG |
20 | |
21 | enum waiter_state { | |
22 | /* WAITER_WAITING is compared directly (futex compares it). */ | |
28ab034a | 23 | WAITER_WAITING = 0, |
287a512f | 24 | /* non-zero are used as masks. */ |
28ab034a JG |
25 | WAITER_WOKEN_UP = (1 << 0), |
26 | WAITER_RUNNING = (1 << 1), | |
27 | WAITER_TEARDOWN = (1 << 2), | |
287a512f | 28 | }; |
32670d71 | 29 | } /* namespace */ |
287a512f | 30 | |
32670d71 | 31 | lttng::synchro::waiter::waiter() |
287a512f | 32 | { |
32670d71 JG |
33 | arm(); |
34 | } | |
35 | ||
36 | void lttng::synchro::waiter::arm() noexcept | |
37 | { | |
38 | cds_wfs_node_init(&_wait_queue_node); | |
39 | uatomic_set(&_state, WAITER_WAITING); | |
287a512f JG |
40 | cmm_smp_mb(); |
41 | } | |
42 | ||
43 | /* | |
32670d71 | 44 | * User must arm "waiter" before passing its memory to waker thread. |
287a512f | 45 | */ |
32670d71 | 46 | void lttng::synchro::waiter::wait() |
287a512f | 47 | { |
f40b76ae JG |
48 | DBG("Beginning of waiter \"wait\" period"); |
49 | ||
50 | /* Load and test condition before read state. */ | |
287a512f | 51 | cmm_smp_rmb(); |
32670d71 JG |
52 | for (unsigned int i = 0; i < wait_attempt_count; i++) { |
53 | if (uatomic_read(&_state) != WAITER_WAITING) { | |
287a512f JG |
54 | goto skip_futex_wait; |
55 | } | |
f40b76ae | 56 | |
287a512f JG |
57 | caa_cpu_relax(); |
58 | } | |
f40b76ae | 59 | |
32670d71 | 60 | while (uatomic_read(&_state) == WAITER_WAITING) { |
11927a78 | 61 | if (!futex_noasync(&_state, FUTEX_WAIT, WAITER_WAITING, nullptr, nullptr, 0)) { |
6e5438dc MD |
62 | /* |
63 | * Prior queued wakeups queued by unrelated code | |
64 | * using the same address can cause futex wait to | |
65 | * return 0 even through the futex value is still | |
66 | * WAITER_WAITING (spurious wakeups). Check | |
67 | * the value again in user-space to validate | |
68 | * whether it really differs from WAITER_WAITING. | |
69 | */ | |
70 | continue; | |
71 | } | |
f40b76ae | 72 | |
287a512f | 73 | switch (errno) { |
6e5438dc | 74 | case EAGAIN: |
287a512f JG |
75 | /* Value already changed. */ |
76 | goto skip_futex_wait; | |
77 | case EINTR: | |
78 | /* Retry if interrupted by signal. */ | |
28ab034a | 79 | break; /* Get out of switch. Check again. */ |
287a512f JG |
80 | default: |
81 | /* Unexpected error. */ | |
82 | PERROR("futex_noasync"); | |
83 | abort(); | |
84 | } | |
85 | } | |
86 | skip_futex_wait: | |
87 | ||
88 | /* Tell waker thread than we are running. */ | |
32670d71 | 89 | uatomic_or(&_state, WAITER_RUNNING); |
287a512f JG |
90 | |
91 | /* | |
92 | * Wait until waker thread lets us know it's ok to tear down | |
93 | * memory allocated for struct lttng_waiter. | |
94 | */ | |
32670d71 JG |
95 | for (unsigned int i = 0; i < wait_attempt_count; i++) { |
96 | if (uatomic_read(&_state) & WAITER_TEARDOWN) { | |
287a512f JG |
97 | break; |
98 | } | |
f40b76ae | 99 | |
287a512f JG |
100 | caa_cpu_relax(); |
101 | } | |
f40b76ae | 102 | |
32670d71 | 103 | while (!(uatomic_read(&_state) & WAITER_TEARDOWN)) { |
cd9adb8b | 104 | poll(nullptr, 0, 10); |
287a512f | 105 | } |
f40b76ae | 106 | |
32670d71 | 107 | LTTNG_ASSERT(uatomic_read(&_state) & WAITER_TEARDOWN); |
f40b76ae | 108 | DBG("End of waiter \"wait\" period"); |
287a512f JG |
109 | } |
110 | ||
32670d71 JG |
111 | lttng::synchro::waker lttng::synchro::waiter::get_waker() |
112 | { | |
113 | return lttng::synchro::waker(_state); | |
114 | } | |
115 | ||
287a512f JG |
116 | /* |
117 | * Note: lttng_waiter_wake needs waiter to stay allocated throughout its | |
118 | * execution. In this scheme, the waiter owns the node memory, and we only allow | |
119 | * it to free this memory when it sees the WAITER_TEARDOWN flag. | |
120 | */ | |
32670d71 | 121 | void lttng::synchro::waker::wake() |
287a512f JG |
122 | { |
123 | cmm_smp_mb(); | |
32670d71 | 124 | |
ac879184 | 125 | LTTNG_ASSERT(uatomic_read(&_state.get()) == WAITER_WAITING); |
32670d71 | 126 | |
ac879184 JG |
127 | uatomic_set(&_state.get(), WAITER_WOKEN_UP); |
128 | if (!(uatomic_read(&_state.get()) & WAITER_RUNNING)) { | |
129 | if (futex_noasync(&_state.get(), FUTEX_WAKE, 1, nullptr, nullptr, 0) < 0) { | |
287a512f JG |
130 | PERROR("futex_noasync"); |
131 | abort(); | |
132 | } | |
133 | } | |
f40b76ae | 134 | |
287a512f | 135 | /* Allow teardown of struct urcu_wait memory. */ |
ac879184 | 136 | uatomic_or(&_state.get(), WAITER_TEARDOWN); |
287a512f | 137 | } |
f40b76ae | 138 | |
32670d71 | 139 | lttng::synchro::wait_queue::wait_queue() |
f40b76ae | 140 | { |
32670d71 | 141 | cds_wfs_init(&_stack); |
f40b76ae JG |
142 | } |
143 | ||
11927a78 | 144 | void lttng::synchro::wait_queue::add(waiter& waiter) noexcept |
f40b76ae | 145 | { |
32670d71 | 146 | (void) cds_wfs_push(&_stack, &waiter._wait_queue_node); |
f40b76ae JG |
147 | } |
148 | ||
32670d71 | 149 | void lttng::synchro::wait_queue::wake_all() |
f40b76ae | 150 | { |
f40b76ae | 151 | /* Move all waiters from the queue to our local stack. */ |
32670d71 | 152 | auto *waiters = __cds_wfs_pop_all(&_stack); |
f40b76ae JG |
153 | |
154 | /* Wake all waiters in our stack head. */ | |
32670d71 | 155 | cds_wfs_node *iter, *iter_n; |
11927a78 | 156 | cds_wfs_for_each_blocking_safe (waiters, iter, iter_n) { |
32670d71 JG |
157 | auto& waiter = *lttng::utils::container_of( |
158 | iter, <tng::synchro::waiter::_wait_queue_node); | |
f40b76ae JG |
159 | |
160 | /* Don't wake already running threads. */ | |
32670d71 | 161 | if (waiter._state & WAITER_RUNNING) { |
f40b76ae JG |
162 | continue; |
163 | } | |
164 | ||
32670d71 | 165 | waiter.get_waker().wake(); |
f40b76ae JG |
166 | } |
167 | } |