2 * Copyright (C) 2007-2011 Mathieu Desnoyers
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation;
7 * version 2.1 of the License.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with this library; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 #include <urcu/rculist.h>
24 #include <urcu/hlist.h>
27 #include <ust/marker.h>
28 #include <ust/tracepoint.h>
29 #include <ust/tracepoint-internal.h>
31 #include "usterr_signal_safe.h"
33 #include "tracercore.h"
36 __thread
long ust_reg_stack
[500];
37 volatile __thread
long *ust_reg_stack_ptr
= (long *) 0;
39 extern struct ust_marker
* const __start___ust_marker_ptrs
[] __attribute__((visibility("hidden")));
40 extern struct ust_marker
* const __stop___ust_marker_ptrs
[] __attribute__((visibility("hidden")));
42 /* Set to 1 to enable ust_marker debug output */
43 static const int ust_marker_debug
;
46 * ust_marker_mutex nests inside module_mutex. ust_marker mutex protects
47 * the builtin and module ust_marker and the hash table.
49 static DEFINE_MUTEX(ust_marker_mutex
);
51 static CDS_LIST_HEAD(ust_marker_libs
);
54 void lock_ust_marker(void)
56 pthread_mutex_lock(&ust_marker_mutex
);
59 void unlock_ust_marker(void)
61 pthread_mutex_unlock(&ust_marker_mutex
);
65 * ust_marker hash table, containing the active ust_marker.
66 * Protected by module_mutex.
68 #define UST_MARKER_HASH_BITS 6
69 #define UST_MARKER_TABLE_SIZE (1 << UST_MARKER_HASH_BITS)
70 static struct cds_hlist_head ust_marker_table
[UST_MARKER_TABLE_SIZE
];
74 * It is used to make sure every handler has finished using its private
75 * data between two consecutive operation (add or remove) on a given
76 * ust_marker. It is also used to delay the free of multiple probes
77 * array until a quiescent state is reached. ust_marker entries
78 * modifications are protected by the ust_marker_mutex.
80 struct ust_marker_entry
{
81 struct cds_hlist_node hlist
;
85 void (*call
)(const struct ust_marker
*mdata
, void *call_private
, ...);
86 struct ust_marker_probe_closure single
;
87 struct ust_marker_probe_closure
*multi
;
88 int refcount
; /* Number of times armed. 0 if disarmed. */
94 unsigned char ptype
:1;
95 unsigned char format_allocated
:1;
96 char channel
[0]; /* Contains channel'\0'name'\0'format'\0' */
99 #ifdef CONFIG_UST_MARKER_USERSPACE
100 static void ust_marker_update_processes(void);
102 static void ust_marker_update_processes(void)
108 * __ust_marker_empty_function - Empty probe callback
109 * @mdata: ust_marker data
110 * @probe_private: probe private data
111 * @call_private: call site private data
112 * @fmt: format string
113 * @...: variable argument list
115 * Empty callback provided as a probe to the ust_marker. By providing
116 * this to a disabled ust_marker, we make sure the execution flow is
117 * always valid even though the function pointer change and the
118 * ust_marker enabling are two distinct operations that modifies the
119 * execution flow of preemptible code.
121 notrace
void __ust_marker_empty_function(const struct ust_marker
*mdata
,
122 void *probe_private
, void *call_private
, const char *fmt
, va_list *args
)
125 //ust// EXPORT_SYMBOL_GPL(__ust_marker_empty_function);
128 * ust_marker_probe_cb Callback that prepares the variable argument list for probes.
129 * @mdata: pointer of type struct ust_marker
130 * @call_private: caller site private data
131 * @...: Variable argument list.
133 * Since we do not use "typical" pointer based RCU in the 1 argument case, we
134 * need to put a full cmm_smp_rmb() in this branch. This is why we do not use
135 * rcu_dereference() for the pointer read.
137 notrace
void ust_marker_probe_cb(const struct ust_marker
*mdata
,
138 void *call_private
, ...)
144 * rcu_read_lock_sched does two things : disabling preemption to make
145 * sure the teardown of the callbacks can be done correctly when they
146 * are in modules and they insure RCU read coherency.
148 //ust// rcu_read_lock_sched_notrace();
149 ptype
= mdata
->ptype
;
150 if (likely(!ptype
)) {
151 ust_marker_probe_func
*func
;
152 /* Must read the ptype before ptr. They are not data dependant,
153 * so we put an explicit cmm_smp_rmb() here. */
155 func
= mdata
->single
.func
;
156 /* Must read the ptr before private data. They are not data
157 * dependant, so we put an explicit cmm_smp_rmb() here. */
159 va_start(args
, call_private
);
160 func(mdata
, mdata
->single
.probe_private
, call_private
,
161 mdata
->format
, &args
);
164 struct ust_marker_probe_closure
*multi
;
167 * Read mdata->ptype before mdata->multi.
170 multi
= mdata
->multi
;
172 * multi points to an array, therefore accessing the array
173 * depends on reading multi. However, even in this case,
174 * we must insure that the pointer is read _before_ the array
175 * data. Same as rcu_dereference, but we need a full cmm_smp_rmb()
176 * in the fast path, so put the explicit cmm_barrier here.
178 cmm_smp_read_barrier_depends();
179 for (i
= 0; multi
[i
].func
; i
++) {
180 va_start(args
, call_private
);
181 multi
[i
].func(mdata
, multi
[i
].probe_private
,
182 call_private
, mdata
->format
, &args
);
186 //ust// rcu_read_unlock_sched_notrace();
188 //ust// EXPORT_SYMBOL_GPL(ust_marker_probe_cb);
191 * ust_marker_probe_cb Callback that does not prepare the variable argument list.
192 * @mdata: pointer of type struct ust_marker
193 * @call_private: caller site private data
194 * @...: Variable argument list.
196 * Should be connected to ust_marker "UST_MARKER_NOARGS".
198 static notrace
void ust_marker_probe_cb_noarg(const struct ust_marker
*mdata
,
199 void *call_private
, ...)
201 va_list args
; /* not initialized */
204 //ust// rcu_read_lock_sched_notrace();
205 ptype
= mdata
->ptype
;
206 if (likely(!ptype
)) {
207 ust_marker_probe_func
*func
;
208 /* Must read the ptype before ptr. They are not data dependant,
209 * so we put an explicit cmm_smp_rmb() here. */
211 func
= mdata
->single
.func
;
212 /* Must read the ptr before private data. They are not data
213 * dependant, so we put an explicit cmm_smp_rmb() here. */
215 func(mdata
, mdata
->single
.probe_private
, call_private
,
216 mdata
->format
, &args
);
218 struct ust_marker_probe_closure
*multi
;
221 * Read mdata->ptype before mdata->multi.
224 multi
= mdata
->multi
;
226 * multi points to an array, therefore accessing the array
227 * depends on reading multi. However, even in this case,
228 * we must insure that the pointer is read _before_ the array
229 * data. Same as rcu_dereference, but we need a full cmm_smp_rmb()
230 * in the fast path, so put the explicit cmm_barrier here.
232 cmm_smp_read_barrier_depends();
233 for (i
= 0; multi
[i
].func
; i
++)
234 multi
[i
].func(mdata
, multi
[i
].probe_private
,
235 call_private
, mdata
->format
, &args
);
237 //ust// rcu_read_unlock_sched_notrace();
240 static void free_old_closure(struct rcu_head
*head
)
242 struct ust_marker_entry
*entry
= _ust_container_of(head
,
243 struct ust_marker_entry
, rcu
);
245 /* Make sure we free the data before setting the pending flag to 0 */
247 entry
->rcu_pending
= 0;
250 static void debug_print_probes(struct ust_marker_entry
*entry
)
254 if (!ust_marker_debug
)
258 DBG("Single probe : %p %p",
260 entry
->single
.probe_private
);
262 for (i
= 0; entry
->multi
[i
].func
; i
++)
263 DBG("Multi probe %d : %p %p", i
,
264 entry
->multi
[i
].func
,
265 entry
->multi
[i
].probe_private
);
269 static struct ust_marker_probe_closure
*
270 ust_marker_entry_add_probe(struct ust_marker_entry
*entry
,
271 ust_marker_probe_func
*probe
, void *probe_private
)
274 struct ust_marker_probe_closure
*old
, *new;
278 debug_print_probes(entry
);
281 if (entry
->single
.func
== probe
&&
282 entry
->single
.probe_private
== probe_private
)
283 return ERR_PTR(-EBUSY
);
284 if (entry
->single
.func
== __ust_marker_empty_function
) {
286 entry
->single
.func
= probe
;
287 entry
->single
.probe_private
= probe_private
;
290 debug_print_probes(entry
);
298 /* (N -> N+1), (N != 0, 1) probes */
299 for (nr_probes
= 0; old
[nr_probes
].func
; nr_probes
++)
300 if (old
[nr_probes
].func
== probe
301 && old
[nr_probes
].probe_private
303 return ERR_PTR(-EBUSY
);
305 /* + 2 : one for new probe, one for NULL func */
306 new = zmalloc((nr_probes
+ 2) * sizeof(struct ust_marker_probe_closure
));
308 return ERR_PTR(-ENOMEM
);
310 new[0] = entry
->single
;
313 nr_probes
* sizeof(struct ust_marker_probe_closure
));
314 new[nr_probes
].func
= probe
;
315 new[nr_probes
].probe_private
= probe_private
;
316 entry
->refcount
= nr_probes
+ 1;
319 debug_print_probes(entry
);
323 static struct ust_marker_probe_closure
*
324 ust_marker_entry_remove_probe(struct ust_marker_entry
*entry
,
325 ust_marker_probe_func
*probe
, void *probe_private
)
327 int nr_probes
= 0, nr_del
= 0, i
;
328 struct ust_marker_probe_closure
*old
, *new;
332 debug_print_probes(entry
);
334 /* 0 -> N is an error */
335 WARN_ON(entry
->single
.func
== __ust_marker_empty_function
);
337 WARN_ON(probe
&& entry
->single
.func
!= probe
);
338 WARN_ON(entry
->single
.probe_private
!= probe_private
);
339 entry
->single
.func
= __ust_marker_empty_function
;
342 debug_print_probes(entry
);
345 /* (N -> M), (N > 1, M >= 0) probes */
346 for (nr_probes
= 0; old
[nr_probes
].func
; nr_probes
++) {
347 if ((!probe
|| old
[nr_probes
].func
== probe
)
348 && old
[nr_probes
].probe_private
354 if (nr_probes
- nr_del
== 0) {
355 /* N -> 0, (N > 1) */
356 entry
->single
.func
= __ust_marker_empty_function
;
359 } else if (nr_probes
- nr_del
== 1) {
360 /* N -> 1, (N > 1) */
361 for (i
= 0; old
[i
].func
; i
++)
362 if ((probe
&& old
[i
].func
!= probe
) ||
363 old
[i
].probe_private
!= probe_private
)
364 entry
->single
= old
[i
];
369 /* N -> M, (N > 1, M > 1) */
371 new = zmalloc((nr_probes
- nr_del
+ 1) * sizeof(struct ust_marker_probe_closure
));
373 return ERR_PTR(-ENOMEM
);
374 for (i
= 0; old
[i
].func
; i
++)
375 if ((probe
&& old
[i
].func
!= probe
) ||
376 old
[i
].probe_private
!= probe_private
)
378 entry
->refcount
= nr_probes
- nr_del
;
382 debug_print_probes(entry
);
387 * Get ust_marker if the ust_marker is present in the ust_marker hash table.
388 * Must be called with ust_marker_mutex held.
389 * Returns NULL if not present.
391 static struct ust_marker_entry
*get_ust_marker(const char *channel
, const char *name
)
393 struct cds_hlist_head
*head
;
394 struct cds_hlist_node
*node
;
395 struct ust_marker_entry
*e
;
396 size_t channel_len
= strlen(channel
) + 1;
397 size_t name_len
= strlen(name
) + 1;
400 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
401 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
402 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
403 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
))
410 * Add the ust_marker to the ust_marker hash table. Must be called with
411 * ust_marker_mutex held.
413 static struct ust_marker_entry
*add_ust_marker(const char *channel
, const char *name
,
416 struct cds_hlist_head
*head
;
417 struct cds_hlist_node
*node
;
418 struct ust_marker_entry
*e
;
419 size_t channel_len
= strlen(channel
) + 1;
420 size_t name_len
= strlen(name
) + 1;
421 size_t format_len
= 0;
424 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
426 format_len
= strlen(format
) + 1;
427 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
428 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
429 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
430 DBG("ust_marker %s.%s busy", channel
, name
);
431 return ERR_PTR(-EBUSY
); /* Already there */
435 * Using zmalloc here to allocate a variable length element. Could
436 * cause some memory fragmentation if overused.
438 e
= zmalloc(sizeof(struct ust_marker_entry
)
439 + channel_len
+ name_len
+ format_len
);
441 return ERR_PTR(-ENOMEM
);
442 memcpy(e
->channel
, channel
, channel_len
);
443 e
->name
= &e
->channel
[channel_len
];
444 memcpy(e
->name
, name
, name_len
);
446 e
->format
= &e
->name
[name_len
];
447 memcpy(e
->format
, format
, format_len
);
448 if (strcmp(e
->format
, UST_MARKER_NOARGS
) == 0)
449 e
->call
= ust_marker_probe_cb_noarg
;
451 e
->call
= ust_marker_probe_cb
;
452 __ust_marker(metadata
, core_marker_format
, NULL
,
453 "channel %s name %s format %s",
454 e
->channel
, e
->name
, e
->format
);
457 e
->call
= ust_marker_probe_cb
;
459 e
->single
.func
= __ust_marker_empty_function
;
460 e
->single
.probe_private
= NULL
;
463 e
->format_allocated
= 0;
466 cds_hlist_add_head(&e
->hlist
, head
);
471 * Remove the ust_marker from the ust_marker hash table. Must be called with mutex_lock
474 static int remove_ust_marker(const char *channel
, const char *name
)
476 struct cds_hlist_head
*head
;
477 struct cds_hlist_node
*node
;
478 struct ust_marker_entry
*e
;
480 size_t channel_len
= strlen(channel
) + 1;
481 size_t name_len
= strlen(name
) + 1;
485 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
486 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
487 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
488 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
495 if (e
->single
.func
!= __ust_marker_empty_function
)
497 cds_hlist_del(&e
->hlist
);
498 if (e
->format_allocated
)
500 ret
= ltt_channels_unregister(e
->channel
);
502 /* Make sure the call_rcu has been executed */
503 //ust// if (e->rcu_pending)
504 //ust// rcu_cmm_barrier_sched();
510 * Set the mark_entry format to the format found in the element.
512 static int ust_marker_set_format(struct ust_marker_entry
*entry
, const char *format
)
514 entry
->format
= strdup(format
);
517 entry
->format_allocated
= 1;
519 __ust_marker(metadata
, core_marker_format
, NULL
,
520 "channel %s name %s format %s",
521 entry
->channel
, entry
->name
, entry
->format
);
526 * Sets the probe callback corresponding to one ust_marker.
528 static int set_ust_marker(struct ust_marker_entry
*entry
, struct ust_marker
*elem
,
532 WARN_ON(strcmp(entry
->name
, elem
->name
) != 0);
535 if (strcmp(entry
->format
, elem
->format
) != 0) {
536 ERR("Format mismatch for probe %s (%s), ust_marker (%s)",
543 ret
= ust_marker_set_format(entry
, elem
->format
);
549 * probe_cb setup (statically known) is done here. It is
550 * asynchronous with the rest of execution, therefore we only
551 * pass from a "safe" callback (with argument) to an "unsafe"
552 * callback (does not set arguments).
554 elem
->call
= entry
->call
;
555 elem
->channel_id
= entry
->channel_id
;
556 elem
->event_id
= entry
->event_id
;
559 * We only update the single probe private data when the ptr is
560 * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1)
562 WARN_ON(elem
->single
.func
!= __ust_marker_empty_function
563 && elem
->single
.probe_private
!= entry
->single
.probe_private
565 elem
->single
.probe_private
= entry
->single
.probe_private
;
567 * Make sure the private data is valid when we update the
571 elem
->single
.func
= entry
->single
.func
;
573 * We also make sure that the new probe callbacks array is consistent
574 * before setting a pointer to it.
576 rcu_assign_pointer(elem
->multi
, entry
->multi
);
578 * Update the function or multi probe array pointer before setting the
582 elem
->ptype
= entry
->ptype
;
584 if (elem
->tp_name
&& (active
^ elem
->state
)) {
585 WARN_ON(!elem
->tp_cb
);
587 * It is ok to directly call the probe registration because type
588 * checking has been done in the __ust_marker_tp() macro.
593 * try_module_get should always succeed because we hold
594 * ust_marker_mutex to get the tp_cb address.
596 //ust// ret = try_module_get(__module_text_address(
597 //ust// (unsigned long)elem->tp_cb));
598 //ust// BUG_ON(!ret);
599 ret
= tracepoint_probe_register_noupdate(
603 ret
= tracepoint_probe_unregister_noupdate(
607 * tracepoint_probe_update_all() must be called
608 * before the module containing tp_cb is unloaded.
610 //ust// module_put(__module_text_address(
611 //ust// (unsigned long)elem->tp_cb));
614 elem
->state
= active
;
620 * Disable a ust_marker and its probe callback.
621 * Note: only waiting an RCU period after setting elem->call to the empty
622 * function insures that the original callback is not used anymore. This insured
623 * by rcu_read_lock_sched around the call site.
625 static void disable_ust_marker(struct ust_marker
*elem
)
629 /* leave "call" as is. It is known statically. */
630 if (elem
->tp_name
&& elem
->state
) {
631 WARN_ON(!elem
->tp_cb
);
633 * It is ok to directly call the probe registration because type
634 * checking has been done in the __ust_marker_tp() macro.
636 ret
= tracepoint_probe_unregister_noupdate(elem
->tp_name
,
640 * tracepoint_probe_update_all() must be called
641 * before the module containing tp_cb is unloaded.
643 //ust// module_put(__module_text_address((unsigned long)elem->tp_cb));
646 elem
->single
.func
= __ust_marker_empty_function
;
647 /* Update the function before setting the ptype */
649 elem
->ptype
= 0; /* single probe */
651 * Leave the private data and channel_id/event_id there, because removal
652 * is racy and should be done only after an RCU period. These are never
653 * used until the next initialization anyway.
658 * is_ust_marker_enabled - Check if a ust_marker is enabled
659 * @channel: channel name
660 * @name: ust_marker name
662 * Returns 1 if the ust_marker is enabled, 0 if disabled.
664 int is_ust_marker_enabled(const char *channel
, const char *name
)
666 struct ust_marker_entry
*entry
;
668 pthread_mutex_lock(&ust_marker_mutex
);
669 entry
= get_ust_marker(channel
, name
);
670 pthread_mutex_unlock(&ust_marker_mutex
);
672 return entry
&& !!entry
->refcount
;
676 * ust_marker_update_probe_range - Update a probe range
677 * @begin: beginning of the range
678 * @end: end of the range
680 * Updates the probe callback corresponding to a range of ust_marker.
682 void ust_marker_update_probe_range(struct ust_marker
* const *begin
,
683 struct ust_marker
* const *end
)
685 struct ust_marker
* const *iter
;
686 struct ust_marker_entry
*mark_entry
;
688 pthread_mutex_lock(&ust_marker_mutex
);
689 for (iter
= begin
; iter
< end
; iter
++) {
691 continue; /* skip dummy */
692 mark_entry
= get_ust_marker((*iter
)->channel
, (*iter
)->name
);
694 set_ust_marker(mark_entry
, *iter
, !!mark_entry
->refcount
);
696 * ignore error, continue
699 disable_ust_marker(*iter
);
702 pthread_mutex_unlock(&ust_marker_mutex
);
705 static void lib_update_ust_marker(void)
707 struct ust_marker_lib
*lib
;
709 /* FIXME: we should probably take a mutex here on libs */
710 //ust// pthread_mutex_lock(&module_mutex);
711 cds_list_for_each_entry(lib
, &ust_marker_libs
, list
)
712 ust_marker_update_probe_range(lib
->ust_marker_start
,
713 lib
->ust_marker_start
+ lib
->ust_marker_count
);
714 //ust// pthread_mutex_unlock(&module_mutex);
718 * Update probes, removing the faulty probes.
720 * Internal callback only changed before the first probe is connected to it.
721 * Single probe private data can only be changed on 0 -> 1 and 2 -> 1
722 * transitions. All other transitions will leave the old private data valid.
723 * This makes the non-atomicity of the callback/private data updates valid.
725 * "special case" updates :
730 * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates.
731 * Site effect : ust_marker_set_format may delete the ust_marker entry (creating a
734 static void ust_marker_update_probes(void)
736 lib_update_ust_marker();
737 tracepoint_probe_update_all();
738 ust_marker_update_processes();
742 * ust_marker_probe_register - Connect a probe to a ust_marker
743 * @channel: ust_marker channel
744 * @name: ust_marker name
745 * @format: format string
746 * @probe: probe handler
747 * @probe_private: probe private data
749 * private data must be a valid allocated memory address, or NULL.
750 * Returns 0 if ok, error value on error.
751 * The probe address must at least be aligned on the architecture pointer size.
753 int ust_marker_probe_register(const char *channel
, const char *name
,
754 const char *format
, ust_marker_probe_func
*probe
,
757 struct ust_marker_entry
*entry
;
758 int ret
= 0, ret_err
;
759 struct ust_marker_probe_closure
*old
;
762 pthread_mutex_lock(&ust_marker_mutex
);
763 entry
= get_ust_marker(channel
, name
);
766 entry
= add_ust_marker(channel
, name
, format
);
768 ret
= PTR_ERR(entry
);
771 ret
= ltt_channels_register(channel
);
773 goto error_remove_ust_marker
;
774 ret
= ltt_channels_get_index_from_name(channel
);
776 goto error_unregister_channel
;
777 entry
->channel_id
= ret
;
778 ret
= ltt_channels_get_event_id(channel
, name
);
780 goto error_unregister_channel
;
781 entry
->event_id
= ret
;
783 __ust_marker(metadata
, core_marker_id
, NULL
,
784 "channel %s name %s event_id %hu "
785 "int #1u%zu long #1u%zu pointer #1u%zu "
786 "size_t #1u%zu alignment #1u%u",
787 channel
, name
, entry
->event_id
,
788 sizeof(int), sizeof(long), sizeof(void *),
789 sizeof(size_t), ltt_get_alignment());
792 ret
= ust_marker_set_format(entry
, format
);
793 else if (strcmp(entry
->format
, format
))
800 * If we detect that a call_rcu is pending for this ust_marker,
801 * make sure it's executed now.
803 //ust// if (entry->rcu_pending)
804 //ust// rcu_cmm_barrier_sched();
805 old
= ust_marker_entry_add_probe(entry
, probe
, probe_private
);
809 goto error_unregister_channel
;
813 pthread_mutex_unlock(&ust_marker_mutex
);
815 /* Activate ust_marker if necessary */
816 ust_marker_update_probes();
818 pthread_mutex_lock(&ust_marker_mutex
);
819 entry
= get_ust_marker(channel
, name
);
822 //ust// if (entry->rcu_pending)
823 //ust// rcu_cmm_barrier_sched();
825 entry
->rcu_pending
= 1;
826 /* write rcu_pending before calling the RCU callback */
828 //ust// call_rcu_sched(&entry->rcu, free_old_closure);
829 synchronize_rcu(); free_old_closure(&entry
->rcu
);
832 error_unregister_channel
:
833 ret_err
= ltt_channels_unregister(channel
);
835 error_remove_ust_marker
:
836 ret_err
= remove_ust_marker(channel
, name
);
839 pthread_mutex_unlock(&ust_marker_mutex
);
842 //ust// EXPORT_SYMBOL_GPL(ust_marker_probe_register);
845 * ust_marker_probe_unregister - Disconnect a probe from a ust_marker
846 * @channel: ust_marker channel
847 * @name: ust_marker name
848 * @probe: probe function pointer
849 * @probe_private: probe private data
851 * Returns the private data given to ust_marker_probe_register, or an ERR_PTR().
852 * We do not need to call a synchronize_sched to make sure the probes have
853 * finished running before doing a module unload, because the module unload
854 * itself uses stop_machine(), which insures that every preempt disabled section
857 int ust_marker_probe_unregister(const char *channel
, const char *name
,
858 ust_marker_probe_func
*probe
, void *probe_private
)
860 struct ust_marker_entry
*entry
;
861 struct ust_marker_probe_closure
*old
;
864 pthread_mutex_lock(&ust_marker_mutex
);
865 entry
= get_ust_marker(channel
, name
);
868 //ust// if (entry->rcu_pending)
869 //ust// rcu_cmm_barrier_sched();
870 old
= ust_marker_entry_remove_probe(entry
, probe
, probe_private
);
871 pthread_mutex_unlock(&ust_marker_mutex
);
873 ust_marker_update_probes();
875 pthread_mutex_lock(&ust_marker_mutex
);
876 entry
= get_ust_marker(channel
, name
);
879 //ust// if (entry->rcu_pending)
880 //ust// rcu_cmm_barrier_sched();
882 entry
->rcu_pending
= 1;
883 /* write rcu_pending before calling the RCU callback */
885 //ust// call_rcu_sched(&entry->rcu, free_old_closure);
886 synchronize_rcu(); free_old_closure(&entry
->rcu
);
887 remove_ust_marker(channel
, name
); /* Ignore busy error message */
890 pthread_mutex_unlock(&ust_marker_mutex
);
893 //ust// EXPORT_SYMBOL_GPL(ust_marker_probe_unregister);
895 static struct ust_marker_entry
*
896 get_ust_marker_from_private_data(ust_marker_probe_func
*probe
, void *probe_private
)
898 struct ust_marker_entry
*entry
;
900 struct cds_hlist_head
*head
;
901 struct cds_hlist_node
*node
;
903 for (i
= 0; i
< UST_MARKER_TABLE_SIZE
; i
++) {
904 head
= &ust_marker_table
[i
];
905 cds_hlist_for_each_entry(entry
, node
, head
, hlist
) {
907 if (entry
->single
.func
== probe
908 && entry
->single
.probe_private
912 struct ust_marker_probe_closure
*closure
;
913 closure
= entry
->multi
;
914 for (i
= 0; closure
[i
].func
; i
++) {
915 if (closure
[i
].func
== probe
&&
916 closure
[i
].probe_private
927 * ust_marker_probe_unregister_private_data - Disconnect a probe from a ust_marker
928 * @probe: probe function
929 * @probe_private: probe private data
931 * Unregister a probe by providing the registered private data.
932 * Only removes the first ust_marker found in hash table.
933 * Return 0 on success or error value.
934 * We do not need to call a synchronize_sched to make sure the probes have
935 * finished running before doing a module unload, because the module unload
936 * itself uses stop_machine(), which insures that every preempt disabled section
939 int ust_marker_probe_unregister_private_data(ust_marker_probe_func
*probe
,
942 struct ust_marker_entry
*entry
;
944 struct ust_marker_probe_closure
*old
;
945 char *channel
= NULL
, *name
= NULL
;
947 pthread_mutex_lock(&ust_marker_mutex
);
948 entry
= get_ust_marker_from_private_data(probe
, probe_private
);
953 //ust// if (entry->rcu_pending)
954 //ust// rcu_cmm_barrier_sched();
955 old
= ust_marker_entry_remove_probe(entry
, NULL
, probe_private
);
956 channel
= strdup(entry
->channel
);
957 name
= strdup(entry
->name
);
958 pthread_mutex_unlock(&ust_marker_mutex
);
960 ust_marker_update_probes();
962 pthread_mutex_lock(&ust_marker_mutex
);
963 entry
= get_ust_marker(channel
, name
);
966 //ust// if (entry->rcu_pending)
967 //ust// rcu_cmm_barrier_sched();
969 entry
->rcu_pending
= 1;
970 /* write rcu_pending before calling the RCU callback */
972 //ust// call_rcu_sched(&entry->rcu, free_old_closure);
973 synchronize_rcu(); free_old_closure(&entry
->rcu
);
974 /* Ignore busy error message */
975 remove_ust_marker(channel
, name
);
977 pthread_mutex_unlock(&ust_marker_mutex
);
982 //ust// EXPORT_SYMBOL_GPL(ust_marker_probe_unregister_private_data);
985 * ust_marker_get_private_data - Get a ust_marker's probe private data
986 * @channel: ust_marker channel
987 * @name: ust_marker name
988 * @probe: probe to match
989 * @num: get the nth matching probe's private data
991 * Returns the nth private data pointer (starting from 0) matching, or an
993 * Returns the private data pointer, or an ERR_PTR.
994 * The private data pointer should _only_ be dereferenced if the caller is the
995 * owner of the data, or its content could vanish. This is mostly used to
996 * confirm that a caller is the owner of a registered probe.
998 void *ust_marker_get_private_data(const char *channel
, const char *name
,
999 ust_marker_probe_func
*probe
, int num
)
1001 struct cds_hlist_head
*head
;
1002 struct cds_hlist_node
*node
;
1003 struct ust_marker_entry
*e
;
1004 size_t channel_len
= strlen(channel
) + 1;
1005 size_t name_len
= strlen(name
) + 1;
1009 hash
= jhash(channel
, channel_len
-1, 0) ^ jhash(name
, name_len
-1, 0);
1010 head
= &ust_marker_table
[hash
& ((1 << UST_MARKER_HASH_BITS
)-1)];
1011 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
1012 if (!strcmp(channel
, e
->channel
) && !strcmp(name
, e
->name
)) {
1014 if (num
== 0 && e
->single
.func
== probe
)
1015 return e
->single
.probe_private
;
1017 struct ust_marker_probe_closure
*closure
;
1020 for (i
= 0; closure
[i
].func
; i
++) {
1021 if (closure
[i
].func
!= probe
)
1024 return closure
[i
].probe_private
;
1030 return ERR_PTR(-ENOENT
);
1032 //ust// EXPORT_SYMBOL_GPL(ust_marker_get_private_data);
1035 * ust_marker_compact_event_ids - Compact ust_marker event IDs and reassign channels
1037 * Called when no channel users are active by the channel infrastructure.
1038 * Called with lock_ust_marker() and channel mutex held.
1040 //ust// void ust_marker_compact_event_ids(void)
1042 //ust// struct ust_marker_entry *entry;
1043 //ust// unsigned int i;
1044 //ust// struct hlist_head *head;
1045 //ust// struct hlist_node *node;
1048 //ust// for (i = 0; i < UST_MARKER_TABLE_SIZE; i++) {
1049 //ust// head = &ust_marker_table[i];
1050 //ust// hlist_for_each_entry(entry, node, head, hlist) {
1051 //ust// ret = ltt_channels_get_index_from_name(entry->channel);
1052 //ust// WARN_ON(ret < 0);
1053 //ust// entry->channel_id = ret;
1054 //ust// ret = _ltt_channels_get_event_id(entry->channel,
1055 //ust// entry->name);
1056 //ust// WARN_ON(ret < 0);
1057 //ust// entry->event_id = ret;
1062 //ust//#ifdef CONFIG_MODULES
1065 * Returns 0 if current not found.
1066 * Returns 1 if current found.
1068 int lib_get_iter_ust_marker(struct ust_marker_iter
*iter
)
1070 struct ust_marker_lib
*iter_lib
;
1073 //ust// pthread_mutex_lock(&module_mutex);
1074 cds_list_for_each_entry(iter_lib
, &ust_marker_libs
, list
) {
1075 if (iter_lib
< iter
->lib
)
1077 else if (iter_lib
> iter
->lib
)
1078 iter
->ust_marker
= NULL
;
1079 found
= ust_marker_get_iter_range(&iter
->ust_marker
,
1080 iter_lib
->ust_marker_start
,
1081 iter_lib
->ust_marker_start
+ iter_lib
->ust_marker_count
);
1083 iter
->lib
= iter_lib
;
1087 //ust// pthread_mutex_unlock(&module_mutex);
1092 * ust_marker_get_iter_range - Get a next ust_marker iterator given a range.
1093 * @ust_marker: current ust_marker (in), next ust_marker (out)
1094 * @begin: beginning of the range
1095 * @end: end of the range
1097 * Returns whether a next ust_marker has been found (1) or not (0).
1098 * Will return the first ust_marker in the range if the input ust_marker is NULL.
1100 int ust_marker_get_iter_range(struct ust_marker
* const **ust_marker
,
1101 struct ust_marker
* const *begin
,
1102 struct ust_marker
* const *end
)
1104 if (!*ust_marker
&& begin
!= end
)
1105 *ust_marker
= begin
;
1106 while (*ust_marker
>= begin
&& *ust_marker
< end
) {
1108 (*ust_marker
)++; /* skip dummy */
1114 //ust// EXPORT_SYMBOL_GPL(ust_marker_get_iter_range);
1116 static void ust_marker_get_iter(struct ust_marker_iter
*iter
)
1120 found
= lib_get_iter_ust_marker(iter
);
1122 ust_marker_iter_reset(iter
);
1125 void ust_marker_iter_start(struct ust_marker_iter
*iter
)
1127 ust_marker_get_iter(iter
);
1129 //ust// EXPORT_SYMBOL_GPL(ust_marker_iter_start);
1131 void ust_marker_iter_next(struct ust_marker_iter
*iter
)
1135 * iter->ust_marker may be invalid because we blindly incremented it.
1136 * Make sure it is valid by marshalling on the ust_marker, getting the
1137 * ust_marker from following modules if necessary.
1139 ust_marker_get_iter(iter
);
1141 //ust// EXPORT_SYMBOL_GPL(ust_marker_iter_next);
1143 void ust_marker_iter_stop(struct ust_marker_iter
*iter
)
1146 //ust// EXPORT_SYMBOL_GPL(ust_marker_iter_stop);
1148 void ust_marker_iter_reset(struct ust_marker_iter
*iter
)
1151 iter
->ust_marker
= NULL
;
1153 //ust// EXPORT_SYMBOL_GPL(ust_marker_iter_reset);
1155 #ifdef CONFIG_UST_MARKER_USERSPACE
1157 * must be called with current->user_ust_marker_mutex held
1159 static void free_user_ust_marker(char __user
*state
, struct cds_hlist_head
*head
)
1161 struct user_ust_marker
*umark
;
1162 struct cds_hlist_node
*pos
, *n
;
1164 cds_hlist_for_each_entry_safe(umark
, pos
, n
, head
, hlist
) {
1165 if (umark
->state
== state
) {
1166 cds_hlist_del(&umark
->hlist
);
1173 * Update current process.
1174 * Note that we have to wait a whole scheduler period before we are sure that
1175 * every running userspace threads have their ust_marker updated.
1176 * (synchronize_sched() can be used to insure this).
1178 //ust// void ust_marker_update_process(void)
1180 //ust// struct user_ust_marker *umark;
1181 //ust// struct hlist_node *pos;
1182 //ust// struct ust_marker_entry *entry;
1184 //ust// pthread_mutex_lock(&ust_marker_mutex);
1185 //ust// pthread_mutex_lock(¤t->group_leader->user_ust_marker_mutex);
1186 //ust// if (strcmp(current->comm, "testprog") == 0)
1187 //ust// DBG("do update pending for testprog");
1188 //ust// hlist_for_each_entry(umark, pos,
1189 //ust// ¤t->group_leader->user_ust_marker, hlist) {
1190 //ust// DBG("Updating ust_marker %s in %s", umark->name, current->comm);
1191 //ust// entry = get_ust_marker("userspace", umark->name);
1192 //ust// if (entry) {
1193 //ust// if (entry->format &&
1194 //ust// strcmp(entry->format, umark->format) != 0) {
1195 //ust// WARN("error, wrong format in process %s",
1196 //ust// current->comm);
1199 //ust// if (put_user(!!entry->refcount, umark->state)) {
1200 //ust// WARN("ust_marker in %s caused a fault",
1201 //ust// current->comm);
1205 //ust// if (put_user(0, umark->state)) {
1206 //ust// WARN("ust_marker in %s caused a fault", current->comm);
1211 //ust// clear_thread_flag(TIF_UST_MARKER_PENDING);
1212 //ust// pthread_mutex_unlock(¤t->group_leader->user_ust_marker_mutex);
1213 //ust// pthread_mutex_unlock(&ust_marker_mutex);
1217 * Called at process exit and upon do_execve().
1218 * We assume that when the leader exits, no more references can be done to the
1219 * leader structure by the other threads.
1221 void exit_user_ust_marker(struct task_struct
*p
)
1223 struct user_ust_marker
*umark
;
1224 struct cds_hlist_node
*pos
, *n
;
1226 if (thread_group_leader(p
)) {
1227 pthread_mutex_lock(&ust_marker_mutex
);
1228 pthread_mutex_lock(&p
->user_ust_marker_mutex
);
1229 cds_hlist_for_each_entry_safe(umark
, pos
, n
, &p
->user_ust_marker
,
1232 INIT_HLIST_HEAD(&p
->user_ust_marker
);
1233 p
->user_ust_marker_sequence
++;
1234 pthread_mutex_unlock(&p
->user_ust_marker_mutex
);
1235 pthread_mutex_unlock(&ust_marker_mutex
);
1239 int is_ust_marker_enabled(const char *channel
, const char *name
)
1241 struct ust_marker_entry
*entry
;
1243 pthread_mutex_lock(&ust_marker_mutex
);
1244 entry
= get_ust_marker(channel
, name
);
1245 pthread_mutex_unlock(&ust_marker_mutex
);
1247 return entry
&& !!entry
->refcount
;
1251 int ust_marker_module_notify(struct notifier_block
*self
,
1252 unsigned long val
, void *data
)
1254 struct module
*mod
= data
;
1257 case MODULE_STATE_COMING
:
1258 ust_marker_update_probe_range(mod
->ust_marker
,
1259 mod
->ust_marker
+ mod
->num_ust_marker
);
1261 case MODULE_STATE_GOING
:
1262 ust_marker_update_probe_range(mod
->ust_marker
,
1263 mod
->ust_marker
+ mod
->num_ust_marker
);
1269 struct notifier_block ust_marker_module_nb
= {
1270 .notifier_call
= ust_marker_module_notify
,
1274 //ust// static int init_ust_marker(void)
1276 //ust// return register_module_notifier(&ust_marker_module_nb);
1278 //ust// __initcall(init_ust_marker);
1279 /* TODO: call ust_marker_module_nb() when a library is linked at runtime (dlopen)? */
1281 #endif /* CONFIG_MODULES */
1283 void ltt_dump_ust_marker_state(struct ust_trace
*trace
)
1285 struct ust_marker_entry
*entry
;
1286 struct ltt_probe_private_data call_data
;
1287 struct cds_hlist_head
*head
;
1288 struct cds_hlist_node
*node
;
1291 pthread_mutex_lock(&ust_marker_mutex
);
1292 call_data
.trace
= trace
;
1293 call_data
.serializer
= NULL
;
1295 for (i
= 0; i
< UST_MARKER_TABLE_SIZE
; i
++) {
1296 head
= &ust_marker_table
[i
];
1297 cds_hlist_for_each_entry(entry
, node
, head
, hlist
) {
1298 __ust_marker(metadata
, core_marker_id
,
1300 "channel %s name %s event_id %hu "
1301 "int #1u%zu long #1u%zu pointer #1u%zu "
1302 "size_t #1u%zu alignment #1u%u",
1306 sizeof(int), sizeof(long),
1307 sizeof(void *), sizeof(size_t),
1308 ltt_get_alignment());
1310 __ust_marker(metadata
,
1313 "channel %s name %s format %s",
1319 pthread_mutex_unlock(&ust_marker_mutex
);
1321 //ust// EXPORT_SYMBOL_GPL(ltt_dump_ust_marker_state);
1323 static void (*new_ust_marker_cb
)(struct ust_marker
*) = NULL
;
1325 void ust_marker_set_new_ust_marker_cb(void (*cb
)(struct ust_marker
*))
1327 new_ust_marker_cb
= cb
;
1330 static void new_ust_marker(struct ust_marker
* const *start
, struct ust_marker
* const *end
)
1332 if (new_ust_marker_cb
) {
1333 struct ust_marker
* const *m
;
1335 for(m
= start
; m
< end
; m
++) {
1337 new_ust_marker_cb(*m
);
1342 int ust_marker_register_lib(struct ust_marker
* const *ust_marker_start
, int ust_marker_count
)
1344 struct ust_marker_lib
*pl
, *iter
;
1346 pl
= (struct ust_marker_lib
*) zmalloc(sizeof(struct ust_marker_lib
));
1348 pl
->ust_marker_start
= ust_marker_start
;
1349 pl
->ust_marker_count
= ust_marker_count
;
1351 /* FIXME: maybe protect this with its own mutex? */
1355 * We sort the libs by struct lib pointer address.
1357 cds_list_for_each_entry_reverse(iter
, &ust_marker_libs
, list
) {
1358 BUG_ON(iter
== pl
); /* Should never be in the list twice */
1360 /* We belong to the location right after iter. */
1361 cds_list_add(&pl
->list
, &iter
->list
);
1365 /* We should be added at the head of the list */
1366 cds_list_add(&pl
->list
, &ust_marker_libs
);
1368 unlock_ust_marker();
1370 new_ust_marker(ust_marker_start
, ust_marker_start
+ ust_marker_count
);
1372 /* FIXME: update just the loaded lib */
1373 lib_update_ust_marker();
1375 DBG("just registered a ust_marker section from %p and having %d ust_marker (minus dummy ust_marker)", ust_marker_start
, ust_marker_count
);
1380 int ust_marker_unregister_lib(struct ust_marker
* const *ust_marker_start
)
1382 struct ust_marker_lib
*lib
;
1384 /*FIXME: implement; but before implementing, ust_marker_register_lib must
1385 have appropriate locking. */
1389 /* FIXME: we should probably take a mutex here on libs */
1390 //ust// pthread_mutex_lock(&module_mutex);
1391 cds_list_for_each_entry(lib
, &ust_marker_libs
, list
) {
1392 if(lib
->ust_marker_start
== ust_marker_start
) {
1393 struct ust_marker_lib
*lib2free
= lib
;
1394 cds_list_del(&lib
->list
);
1400 unlock_ust_marker();
1405 static int initialized
= 0;
1407 void __attribute__((constructor
)) init_ust_marker(void)
1410 ust_marker_register_lib(__start___ust_marker_ptrs
,
1411 __stop___ust_marker_ptrs
1412 - __start___ust_marker_ptrs
);
1417 void __attribute__((destructor
)) destroy_ust_marker(void)
1419 ust_marker_unregister_lib(__start___ust_marker_ptrs
);