probes/lttng-tracepoint-event-impl.h

   1 /*
   2  * lttng-tracepoint-event-impl.h
   3  *
   4  * Copyright (C) 2009 Steven Rostedt <rostedt@goodmis.org>
   5  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; only
  10  * version 2.1 of the License.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20  */
  21
  22 #include <linux/uaccess.h>
  23 #include <linux/debugfs.h>
  24 #include <linux/rculist.h>
  25 #include <asm/byteorder.h>
  26 #include <linux/swab.h>
  27
  28 #include <probes/lttng.h>
  29 #include <probes/lttng-types.h>
  30 #include <probes/lttng-probe-user.h>
  31 #include <wrapper/vmalloc.h>    /* for wrapper_vmalloc_sync_all() */
  32 #include <wrapper/ringbuffer/frontend_types.h>
  33 #include <wrapper/ringbuffer/backend.h>
  34 #include <wrapper/rcu.h>
  35 #include <lttng-events.h>
  36 #include <lttng-tracer-core.h>
  37
  38 /*
  39  * Macro declarations used for all stages.
  40  */
  41
  42 /*
  43  * LTTng name mapping macros. LTTng remaps some of the kernel events to
  44  * enforce name-spacing.
  45  */
  46 #undef LTTNG_TRACEPOINT_EVENT_MAP
  47 #define LTTNG_TRACEPOINT_EVENT_MAP(name, map, proto, args, fields) \
  48         LTTNG_TRACEPOINT_EVENT_CLASS(map,                               \
  49                              PARAMS(proto),                             \
  50                              PARAMS(args),                              \
  51                              PARAMS(fields))                            \
  52         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  53
  54 #undef LTTNG_TRACEPOINT_EVENT_MAP_NOARGS
  55 #define LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, map, fields)            \
  56         LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(map,                        \
  57                              PARAMS(fields))                            \
  58         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(map, name, map)
  59
  60 #undef LTTNG_TRACEPOINT_EVENT_CODE_MAP
  61 #define LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, map, proto, args, _locvar, _code_pre, fields, _code_post) \
  62         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(map,                          \
  63                              PARAMS(proto),                             \
  64                              PARAMS(args),                              \
  65                              PARAMS(_locvar),                           \
  66                              PARAMS(_code_pre),                         \
  67                              PARAMS(fields),                            \
  68                              PARAMS(_code_post))                        \
  69         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  70
  71 #undef LTTNG_TRACEPOINT_EVENT_CODE
  72 #define LTTNG_TRACEPOINT_EVENT_CODE(name, proto, args, _locvar, _code_pre, fields, _code_post) \
  73         LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, name,                     \
  74                              PARAMS(proto),                             \
  75                              PARAMS(args),                              \
  76                              PARAMS(_locvar),                           \
  77                              PARAMS(_code_pre),                         \
  78                              PARAMS(fields),                            \
  79                              PARAMS(_code_post))
  80
  81 /*
  82  * LTTNG_TRACEPOINT_EVENT_CLASS can be used to add a generic function
  83  * handlers for events. That is, if all events have the same parameters
  84  * and just have distinct trace points.  Each tracepoint can be defined
  85  * with LTTNG_TRACEPOINT_EVENT_INSTANCE and that will map the
  86  * LTTNG_TRACEPOINT_EVENT_CLASS to the tracepoint.
  87  *
  88  * LTTNG_TRACEPOINT_EVENT is a one to one mapping between tracepoint and
  89  * template.
  90  */
  91
  92 #undef LTTNG_TRACEPOINT_EVENT
  93 #define LTTNG_TRACEPOINT_EVENT(name, proto, args, fields)               \
  94         LTTNG_TRACEPOINT_EVENT_MAP(name, name,                          \
  95                         PARAMS(proto),                                  \
  96                         PARAMS(args),                                   \
  97                         PARAMS(fields))
  98
  99 #undef LTTNG_TRACEPOINT_EVENT_NOARGS
 100 #define LTTNG_TRACEPOINT_EVENT_NOARGS(name, fields)                     \
 101         LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, name, PARAMS(fields))
 102
 103 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE
 104 #define LTTNG_TRACEPOINT_EVENT_INSTANCE(template, name, proto, args)    \
 105         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(template, name, name, PARAMS(proto), PARAMS(args))
 106
 107 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS
 108 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS(template, name)  \
 109         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(template, name, name)
 110
 111 #undef LTTNG_TRACEPOINT_EVENT_CLASS
 112 #define LTTNG_TRACEPOINT_EVENT_CLASS(_name, _proto, _args, _fields) \
 113         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(_proto), PARAMS(_args), , , \
 114                 PARAMS(_fields), )
 115
 116 #undef LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS
 117 #define LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(_name, _fields) \
 118         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, , , PARAMS(_fields), )
 119
 120
 121 /*
 122  * Stage 1 of the trace events.
 123  *
 124  * Create dummy trace calls for each events, verifying that the LTTng module
 125  * instrumentation headers match the kernel arguments. Will be optimized
 126  * out by the compiler.
 127  */
 128
 129 /* Reset all macros within TRACEPOINT_EVENT */
 130 #include <probes/lttng-events-reset.h>
 131
 132 #undef TP_PROTO
 133 #define TP_PROTO(...)   __VA_ARGS__
 134
 135 #undef TP_ARGS
 136 #define TP_ARGS(...)    __VA_ARGS__
 137
 138 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 139 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 140 void trace_##_name(_proto);
 141
 142 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 143 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 144 void trace_##_name(void);
 145
 146 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 147
 148 /*
 149  * Stage 1.1 of the trace events.
 150  *
 151  * Create dummy trace prototypes for each event class, and for each used
 152  * template. This will allow checking whether the prototypes from the
 153  * class and the instance using the class actually match.
 154  */
 155
 156 #include <probes/lttng-events-reset.h>  /* Reset all macros within TRACE_EVENT */
 157
 158 #undef TP_PROTO
 159 #define TP_PROTO(...)   __VA_ARGS__
 160
 161 #undef TP_ARGS
 162 #define TP_ARGS(...)    __VA_ARGS__
 163
 164 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 165 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 166 void __event_template_proto___##_template(_proto);
 167
 168 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 169 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 170 void __event_template_proto___##_template(void);
 171
 172 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 173 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 174 void __event_template_proto___##_name(_proto);
 175
 176 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 177 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 178 void __event_template_proto___##_name(void);
 179
 180 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 181
 182 /*
 183  * Stage 2 of the trace events.
 184  *
 185  * Create event field type metadata section.
 186  * Each event produce an array of fields.
 187  */
 188
 189 /* Reset all macros within TRACEPOINT_EVENT */
 190 #include <probes/lttng-events-reset.h>
 191 #include <probes/lttng-events-write.h>
 192 #include <probes/lttng-events-nowrite.h>
 193
 194 #undef _ctf_integer_ext
 195 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 196         {                                                       \
 197           .name = #_item,                                       \
 198           .type = __type_integer(_type, 0, 0, -1, _byte_order, _base, none),\
 199           .nowrite = _nowrite,                                  \
 200           .user = _user,                                        \
 201         },
 202
 203 #undef _ctf_array_encoded
 204 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 205         {                                                       \
 206           .name = #_item,                                       \
 207           .type =                                               \
 208                 {                                               \
 209                   .atype = atype_array,                         \
 210                   .u =                                          \
 211                         {                                       \
 212                           .array =                              \
 213                                 {                               \
 214                                   .elem_type = __type_integer(_type, 0, 0, 0, __BYTE_ORDER, 10, _encoding), \
 215                                   .length = _length,            \
 216                                 }                               \
 217                         }                                       \
 218                 },                                              \
 219           .nowrite = _nowrite,                                  \
 220           .user = _user,                                        \
 221         },
 222
 223 #undef _ctf_array_bitfield
 224 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 225         {                                                       \
 226           .name = #_item,                                       \
 227           .type =                                               \
 228                 {                                               \
 229                   .atype = atype_array,                         \
 230                   .u =                                          \
 231                         {                                       \
 232                           .array =                              \
 233                                 {                               \
 234                                   .elem_type = __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none), \
 235                                   .length = (_length) * sizeof(_type) * CHAR_BIT, \
 236                                   .elem_alignment = lttng_alignof(_type), \
 237                                 }                               \
 238                         }                                       \
 239                 },                                              \
 240           .nowrite = _nowrite,                                  \
 241           .user = _user,                                        \
 242         },
 243
 244
 245 #undef _ctf_sequence_encoded
 246 #define _ctf_sequence_encoded(_type, _item, _src,               \
 247                         _length_type, _src_length, _encoding,   \
 248                         _byte_order, _base, _user, _nowrite)    \
 249         {                                                       \
 250           .name = #_item,                                       \
 251           .type =                                               \
 252                 {                                               \
 253                   .atype = atype_sequence,                      \
 254                   .u =                                          \
 255                         {                                       \
 256                           .sequence =                           \
 257                                 {                               \
 258                                   .length_type = __type_integer(_length_type, 0, 0, 0, __BYTE_ORDER, 10, none), \
 259                                   .elem_type = __type_integer(_type, 0, 0, -1, _byte_order, _base, _encoding), \
 260                                 },                              \
 261                         },                                      \
 262                 },                                              \
 263           .nowrite = _nowrite,                                  \
 264           .user = _user,                                        \
 265         },
 266
 267 #undef _ctf_sequence_bitfield
 268 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 269                         _length_type, _src_length,              \
 270                         _user, _nowrite)                        \
 271         {                                                       \
 272           .name = #_item,                                       \
 273           .type =                                               \
 274                 {                                               \
 275                   .atype = atype_sequence,                      \
 276                   .u =                                          \
 277                         {                                       \
 278                           .sequence =                           \
 279                                 {                               \
 280                                   .length_type = __type_integer(_length_type, 0, 0, 0, __BYTE_ORDER, 10, none), \
 281                                   .elem_type = __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none), \
 282                                   .elem_alignment = lttng_alignof(_type), \
 283                                 },                              \
 284                         },                                      \
 285                 },                                              \
 286           .nowrite = _nowrite,                                  \
 287           .user = _user,                                        \
 288         },
 289
 290 #undef _ctf_string
 291 #define _ctf_string(_item, _src, _user, _nowrite)               \
 292         {                                                       \
 293           .name = #_item,                                       \
 294           .type =                                               \
 295                 {                                               \
 296                   .atype = atype_string,                        \
 297                   .u =                                          \
 298                         {                                       \
 299                           .basic = { .string = { .encoding = lttng_encode_UTF8 } } \
 300                         },                                      \
 301                 },                                              \
 302           .nowrite = _nowrite,                                  \
 303           .user = _user,                                        \
 304         },
 305
 306 #undef TP_FIELDS
 307 #define TP_FIELDS(...)  __VA_ARGS__     /* Only one used in this phase */
 308
 309 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 310 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 311         static const struct lttng_event_field __event_fields___##_name[] = { \
 312                 _fields                                                      \
 313         };
 314
 315 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 316 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 317         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, PARAMS(_fields), _code_post)
 318
 319 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 320
 321 /*
 322  * Stage 3 of the trace events.
 323  *
 324  * Create probe callback prototypes.
 325  */
 326
 327 /* Reset all macros within TRACEPOINT_EVENT */
 328 #include <probes/lttng-events-reset.h>
 329
 330 #undef TP_PROTO
 331 #define TP_PROTO(...)   __VA_ARGS__
 332
 333 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 334 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 335 static void __event_probe__##_name(void *__data, _proto);
 336
 337 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 338 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 339 static void __event_probe__##_name(void *__data);
 340
 341 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 342
 343 /*
 344  * Stage 4 of the trace events.
 345  *
 346  * Create static inline function that calculates event size.
 347  */
 348
 349 /* Reset all macros within TRACEPOINT_EVENT */
 350 #include <probes/lttng-events-reset.h>
 351 #include <probes/lttng-events-write.h>
 352
 353 #undef _ctf_integer_ext
 354 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 355         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 356         __event_len += sizeof(_type);
 357
 358 #undef _ctf_array_encoded
 359 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 360         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 361         __event_len += sizeof(_type) * (_length);
 362
 363 #undef _ctf_array_bitfield
 364 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 365         _ctf_array_encoded(_type, _item, _src, _length, none, _user, _nowrite)
 366
 367 #undef _ctf_sequence_encoded
 368 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 369                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 370         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_length_type)); \
 371         __event_len += sizeof(_length_type);                                   \
 372         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 373         {                                                                               \
 374                 size_t __seqlen = (_src_length);                                        \
 375                                                                                         \
 376                 if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 377                         goto error;                                                     \
 378                 barrier();      /* reserve before use. */                               \
 379                 this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = __seqlen; \
 380                 __event_len += sizeof(_type) * __seqlen;                                \
 381         }
 382
 383 #undef _ctf_sequence_bitfield
 384 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 385                         _length_type, _src_length,              \
 386                         _user, _nowrite)                        \
 387         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 388                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 389
 390 /*
 391  * ctf_user_string includes \0. If returns 0, it faulted, so we set size to
 392  * 1 (\0 only).
 393  */
 394 #undef _ctf_string
 395 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 396         if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 397                 goto error;                                                    \
 398         barrier();      /* reserve before use. */                              \
 399         if (_user) {                                                           \
 400                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 401                         max_t(size_t, lttng_strlen_user_inatomic(_src), 1);    \
 402         } else {                                                               \
 403                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 404                         strlen(_src) + 1;                                      \
 405         }
 406
 407 #undef TP_PROTO
 408 #define TP_PROTO(...)   __VA_ARGS__
 409
 410 #undef TP_FIELDS
 411 #define TP_FIELDS(...)  __VA_ARGS__
 412
 413 #undef TP_locvar
 414 #define TP_locvar(...)  __VA_ARGS__
 415
 416 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 417 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 418 static inline ssize_t __event_get_size__##_name(void *__tp_locvar, _proto)     \
 419 {                                                                             \
 420         size_t __event_len = 0;                                               \
 421         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 422         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 423                                                                               \
 424         _fields                                                               \
 425         return __event_len;                                                   \
 426                                                                               \
 427 error:                                                                        \
 428         __attribute__((unused));                                              \
 429         return -1;                                                            \
 430 }
 431
 432 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 433 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 434 static inline ssize_t __event_get_size__##_name(void *__tp_locvar)            \
 435 {                                                                             \
 436         size_t __event_len = 0;                                               \
 437         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 438         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 439                                                                               \
 440         _fields                                                               \
 441         return __event_len;                                                   \
 442                                                                               \
 443 error:                                                                        \
 444         __attribute__((unused));                                              \
 445         return -1;                                                            \
 446 }
 447
 448 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 449
 450
 451 /*
 452  * Stage 4.1 of tracepoint event generation.
 453  *
 454  * Create static inline function that layout the filter stack data.
 455  * We make both write and nowrite data available to the filter.
 456  */
 457
 458 /* Reset all macros within TRACEPOINT_EVENT */
 459 #include <probes/lttng-events-reset.h>
 460 #include <probes/lttng-events-write.h>
 461 #include <probes/lttng-events-nowrite.h>
 462
 463 #undef _ctf_integer_ext_fetched
 464 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 465         if (lttng_is_signed_type(_type)) {                                     \
 466                 int64_t __ctf_tmp_int64;                                       \
 467                 switch (sizeof(_type)) {                                       \
 468                 case 1:                                                        \
 469                 {                                                              \
 470                         union { _type t; int8_t v; } __tmp = { (_type) (_src) }; \
 471                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 472                         break;                                                 \
 473                 }                                                              \
 474                 case 2:                                                        \
 475                 {                                                              \
 476                         union { _type t; int16_t v; } __tmp = { (_type) (_src) }; \
 477                         if (_byte_order != __BYTE_ORDER)                       \
 478                                 __swab16s(&__tmp.v);                           \
 479                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 480                         break;                                                 \
 481                 }                                                              \
 482                 case 4:                                                        \
 483                 {                                                              \
 484                         union { _type t; int32_t v; } __tmp = { (_type) (_src) }; \
 485                         if (_byte_order != __BYTE_ORDER)                       \
 486                                 __swab32s(&__tmp.v);                           \
 487                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 488                         break;                                                 \
 489                 }                                                              \
 490                 case 8:                                                        \
 491                 {                                                              \
 492                         union { _type t; int64_t v; } __tmp = { (_type) (_src) }; \
 493                         if (_byte_order != __BYTE_ORDER)                       \
 494                                 __swab64s(&__tmp.v);                           \
 495                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 496                         break;                                                 \
 497                 }                                                              \
 498                 default:                                                       \
 499                         BUG_ON(1);                                             \
 500                 };                                                             \
 501                 memcpy(__stack_data, &__ctf_tmp_int64, sizeof(int64_t));       \
 502         } else {                                                               \
 503                 uint64_t __ctf_tmp_uint64;                                     \
 504                 switch (sizeof(_type)) {                                       \
 505                 case 1:                                                        \
 506                 {                                                              \
 507                         union { _type t; uint8_t v; } __tmp = { (_type) (_src) }; \
 508                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 509                         break;                                                 \
 510                 }                                                              \
 511                 case 2:                                                        \
 512                 {                                                              \
 513                         union { _type t; uint16_t v; } __tmp = { (_type) (_src) }; \
 514                         if (_byte_order != __BYTE_ORDER)                       \
 515                                 __swab16s(&__tmp.v);                           \
 516                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 517                         break;                                                 \
 518                 }                                                              \
 519                 case 4:                                                        \
 520                 {                                                              \
 521                         union { _type t; uint32_t v; } __tmp = { (_type) (_src) }; \
 522                         if (_byte_order != __BYTE_ORDER)                       \
 523                                 __swab32s(&__tmp.v);                           \
 524                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 525                         break;                                                 \
 526                 }                                                              \
 527                 case 8:                                                        \
 528                 {                                                              \
 529                         union { _type t; uint64_t v; } __tmp = { (_type) (_src) }; \
 530                         if (_byte_order != __BYTE_ORDER)                       \
 531                                 __swab64s(&__tmp.v);                           \
 532                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 533                         break;                                                 \
 534                 }                                                              \
 535                 default:                                                       \
 536                         BUG_ON(1);                                             \
 537                 };                                                             \
 538                 memcpy(__stack_data, &__ctf_tmp_uint64, sizeof(uint64_t));     \
 539         }                                                                      \
 540         __stack_data += sizeof(int64_t);
 541
 542 #undef _ctf_integer_ext_isuser0
 543 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 544         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 545
 546 #undef _ctf_integer_ext_isuser1
 547 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 548 {                                                                                       \
 549         union {                                                                         \
 550                 char __array[sizeof(_user_src)];                                        \
 551                 __typeof__(_user_src) __v;                                              \
 552         } __tmp_fetch;                                                                  \
 553         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 554                                 &(_user_src), sizeof(_user_src)))                       \
 555                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 556         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 557 }
 558
 559 #undef _ctf_integer_ext
 560 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 561         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 562
 563 #undef _ctf_array_encoded
 564 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 565         {                                                                      \
 566                 unsigned long __ctf_tmp_ulong = (unsigned long) (_length);     \
 567                 const void *__ctf_tmp_ptr = (_src);                            \
 568                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 569                 __stack_data += sizeof(unsigned long);                         \
 570                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 571                 __stack_data += sizeof(void *);                                \
 572         }
 573
 574 #undef _ctf_array_bitfield
 575 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 576         _ctf_array_encoded(_type, _item, _src, _length, none, _user, _nowrite)
 577
 578 #undef _ctf_sequence_encoded
 579 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                \
 580                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 581         {                                                                      \
 582                 unsigned long __ctf_tmp_ulong = (unsigned long) (_src_length); \
 583                 const void *__ctf_tmp_ptr = (_src);                            \
 584                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 585                 __stack_data += sizeof(unsigned long);                         \
 586                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 587                 __stack_data += sizeof(void *);                                \
 588         }
 589
 590 #undef _ctf_sequence_bitfield
 591 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 592                         _length_type, _src_length,              \
 593                         _user, _nowrite)                        \
 594         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 595                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 596
 597 #undef _ctf_string
 598 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 599         {                                                                      \
 600                 const void *__ctf_tmp_ptr = (_src);                            \
 601                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 602                 __stack_data += sizeof(void *);                                \
 603         }
 604
 605 #undef TP_PROTO
 606 #define TP_PROTO(...) __VA_ARGS__
 607
 608 #undef TP_FIELDS
 609 #define TP_FIELDS(...) __VA_ARGS__
 610
 611 #undef TP_locvar
 612 #define TP_locvar(...)  __VA_ARGS__
 613
 614 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 615 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 616 static inline                                                                 \
 617 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 618                 void *__tp_locvar)                                            \
 619 {                                                                             \
 620         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 621                                                                               \
 622         _fields                                                               \
 623 }
 624
 625 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 626 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 627 static inline                                                                 \
 628 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 629                 void *__tp_locvar, _proto)                                    \
 630 {                                                                             \
 631         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 632                                                                               \
 633         _fields                                                               \
 634 }
 635
 636 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 637
 638 /*
 639  * Stage 5 of the trace events.
 640  *
 641  * Create static inline function that calculates event payload alignment.
 642  */
 643
 644 /* Reset all macros within TRACEPOINT_EVENT */
 645 #include <probes/lttng-events-reset.h>
 646 #include <probes/lttng-events-write.h>
 647
 648 #undef _ctf_integer_ext
 649 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 650         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 651
 652 #undef _ctf_array_encoded
 653 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 654         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 655
 656 #undef _ctf_array_bitfield
 657 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 658         _ctf_array_encoded(_type, _item, _src, _length, none, _user, _nowrite)
 659
 660 #undef _ctf_sequence_encoded
 661 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 662                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 663         __event_align = max_t(size_t, __event_align, lttng_alignof(_length_type)); \
 664         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 665
 666 #undef _ctf_sequence_bitfield
 667 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 668                         _length_type, _src_length,              \
 669                         _user, _nowrite)                        \
 670         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 671                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 672
 673 #undef _ctf_string
 674 #define _ctf_string(_item, _src, _user, _nowrite)
 675
 676 #undef TP_PROTO
 677 #define TP_PROTO(...)   __VA_ARGS__
 678
 679 #undef TP_FIELDS
 680 #define TP_FIELDS(...)  __VA_ARGS__
 681
 682 #undef TP_locvar
 683 #define TP_locvar(...)  __VA_ARGS__
 684
 685 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 686 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 687 static inline size_t __event_get_align__##_name(void *__tp_locvar, _proto)    \
 688 {                                                                             \
 689         size_t __event_align = 1;                                             \
 690         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 691                                                                               \
 692         _fields                                                               \
 693         return __event_align;                                                 \
 694 }
 695
 696 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 697 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 698 static inline size_t __event_get_align__##_name(void *__tp_locvar)            \
 699 {                                                                             \
 700         size_t __event_align = 1;                                             \
 701         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 702                                                                               \
 703         _fields                                                               \
 704         return __event_align;                                                 \
 705 }
 706
 707 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 708
 709 /*
 710  * Stage 6 of tracepoint event generation.
 711  *
 712  * Create the probe function. This function calls event size calculation
 713  * and writes event data into the buffer.
 714  */
 715
 716 /* Reset all macros within TRACEPOINT_EVENT */
 717 #include <probes/lttng-events-reset.h>
 718 #include <probes/lttng-events-write.h>
 719
 720 #undef _ctf_integer_ext_fetched
 721 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 722         {                                                               \
 723                 _type __tmp = _src;                                     \
 724                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(__tmp));\
 725                 __chan->ops->event_write(&__ctx, &__tmp, sizeof(__tmp));\
 726         }
 727
 728 #undef _ctf_integer_ext_isuser0
 729 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 730         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 731
 732 #undef _ctf_integer_ext_isuser1
 733 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 734 {                                                                              \
 735         union {                                                                         \
 736                 char __array[sizeof(_user_src)];                                        \
 737                 __typeof__(_user_src) __v;                                              \
 738         } __tmp_fetch;                                                                  \
 739         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 740                                 &(_user_src), sizeof(_user_src)))                       \
 741                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 742         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 743 }
 744
 745 #undef _ctf_integer_ext
 746 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 747         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 748
 749 #undef _ctf_array_encoded
 750 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 751         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 752         if (_user) {                                                    \
 753                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 754         } else {                                                        \
 755                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 756         }
 757
 758 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 759 #undef _ctf_array_bitfield
 760 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 761         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 762         if (_user) {                                                    \
 763                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 764         } else {                                                        \
 765                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 766         }
 767 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 768 /*
 769  * For big endian, we need to byteswap into little endian.
 770  */
 771 #undef _ctf_array_bitfield
 772 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 773         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 774         {                                                               \
 775                 size_t _i;                                              \
 776                                                                         \
 777                 for (_i = 0; _i < (_length); _i++) {                    \
 778                         _type _tmp;                                     \
 779                                                                         \
 780                         if (_user) {                                    \
 781                                 if (get_user(_tmp, (_type *) _src + _i)) \
 782                                         _tmp = 0;                       \
 783                         } else {                                        \
 784                                 _tmp = ((_type *) _src)[_i];            \
 785                         }                                               \
 786                         switch (sizeof(_type)) {                        \
 787                         case 1:                                         \
 788                                 break;                                  \
 789                         case 2:                                         \
 790                                 _tmp = cpu_to_le16(_tmp);               \
 791                                 break;                                  \
 792                         case 4:                                         \
 793                                 _tmp = cpu_to_le32(_tmp);               \
 794                                 break;                                  \
 795                         case 8:                                         \
 796                                 _tmp = cpu_to_le64(_tmp);               \
 797                                 break;                                  \
 798                         default:                                        \
 799                                 BUG_ON(1);                              \
 800                         }                                               \
 801                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
 802                 }                                                       \
 803         }
 804 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 805
 806 #undef _ctf_sequence_encoded
 807 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,         \
 808                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 809         {                                                               \
 810                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx]; \
 811                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 812                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 813         }                                                               \
 814         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 815         if (_user) {                                                    \
 816                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 817                         sizeof(_type) * __get_dynamic_len(dest));       \
 818         } else {                                                        \
 819                 __chan->ops->event_write(&__ctx, _src,                  \
 820                         sizeof(_type) * __get_dynamic_len(dest));       \
 821         }
 822
 823 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 824 #undef _ctf_sequence_bitfield
 825 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 826                         _length_type, _src_length,              \
 827                         _user, _nowrite)                        \
 828         {                                                               \
 829                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 830                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 831                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 832         }                                                               \
 833         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 834         if (_user) {                                                    \
 835                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 836                         sizeof(_type) * __get_dynamic_len(dest));       \
 837         } else {                                                        \
 838                 __chan->ops->event_write(&__ctx, _src,                  \
 839                         sizeof(_type) * __get_dynamic_len(dest));       \
 840         }
 841 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 842 /*
 843  * For big endian, we need to byteswap into little endian.
 844  */
 845 #undef _ctf_sequence_bitfield
 846 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 847                         _length_type, _src_length,              \
 848                         _user, _nowrite)                        \
 849         {                                                       \
 850                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 851                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 852                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 853         }                                                               \
 854         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 855         {                                                               \
 856                 size_t _i, _length;                                     \
 857                                                                         \
 858                 _length = __get_dynamic_len(dest);                      \
 859                 for (_i = 0; _i < _length; _i++) {                      \
 860                         _type _tmp;                                     \
 861                                                                         \
 862                         if (_user) {                                    \
 863                                 if (get_user(_tmp, (_type *) _src + _i)) \
 864                                         _tmp = 0;                       \
 865                         } else {                                        \
 866                                 _tmp = ((_type *) _src)[_i];            \
 867                         }                                               \
 868                         switch (sizeof(_type)) {                        \
 869                         case 1:                                         \
 870                                 break;                                  \
 871                         case 2:                                         \
 872                                 _tmp = cpu_to_le16(_tmp);               \
 873                                 break;                                  \
 874                         case 4:                                         \
 875                                 _tmp = cpu_to_le32(_tmp);               \
 876                                 break;                                  \
 877                         case 8:                                         \
 878                                 _tmp = cpu_to_le64(_tmp);               \
 879                                 break;                                  \
 880                         default:                                        \
 881                                 BUG_ON(1);                              \
 882                         }                                               \
 883                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
 884                 }                                                       \
 885         }
 886 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 887
 888 #undef _ctf_string
 889 #define _ctf_string(_item, _src, _user, _nowrite)                       \
 890         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(*(_src)));      \
 891         if (_user) {                                                    \
 892                 __chan->ops->event_strcpy_from_user(&__ctx, _src,       \
 893                         __get_dynamic_len(dest));                       \
 894         } else {                                                        \
 895                 __chan->ops->event_strcpy(&__ctx, _src,                 \
 896                         __get_dynamic_len(dest));                       \
 897         }
 898
 899 /* Beware: this get len actually consumes the len value */
 900 #undef __get_dynamic_len
 901 #define __get_dynamic_len(field)        this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx++]
 902
 903 #undef TP_PROTO
 904 #define TP_PROTO(...)   __VA_ARGS__
 905
 906 #undef TP_ARGS
 907 #define TP_ARGS(...)    __VA_ARGS__
 908
 909 #undef TP_FIELDS
 910 #define TP_FIELDS(...)  __VA_ARGS__
 911
 912 #undef TP_locvar
 913 #define TP_locvar(...)  __VA_ARGS__
 914
 915 #undef TP_code_pre
 916 #define TP_code_pre(...)        __VA_ARGS__
 917
 918 #undef TP_code_post
 919 #define TP_code_post(...)       __VA_ARGS__
 920
 921 /*
 922  * For state dump, check that "session" argument (mandatory) matches the
 923  * session this event belongs to. Ensures that we write state dump data only
 924  * into the started session, not into all sessions.
 925  */
 926 #ifdef TP_SESSION_CHECK
 927 #define _TP_SESSION_CHECK(session, csession)    (session == csession)
 928 #else /* TP_SESSION_CHECK */
 929 #define _TP_SESSION_CHECK(session, csession)    1
 930 #endif /* TP_SESSION_CHECK */
 931
 932 /*
 933  * Using twice size for filter stack data to hold size and pointer for
 934  * each field (worse case). For integers, max size required is 64-bit.
 935  * Same for double-precision floats. Those fit within
 936  * 2*sizeof(unsigned long) for all supported architectures.
 937  * Perform UNION (||) of filter runtime list.
 938  */
 939 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 940 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 941 static void __event_probe__##_name(void *__data, _proto)                      \
 942 {                                                                             \
 943         struct probe_local_vars { _locvar };                                  \
 944         struct lttng_event *__event = __data;                                 \
 945         struct lttng_probe_ctx __lttng_probe_ctx = {                                  \
 946                 .event = __event,                                             \
 947                 .interruptible = !irqs_disabled(),                            \
 948         };                                                                    \
 949         struct lttng_channel *__chan = __event->chan;                         \
 950         struct lttng_session *__session = __chan->session;                    \
 951         struct lib_ring_buffer_ctx __ctx;                                     \
 952         ssize_t __event_len;                                                  \
 953         size_t __event_align;                                                 \
 954         size_t __orig_dynamic_len_offset, __dynamic_len_idx;                  \
 955         union {                                                               \
 956                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
 957                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
 958         } __stackvar;                                                         \
 959         int __ret;                                                            \
 960         struct probe_local_vars __tp_locvar;                                  \
 961         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
 962                         &__tp_locvar;                                         \
 963         struct lttng_pid_tracker *__lpf;                                      \
 964                                                                               \
 965         if (!_TP_SESSION_CHECK(session, __session))                           \
 966                 return;                                                       \
 967         if (unlikely(!ACCESS_ONCE(__session->active)))                        \
 968                 return;                                                       \
 969         if (unlikely(!ACCESS_ONCE(__chan->enabled)))                          \
 970                 return;                                                       \
 971         if (unlikely(!ACCESS_ONCE(__event->enabled)))                         \
 972                 return;                                                       \
 973         __lpf = lttng_rcu_dereference(__session->pid_tracker);                \
 974         if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid)))  \
 975                 return;                                                       \
 976         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
 977         __dynamic_len_idx = __orig_dynamic_len_offset;                        \
 978         _code_pre                                                             \
 979         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
 980                 struct lttng_bytecode_runtime *bc_runtime;                    \
 981                 int __filter_record = __event->has_enablers_without_bytecode; \
 982                                                                               \
 983                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
 984                                 tp_locvar, _args);                                    \
 985                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
 986                         if (unlikely(bc_runtime->filter(bc_runtime, &__lttng_probe_ctx,       \
 987                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) \
 988                                 __filter_record = 1;                          \
 989                 }                                                             \
 990                 if (likely(!__filter_record))                                 \
 991                         goto __post;                                          \
 992         }                                                                     \
 993         __event_len = __event_get_size__##_name(tp_locvar, _args);            \
 994         if (unlikely(__event_len < 0)) {                                      \
 995                 lib_ring_buffer_lost_event_too_big(__chan->chan);             \
 996                 goto __post;                                                  \
 997         }                                                                     \
 998         __event_align = __event_get_align__##_name(tp_locvar, _args);         \
 999         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
1000                                  __event_align, -1);                          \
1001         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1002         if (__ret < 0)                                                        \
1003                 goto __post;                                                  \
1004         _fields                                                               \
1005         __chan->ops->event_commit(&__ctx);                                    \
1006 __post:                                                                       \
1007         _code_post                                                            \
1008         barrier();      /* use before un-reserve. */                          \
1009         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1010         return;                                                               \
1011 }
1012
1013 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
1014 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
1015 static void __event_probe__##_name(void *__data)                              \
1016 {                                                                             \
1017         struct probe_local_vars { _locvar };                                  \
1018         struct lttng_event *__event = __data;                                 \
1019         struct lttng_probe_ctx __lttng_probe_ctx = {                                  \
1020                 .event = __event,                                             \
1021                 .interruptible = !irqs_disabled(),                            \
1022         };                                                                    \
1023         struct lttng_channel *__chan = __event->chan;                         \
1024         struct lttng_session *__session = __chan->session;                    \
1025         struct lib_ring_buffer_ctx __ctx;                                     \
1026         ssize_t __event_len;                                                  \
1027         size_t __event_align;                                                 \
1028         size_t __orig_dynamic_len_offset, __dynamic_len_idx;                  \
1029         union {                                                               \
1030                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
1031                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1032         } __stackvar;                                                         \
1033         int __ret;                                                            \
1034         struct probe_local_vars __tp_locvar;                                  \
1035         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
1036                         &__tp_locvar;                                         \
1037         struct lttng_pid_tracker *__lpf;                                      \
1038                                                                               \
1039         if (!_TP_SESSION_CHECK(session, __session))                           \
1040                 return;                                                       \
1041         if (unlikely(!ACCESS_ONCE(__session->active)))                        \
1042                 return;                                                       \
1043         if (unlikely(!ACCESS_ONCE(__chan->enabled)))                          \
1044                 return;                                                       \
1045         if (unlikely(!ACCESS_ONCE(__event->enabled)))                         \
1046                 return;                                                       \
1047         __lpf = lttng_rcu_dereference(__session->pid_tracker);                \
1048         if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid)))  \
1049                 return;                                                       \
1050         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
1051         __dynamic_len_idx = __orig_dynamic_len_offset;                        \
1052         _code_pre                                                             \
1053         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
1054                 struct lttng_bytecode_runtime *bc_runtime;                    \
1055                 int __filter_record = __event->has_enablers_without_bytecode; \
1056                                                                               \
1057                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
1058                                 tp_locvar);                                   \
1059                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
1060                         if (unlikely(bc_runtime->filter(bc_runtime, &__lttng_probe_ctx, \
1061                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) \
1062                                 __filter_record = 1;                          \
1063                 }                                                             \
1064                 if (likely(!__filter_record))                                 \
1065                         goto __post;                                          \
1066         }                                                                     \
1067         __event_len = __event_get_size__##_name(tp_locvar);                   \
1068         if (unlikely(__event_len < 0)) {                                      \
1069                 lib_ring_buffer_lost_event_too_big(__chan->chan);             \
1070                 goto __post;                                                  \
1071         }                                                                     \
1072         __event_align = __event_get_align__##_name(tp_locvar);                \
1073         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
1074                                  __event_align, -1);                          \
1075         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1076         if (__ret < 0)                                                        \
1077                 goto __post;                                                  \
1078         _fields                                                               \
1079         __chan->ops->event_commit(&__ctx);                                    \
1080 __post:                                                                       \
1081         _code_post                                                            \
1082         barrier();      /* use before un-reserve. */                          \
1083         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1084         return;                                                               \
1085 }
1086
1087 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1088
1089 #undef __get_dynamic_len
1090
1091 /*
1092  * Stage 7 of the trace events.
1093  *
1094  * Create event descriptions.
1095  */
1096
1097 /* Named field types must be defined in lttng-types.h */
1098
1099 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1100
1101 #ifndef TP_PROBE_CB
1102 #define TP_PROBE_CB(_template)  &__event_probe__##_template
1103 #endif
1104
1105 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1106 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)      \
1107 static const struct lttng_event_desc __event_desc___##_map = {          \
1108         .fields = __event_fields___##_template,                         \
1109         .name = #_map,                                                  \
1110         .kname = #_name,                                                \
1111         .probe_callback = (void *) TP_PROBE_CB(_template),              \
1112         .nr_fields = ARRAY_SIZE(__event_fields___##_template),          \
1113         .owner = THIS_MODULE,                                           \
1114 };
1115
1116 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1117 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1118         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1119
1120 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1121
1122 /*
1123  * Stage 8 of the trace events.
1124  *
1125  * Create an array of event description pointers.
1126  */
1127
1128 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1129
1130 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1131 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
1132                 &__event_desc___##_map,
1133
1134 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1135 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1136         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1137
1138 #define TP_ID1(_token, _system) _token##_system
1139 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1140
1141 static const struct lttng_event_desc *TP_ID(__event_desc___, TRACE_SYSTEM)[] = {
1142 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1143 };
1144
1145 #undef TP_ID1
1146 #undef TP_ID
1147
1148 /*
1149  * Stage 9 of the trace events.
1150  *
1151  * Create a toplevel descriptor for the whole probe.
1152  */
1153
1154 #define TP_ID1(_token, _system) _token##_system
1155 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1156
1157 /* non-const because list head will be modified when registered. */
1158 static __used struct lttng_probe_desc TP_ID(__probe_desc___, TRACE_SYSTEM) = {
1159         .provider = __stringify(TRACE_SYSTEM),
1160         .event_desc = TP_ID(__event_desc___, TRACE_SYSTEM),
1161         .nr_events = ARRAY_SIZE(TP_ID(__event_desc___, TRACE_SYSTEM)),
1162         .head = { NULL, NULL },
1163         .lazy_init_head = { NULL, NULL },
1164         .lazy = 0,
1165 };
1166
1167 #undef TP_ID1
1168 #undef TP_ID
1169
1170 /*
1171  * Stage 10 of the trace events.
1172  *
1173  * Register/unregister probes at module load/unload.
1174  */
1175
1176 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1177
1178 #define TP_ID1(_token, _system) _token##_system
1179 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1180 #define module_init_eval1(_token, _system)      module_init(_token##_system)
1181 #define module_init_eval(_token, _system)       module_init_eval1(_token, _system)
1182 #define module_exit_eval1(_token, _system)      module_exit(_token##_system)
1183 #define module_exit_eval(_token, _system)       module_exit_eval1(_token, _system)
1184
1185 #ifndef TP_MODULE_NOINIT
1186 static int TP_ID(__lttng_events_init__, TRACE_SYSTEM)(void)
1187 {
1188         wrapper_vmalloc_sync_all();
1189         return lttng_probe_register(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1190 }
1191
1192 static void TP_ID(__lttng_events_exit__, TRACE_SYSTEM)(void)
1193 {
1194         lttng_probe_unregister(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1195 }
1196
1197 #ifndef TP_MODULE_NOAUTOLOAD
1198 module_init_eval(__lttng_events_init__, TRACE_SYSTEM);
1199 module_exit_eval(__lttng_events_exit__, TRACE_SYSTEM);
1200 #endif
1201
1202 #endif
1203
1204 #undef module_init_eval
1205 #undef module_exit_eval
1206 #undef TP_ID1
1207 #undef TP_ID
1208
1209 #undef TP_PROTO
1210 #undef TP_ARGS