probes/lttng-tracepoint-event-impl.h

   1 /*
   2  * lttng-tracepoint-event-impl.h
   3  *
   4  * Copyright (C) 2009 Steven Rostedt <rostedt@goodmis.org>
   5  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; only
  10  * version 2.1 of the License.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20  */
  21
  22 #include <linux/uaccess.h>
  23 #include <linux/debugfs.h>
  24 #include <linux/rculist.h>
  25 #include <asm/byteorder.h>
  26 #include <linux/swab.h>
  27
  28 #include <probes/lttng.h>
  29 #include <probes/lttng-types.h>
  30 #include <probes/lttng-probe-user.h>
  31 #include <wrapper/vmalloc.h>    /* for wrapper_vmalloc_sync_all() */
  32 #include <wrapper/ringbuffer/frontend_types.h>
  33 #include <wrapper/ringbuffer/backend.h>
  34 #include <wrapper/rcu.h>
  35 #include <lttng-events.h>
  36 #include <lttng-tracer-core.h>
  37
  38 /*
  39  * Macro declarations used for all stages.
  40  */
  41
  42 /*
  43  * LTTng name mapping macros. LTTng remaps some of the kernel events to
  44  * enforce name-spacing.
  45  */
  46 #undef LTTNG_TRACEPOINT_EVENT_MAP
  47 #define LTTNG_TRACEPOINT_EVENT_MAP(name, map, proto, args, fields) \
  48         LTTNG_TRACEPOINT_EVENT_CLASS(map,                               \
  49                              PARAMS(proto),                             \
  50                              PARAMS(args),                              \
  51                              PARAMS(fields))                            \
  52         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  53
  54 #undef LTTNG_TRACEPOINT_EVENT_MAP_NOARGS
  55 #define LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, map, fields)            \
  56         LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(map,                        \
  57                              PARAMS(fields))                            \
  58         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(map, name, map)
  59
  60 #undef LTTNG_TRACEPOINT_EVENT_CODE_MAP
  61 #define LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, map, proto, args, _locvar, _code_pre, fields, _code_post) \
  62         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(map,                          \
  63                              PARAMS(proto),                             \
  64                              PARAMS(args),                              \
  65                              PARAMS(_locvar),                           \
  66                              PARAMS(_code_pre),                         \
  67                              PARAMS(fields),                            \
  68                              PARAMS(_code_post))                        \
  69         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  70
  71 #undef LTTNG_TRACEPOINT_EVENT_CODE
  72 #define LTTNG_TRACEPOINT_EVENT_CODE(name, proto, args, _locvar, _code_pre, fields, _code_post) \
  73         LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, name,                     \
  74                              PARAMS(proto),                             \
  75                              PARAMS(args),                              \
  76                              PARAMS(_locvar),                           \
  77                              PARAMS(_code_pre),                         \
  78                              PARAMS(fields),                            \
  79                              PARAMS(_code_post))
  80
  81 /*
  82  * LTTNG_TRACEPOINT_EVENT_CLASS can be used to add a generic function
  83  * handlers for events. That is, if all events have the same parameters
  84  * and just have distinct trace points.  Each tracepoint can be defined
  85  * with LTTNG_TRACEPOINT_EVENT_INSTANCE and that will map the
  86  * LTTNG_TRACEPOINT_EVENT_CLASS to the tracepoint.
  87  *
  88  * LTTNG_TRACEPOINT_EVENT is a one to one mapping between tracepoint and
  89  * template.
  90  */
  91
  92 #undef LTTNG_TRACEPOINT_EVENT
  93 #define LTTNG_TRACEPOINT_EVENT(name, proto, args, fields)               \
  94         LTTNG_TRACEPOINT_EVENT_MAP(name, name,                          \
  95                         PARAMS(proto),                                  \
  96                         PARAMS(args),                                   \
  97                         PARAMS(fields))
  98
  99 #undef LTTNG_TRACEPOINT_EVENT_NOARGS
 100 #define LTTNG_TRACEPOINT_EVENT_NOARGS(name, fields)                     \
 101         LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, name, PARAMS(fields))
 102
 103 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE
 104 #define LTTNG_TRACEPOINT_EVENT_INSTANCE(template, name, proto, args)    \
 105         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(template, name, name, PARAMS(proto), PARAMS(args))
 106
 107 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS
 108 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS(template, name)  \
 109         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(template, name, name)
 110
 111 #undef LTTNG_TRACEPOINT_EVENT_CLASS
 112 #define LTTNG_TRACEPOINT_EVENT_CLASS(_name, _proto, _args, _fields) \
 113         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(_proto), PARAMS(_args), , , \
 114                 PARAMS(_fields), )
 115
 116 #undef LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS
 117 #define LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(_name, _fields) \
 118         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, , , PARAMS(_fields), )
 119
 120
 121 /*
 122  * Stage 1 of the trace events.
 123  *
 124  * Create dummy trace calls for each events, verifying that the LTTng module
 125  * instrumentation headers match the kernel arguments. Will be optimized
 126  * out by the compiler.
 127  */
 128
 129 /* Reset all macros within TRACEPOINT_EVENT */
 130 #include <probes/lttng-events-reset.h>
 131
 132 #undef TP_PROTO
 133 #define TP_PROTO(...)   __VA_ARGS__
 134
 135 #undef TP_ARGS
 136 #define TP_ARGS(...)    __VA_ARGS__
 137
 138 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 139 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 140 void trace_##_name(_proto);
 141
 142 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 143 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 144 void trace_##_name(void);
 145
 146 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 147
 148 /*
 149  * Stage 1.1 of the trace events.
 150  *
 151  * Create dummy trace prototypes for each event class, and for each used
 152  * template. This will allow checking whether the prototypes from the
 153  * class and the instance using the class actually match.
 154  */
 155
 156 #include <probes/lttng-events-reset.h>  /* Reset all macros within TRACE_EVENT */
 157
 158 #undef TP_PROTO
 159 #define TP_PROTO(...)   __VA_ARGS__
 160
 161 #undef TP_ARGS
 162 #define TP_ARGS(...)    __VA_ARGS__
 163
 164 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 165 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 166 void __event_template_proto___##_template(_proto);
 167
 168 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 169 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 170 void __event_template_proto___##_template(void);
 171
 172 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 173 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 174 void __event_template_proto___##_name(_proto);
 175
 176 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 177 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 178 void __event_template_proto___##_name(void);
 179
 180 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 181
 182 /*
 183  * Stage 2 of the trace events.
 184  *
 185  * Create event field type metadata section.
 186  * Each event produce an array of fields.
 187  */
 188
 189 /* Reset all macros within TRACEPOINT_EVENT */
 190 #include <probes/lttng-events-reset.h>
 191 #include <probes/lttng-events-write.h>
 192 #include <probes/lttng-events-nowrite.h>
 193
 194 #undef _ctf_integer_ext
 195 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 196         {                                                       \
 197           .name = #_item,                                       \
 198           .type = __type_integer(_type, 0, 0, -1, _byte_order, _base, none),\
 199           .nowrite = _nowrite,                                  \
 200           .user = _user,                                        \
 201         },
 202
 203 #undef _ctf_array_encoded
 204 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 205         {                                                       \
 206           .name = #_item,                                       \
 207           .type =                                               \
 208                 {                                               \
 209                   .atype = atype_array,                         \
 210                   .u =                                          \
 211                         {                                       \
 212                           .array =                              \
 213                                 {                               \
 214                                   .elem_type = __type_integer(_type, 0, 0, 0, __BYTE_ORDER, 10, _encoding), \
 215                                   .length = _length,            \
 216                                 }                               \
 217                         }                                       \
 218                 },                                              \
 219           .nowrite = _nowrite,                                  \
 220           .user = _user,                                        \
 221         },
 222
 223 #undef _ctf_array_bitfield
 224 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 225         {                                                       \
 226           .name = #_item,                                       \
 227           .type =                                               \
 228                 {                                               \
 229                   .atype = atype_array,                         \
 230                   .u =                                          \
 231                         {                                       \
 232                           .array =                              \
 233                                 {                               \
 234                                   .elem_type = __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none), \
 235                                   .length = (_length) * sizeof(_type) * CHAR_BIT, \
 236                                   .elem_alignment = lttng_alignof(_type), \
 237                                 }                               \
 238                         }                                       \
 239                 },                                              \
 240           .nowrite = _nowrite,                                  \
 241           .user = _user,                                        \
 242         },
 243
 244
 245 #undef _ctf_sequence_encoded
 246 #define _ctf_sequence_encoded(_type, _item, _src,               \
 247                         _length_type, _src_length, _encoding,   \
 248                         _byte_order, _base, _user, _nowrite)    \
 249         {                                                       \
 250           .name = #_item,                                       \
 251           .type =                                               \
 252                 {                                               \
 253                   .atype = atype_sequence,                      \
 254                   .u =                                          \
 255                         {                                       \
 256                           .sequence =                           \
 257                                 {                               \
 258                                   .length_type = __type_integer(_length_type, 0, 0, 0, __BYTE_ORDER, 10, none), \
 259                                   .elem_type = __type_integer(_type, 0, 0, -1, _byte_order, _base, _encoding), \
 260                                 },                              \
 261                         },                                      \
 262                 },                                              \
 263           .nowrite = _nowrite,                                  \
 264           .user = _user,                                        \
 265         },
 266
 267 #undef _ctf_sequence_bitfield
 268 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 269                         _length_type, _src_length,              \
 270                         _user, _nowrite)                        \
 271         {                                                       \
 272           .name = #_item,                                       \
 273           .type =                                               \
 274                 {                                               \
 275                   .atype = atype_sequence,                      \
 276                   .u =                                          \
 277                         {                                       \
 278                           .sequence =                           \
 279                                 {                               \
 280                                   .length_type = __type_integer(_length_type, 0, 0, 0, __BYTE_ORDER, 10, none), \
 281                                   .elem_type = __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none), \
 282                                   .elem_alignment = lttng_alignof(_type), \
 283                                 },                              \
 284                         },                                      \
 285                 },                                              \
 286           .nowrite = _nowrite,                                  \
 287           .user = _user,                                        \
 288         },
 289
 290 #undef _ctf_string
 291 #define _ctf_string(_item, _src, _user, _nowrite)               \
 292         {                                                       \
 293           .name = #_item,                                       \
 294           .type =                                               \
 295                 {                                               \
 296                   .atype = atype_string,                        \
 297                   .u =                                          \
 298                         {                                       \
 299                           .basic = { .string = { .encoding = lttng_encode_UTF8 } } \
 300                         },                                      \
 301                 },                                              \
 302           .nowrite = _nowrite,                                  \
 303           .user = _user,                                        \
 304         },
 305
 306
 307 #undef ctf_custom_field
 308 #define ctf_custom_field(_type, _item, _code)                   \
 309         {                                                       \
 310                 .name = #_item,                                 \
 311                 .type = { _type },                              \
 312                 .nowrite = 0,                                   \
 313                 .user = 0,                                      \
 314         },
 315
 316 #undef ctf_custom_type
 317 #define ctf_custom_type(...)    __VA_ARGS__
 318
 319 #undef TP_FIELDS
 320 #define TP_FIELDS(...)  __VA_ARGS__     /* Only one used in this phase */
 321
 322 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 323 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 324         static const struct lttng_event_field __event_fields___##_name[] = { \
 325                 _fields                                                      \
 326         };
 327
 328 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 329 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 330         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, PARAMS(_fields), _code_post)
 331
 332 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 333
 334 /*
 335  * Stage 3 of the trace events.
 336  *
 337  * Create probe callback prototypes.
 338  */
 339
 340 /* Reset all macros within TRACEPOINT_EVENT */
 341 #include <probes/lttng-events-reset.h>
 342
 343 #undef TP_PROTO
 344 #define TP_PROTO(...)   __VA_ARGS__
 345
 346 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 347 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 348 static void __event_probe__##_name(void *__data, _proto);
 349
 350 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 351 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 352 static void __event_probe__##_name(void *__data);
 353
 354 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 355
 356 /*
 357  * Stage 4 of the trace events.
 358  *
 359  * Create static inline function that calculates event size.
 360  */
 361
 362 /* Reset all macros within TRACEPOINT_EVENT */
 363 #include <probes/lttng-events-reset.h>
 364 #include <probes/lttng-events-write.h>
 365
 366 #undef _ctf_integer_ext
 367 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 368         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 369         __event_len += sizeof(_type);
 370
 371 #undef _ctf_array_encoded
 372 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 373         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 374         __event_len += sizeof(_type) * (_length);
 375
 376 #undef _ctf_array_bitfield
 377 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 378         _ctf_array_encoded(_type, _item, _src, _length, none, _user, _nowrite)
 379
 380 #undef _ctf_sequence_encoded
 381 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 382                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 383         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_length_type)); \
 384         __event_len += sizeof(_length_type);                                   \
 385         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 386         {                                                                               \
 387                 size_t __seqlen = (_src_length);                                        \
 388                                                                                         \
 389                 if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 390                         goto error;                                                     \
 391                 barrier();      /* reserve before use. */                               \
 392                 this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = __seqlen; \
 393                 __event_len += sizeof(_type) * __seqlen;                                \
 394         }
 395
 396 #undef _ctf_sequence_bitfield
 397 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 398                         _length_type, _src_length,              \
 399                         _user, _nowrite)                        \
 400         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 401                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 402
 403 /*
 404  * ctf_user_string includes \0. If returns 0, it faulted, so we set size to
 405  * 1 (\0 only).
 406  */
 407 #undef _ctf_string
 408 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 409         if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 410                 goto error;                                                    \
 411         barrier();      /* reserve before use. */                              \
 412         if (_user) {                                                           \
 413                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 414                         max_t(size_t, lttng_strlen_user_inatomic(_src), 1);    \
 415         } else {                                                               \
 416                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 417                         strlen(_src) + 1;                                      \
 418         }
 419
 420 #undef ctf_align
 421 #define ctf_align(_type)                                                \
 422         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type));
 423
 424 #undef ctf_custom_field
 425 #define ctf_custom_field(_type, _item, _code)                           \
 426         {                                                               \
 427                 _code                                                   \
 428         }
 429
 430 #undef ctf_custom_code
 431 #define ctf_custom_code(...)            __VA_ARGS__
 432
 433 #undef TP_PROTO
 434 #define TP_PROTO(...)   __VA_ARGS__
 435
 436 #undef TP_FIELDS
 437 #define TP_FIELDS(...)  __VA_ARGS__
 438
 439 #undef TP_locvar
 440 #define TP_locvar(...)  __VA_ARGS__
 441
 442 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 443 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 444 static inline ssize_t __event_get_size__##_name(void *__tp_locvar, _proto)     \
 445 {                                                                             \
 446         size_t __event_len = 0;                                               \
 447         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 448         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 449                                                                               \
 450         _fields                                                               \
 451         return __event_len;                                                   \
 452                                                                               \
 453 error:                                                                        \
 454         __attribute__((unused));                                              \
 455         return -1;                                                            \
 456 }
 457
 458 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 459 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 460 static inline ssize_t __event_get_size__##_name(void *__tp_locvar)            \
 461 {                                                                             \
 462         size_t __event_len = 0;                                               \
 463         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 464         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 465                                                                               \
 466         _fields                                                               \
 467         return __event_len;                                                   \
 468                                                                               \
 469 error:                                                                        \
 470         __attribute__((unused));                                              \
 471         return -1;                                                            \
 472 }
 473
 474 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 475
 476
 477 /*
 478  * Stage 4.1 of tracepoint event generation.
 479  *
 480  * Create static inline function that layout the filter stack data.
 481  * We make both write and nowrite data available to the filter.
 482  */
 483
 484 /* Reset all macros within TRACEPOINT_EVENT */
 485 #include <probes/lttng-events-reset.h>
 486 #include <probes/lttng-events-write.h>
 487 #include <probes/lttng-events-nowrite.h>
 488
 489 #undef _ctf_integer_ext_fetched
 490 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 491         if (lttng_is_signed_type(_type)) {                                     \
 492                 int64_t __ctf_tmp_int64;                                       \
 493                 switch (sizeof(_type)) {                                       \
 494                 case 1:                                                        \
 495                 {                                                              \
 496                         union { _type t; int8_t v; } __tmp = { (_type) (_src) }; \
 497                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 498                         break;                                                 \
 499                 }                                                              \
 500                 case 2:                                                        \
 501                 {                                                              \
 502                         union { _type t; int16_t v; } __tmp = { (_type) (_src) }; \
 503                         if (_byte_order != __BYTE_ORDER)                       \
 504                                 __swab16s(&__tmp.v);                           \
 505                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 506                         break;                                                 \
 507                 }                                                              \
 508                 case 4:                                                        \
 509                 {                                                              \
 510                         union { _type t; int32_t v; } __tmp = { (_type) (_src) }; \
 511                         if (_byte_order != __BYTE_ORDER)                       \
 512                                 __swab32s(&__tmp.v);                           \
 513                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 514                         break;                                                 \
 515                 }                                                              \
 516                 case 8:                                                        \
 517                 {                                                              \
 518                         union { _type t; int64_t v; } __tmp = { (_type) (_src) }; \
 519                         if (_byte_order != __BYTE_ORDER)                       \
 520                                 __swab64s(&__tmp.v);                           \
 521                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 522                         break;                                                 \
 523                 }                                                              \
 524                 default:                                                       \
 525                         BUG_ON(1);                                             \
 526                 };                                                             \
 527                 memcpy(__stack_data, &__ctf_tmp_int64, sizeof(int64_t));       \
 528         } else {                                                               \
 529                 uint64_t __ctf_tmp_uint64;                                     \
 530                 switch (sizeof(_type)) {                                       \
 531                 case 1:                                                        \
 532                 {                                                              \
 533                         union { _type t; uint8_t v; } __tmp = { (_type) (_src) }; \
 534                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 535                         break;                                                 \
 536                 }                                                              \
 537                 case 2:                                                        \
 538                 {                                                              \
 539                         union { _type t; uint16_t v; } __tmp = { (_type) (_src) }; \
 540                         if (_byte_order != __BYTE_ORDER)                       \
 541                                 __swab16s(&__tmp.v);                           \
 542                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 543                         break;                                                 \
 544                 }                                                              \
 545                 case 4:                                                        \
 546                 {                                                              \
 547                         union { _type t; uint32_t v; } __tmp = { (_type) (_src) }; \
 548                         if (_byte_order != __BYTE_ORDER)                       \
 549                                 __swab32s(&__tmp.v);                           \
 550                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 551                         break;                                                 \
 552                 }                                                              \
 553                 case 8:                                                        \
 554                 {                                                              \
 555                         union { _type t; uint64_t v; } __tmp = { (_type) (_src) }; \
 556                         if (_byte_order != __BYTE_ORDER)                       \
 557                                 __swab64s(&__tmp.v);                           \
 558                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 559                         break;                                                 \
 560                 }                                                              \
 561                 default:                                                       \
 562                         BUG_ON(1);                                             \
 563                 };                                                             \
 564                 memcpy(__stack_data, &__ctf_tmp_uint64, sizeof(uint64_t));     \
 565         }                                                                      \
 566         __stack_data += sizeof(int64_t);
 567
 568 #undef _ctf_integer_ext_isuser0
 569 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 570         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 571
 572 #undef _ctf_integer_ext_isuser1
 573 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 574 {                                                                                       \
 575         union {                                                                         \
 576                 char __array[sizeof(_user_src)];                                        \
 577                 __typeof__(_user_src) __v;                                              \
 578         } __tmp_fetch;                                                                  \
 579         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 580                                 &(_user_src), sizeof(_user_src)))                       \
 581                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 582         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 583 }
 584
 585 #undef _ctf_integer_ext
 586 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 587         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 588
 589 #undef _ctf_array_encoded
 590 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 591         {                                                                      \
 592                 unsigned long __ctf_tmp_ulong = (unsigned long) (_length);     \
 593                 const void *__ctf_tmp_ptr = (_src);                            \
 594                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 595                 __stack_data += sizeof(unsigned long);                         \
 596                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 597                 __stack_data += sizeof(void *);                                \
 598         }
 599
 600 #undef _ctf_array_bitfield
 601 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 602         _ctf_array_encoded(_type, _item, _src, _length, none, _user, _nowrite)
 603
 604 #undef _ctf_sequence_encoded
 605 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                \
 606                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 607         {                                                                      \
 608                 unsigned long __ctf_tmp_ulong = (unsigned long) (_src_length); \
 609                 const void *__ctf_tmp_ptr = (_src);                            \
 610                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 611                 __stack_data += sizeof(unsigned long);                         \
 612                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 613                 __stack_data += sizeof(void *);                                \
 614         }
 615
 616 #undef _ctf_sequence_bitfield
 617 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 618                         _length_type, _src_length,              \
 619                         _user, _nowrite)                        \
 620         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 621                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 622
 623 #undef _ctf_string
 624 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 625         {                                                                      \
 626                 const void *__ctf_tmp_ptr = (_src);                            \
 627                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 628                 __stack_data += sizeof(void *);                                \
 629         }
 630
 631 #undef TP_PROTO
 632 #define TP_PROTO(...) __VA_ARGS__
 633
 634 #undef TP_FIELDS
 635 #define TP_FIELDS(...) __VA_ARGS__
 636
 637 #undef TP_locvar
 638 #define TP_locvar(...)  __VA_ARGS__
 639
 640 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 641 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 642 static inline                                                                 \
 643 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 644                 void *__tp_locvar)                                            \
 645 {                                                                             \
 646         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 647                                                                               \
 648         _fields                                                               \
 649 }
 650
 651 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 652 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 653 static inline                                                                 \
 654 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 655                 void *__tp_locvar, _proto)                                    \
 656 {                                                                             \
 657         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 658                                                                               \
 659         _fields                                                               \
 660 }
 661
 662 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 663
 664 /*
 665  * Stage 5 of the trace events.
 666  *
 667  * Create static inline function that calculates event payload alignment.
 668  */
 669
 670 /* Reset all macros within TRACEPOINT_EVENT */
 671 #include <probes/lttng-events-reset.h>
 672 #include <probes/lttng-events-write.h>
 673
 674 #undef _ctf_integer_ext
 675 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 676         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 677
 678 #undef _ctf_array_encoded
 679 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 680         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 681
 682 #undef _ctf_array_bitfield
 683 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 684         _ctf_array_encoded(_type, _item, _src, _length, none, _user, _nowrite)
 685
 686 #undef _ctf_sequence_encoded
 687 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 688                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 689         __event_align = max_t(size_t, __event_align, lttng_alignof(_length_type)); \
 690         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 691
 692 #undef _ctf_sequence_bitfield
 693 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 694                         _length_type, _src_length,              \
 695                         _user, _nowrite)                        \
 696         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 697                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 698
 699 #undef _ctf_string
 700 #define _ctf_string(_item, _src, _user, _nowrite)
 701
 702 #undef ctf_align
 703 #define ctf_align(_type)                                                \
 704         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 705
 706 #undef TP_PROTO
 707 #define TP_PROTO(...)   __VA_ARGS__
 708
 709 #undef TP_FIELDS
 710 #define TP_FIELDS(...)  __VA_ARGS__
 711
 712 #undef TP_locvar
 713 #define TP_locvar(...)  __VA_ARGS__
 714
 715 #undef ctf_custom_field
 716 #define ctf_custom_field(_type, _item, _code)           _code
 717
 718 #undef ctf_custom_code
 719 #define ctf_custom_code(...)                                            \
 720         {                                                               \
 721                 __VA_ARGS__                                             \
 722         }
 723
 724 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 725 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 726 static inline size_t __event_get_align__##_name(void *__tp_locvar, _proto)    \
 727 {                                                                             \
 728         size_t __event_align = 1;                                             \
 729         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 730                                                                               \
 731         _fields                                                               \
 732         return __event_align;                                                 \
 733 }
 734
 735 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 736 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 737 static inline size_t __event_get_align__##_name(void *__tp_locvar)            \
 738 {                                                                             \
 739         size_t __event_align = 1;                                             \
 740         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 741                                                                               \
 742         _fields                                                               \
 743         return __event_align;                                                 \
 744 }
 745
 746 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 747
 748 /*
 749  * Stage 6 of tracepoint event generation.
 750  *
 751  * Create the probe function. This function calls event size calculation
 752  * and writes event data into the buffer.
 753  */
 754
 755 /* Reset all macros within TRACEPOINT_EVENT */
 756 #include <probes/lttng-events-reset.h>
 757 #include <probes/lttng-events-write.h>
 758
 759 #undef _ctf_integer_ext_fetched
 760 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 761         {                                                               \
 762                 _type __tmp = _src;                                     \
 763                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(__tmp));\
 764                 __chan->ops->event_write(&__ctx, &__tmp, sizeof(__tmp));\
 765         }
 766
 767 #undef _ctf_integer_ext_isuser0
 768 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 769         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 770
 771 #undef _ctf_integer_ext_isuser1
 772 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 773 {                                                                              \
 774         union {                                                                         \
 775                 char __array[sizeof(_user_src)];                                        \
 776                 __typeof__(_user_src) __v;                                              \
 777         } __tmp_fetch;                                                                  \
 778         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 779                                 &(_user_src), sizeof(_user_src)))                       \
 780                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 781         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 782 }
 783
 784 #undef _ctf_integer_ext
 785 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 786         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 787
 788 #undef _ctf_array_encoded
 789 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 790         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 791         if (_user) {                                                    \
 792                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 793         } else {                                                        \
 794                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 795         }
 796
 797 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 798 #undef _ctf_array_bitfield
 799 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 800         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 801         if (_user) {                                                    \
 802                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 803         } else {                                                        \
 804                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 805         }
 806 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 807 /*
 808  * For big endian, we need to byteswap into little endian.
 809  */
 810 #undef _ctf_array_bitfield
 811 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 812         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 813         {                                                               \
 814                 size_t _i;                                              \
 815                                                                         \
 816                 for (_i = 0; _i < (_length); _i++) {                    \
 817                         _type _tmp;                                     \
 818                                                                         \
 819                         if (_user) {                                    \
 820                                 if (get_user(_tmp, (_type *) _src + _i)) \
 821                                         _tmp = 0;                       \
 822                         } else {                                        \
 823                                 _tmp = ((_type *) _src)[_i];            \
 824                         }                                               \
 825                         switch (sizeof(_type)) {                        \
 826                         case 1:                                         \
 827                                 break;                                  \
 828                         case 2:                                         \
 829                                 _tmp = cpu_to_le16(_tmp);               \
 830                                 break;                                  \
 831                         case 4:                                         \
 832                                 _tmp = cpu_to_le32(_tmp);               \
 833                                 break;                                  \
 834                         case 8:                                         \
 835                                 _tmp = cpu_to_le64(_tmp);               \
 836                                 break;                                  \
 837                         default:                                        \
 838                                 BUG_ON(1);                              \
 839                         }                                               \
 840                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
 841                 }                                                       \
 842         }
 843 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 844
 845 #undef _ctf_sequence_encoded
 846 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,         \
 847                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 848         {                                                               \
 849                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx]; \
 850                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 851                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 852         }                                                               \
 853         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 854         if (_user) {                                                    \
 855                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 856                         sizeof(_type) * __get_dynamic_len(dest));       \
 857         } else {                                                        \
 858                 __chan->ops->event_write(&__ctx, _src,                  \
 859                         sizeof(_type) * __get_dynamic_len(dest));       \
 860         }
 861
 862 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 863 #undef _ctf_sequence_bitfield
 864 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 865                         _length_type, _src_length,              \
 866                         _user, _nowrite)                        \
 867         {                                                               \
 868                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 869                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 870                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 871         }                                                               \
 872         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 873         if (_user) {                                                    \
 874                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 875                         sizeof(_type) * __get_dynamic_len(dest));       \
 876         } else {                                                        \
 877                 __chan->ops->event_write(&__ctx, _src,                  \
 878                         sizeof(_type) * __get_dynamic_len(dest));       \
 879         }
 880 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 881 /*
 882  * For big endian, we need to byteswap into little endian.
 883  */
 884 #undef _ctf_sequence_bitfield
 885 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 886                         _length_type, _src_length,              \
 887                         _user, _nowrite)                        \
 888         {                                                       \
 889                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 890                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 891                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 892         }                                                               \
 893         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 894         {                                                               \
 895                 size_t _i, _length;                                     \
 896                                                                         \
 897                 _length = __get_dynamic_len(dest);                      \
 898                 for (_i = 0; _i < _length; _i++) {                      \
 899                         _type _tmp;                                     \
 900                                                                         \
 901                         if (_user) {                                    \
 902                                 if (get_user(_tmp, (_type *) _src + _i)) \
 903                                         _tmp = 0;                       \
 904                         } else {                                        \
 905                                 _tmp = ((_type *) _src)[_i];            \
 906                         }                                               \
 907                         switch (sizeof(_type)) {                        \
 908                         case 1:                                         \
 909                                 break;                                  \
 910                         case 2:                                         \
 911                                 _tmp = cpu_to_le16(_tmp);               \
 912                                 break;                                  \
 913                         case 4:                                         \
 914                                 _tmp = cpu_to_le32(_tmp);               \
 915                                 break;                                  \
 916                         case 8:                                         \
 917                                 _tmp = cpu_to_le64(_tmp);               \
 918                                 break;                                  \
 919                         default:                                        \
 920                                 BUG_ON(1);                              \
 921                         }                                               \
 922                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
 923                 }                                                       \
 924         }
 925 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 926
 927 #undef _ctf_string
 928 #define _ctf_string(_item, _src, _user, _nowrite)                       \
 929         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(*(_src)));      \
 930         if (_user) {                                                    \
 931                 __chan->ops->event_strcpy_from_user(&__ctx, _src,       \
 932                         __get_dynamic_len(dest));                       \
 933         } else {                                                        \
 934                 __chan->ops->event_strcpy(&__ctx, _src,                 \
 935                         __get_dynamic_len(dest));                       \
 936         }
 937
 938
 939 #undef ctf_align
 940 #define ctf_align(_type)                                                \
 941         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));
 942
 943 #undef ctf_custom_field
 944 #define ctf_custom_field(_type, _item, _code)           _code
 945
 946 #undef ctf_custom_code
 947 #define ctf_custom_code(...)                                            \
 948         {                                                               \
 949                 __VA_ARGS__                                             \
 950         }
 951
 952 /* Beware: this get len actually consumes the len value */
 953 #undef __get_dynamic_len
 954 #define __get_dynamic_len(field)        this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx++]
 955
 956 #undef TP_PROTO
 957 #define TP_PROTO(...)   __VA_ARGS__
 958
 959 #undef TP_ARGS
 960 #define TP_ARGS(...)    __VA_ARGS__
 961
 962 #undef TP_FIELDS
 963 #define TP_FIELDS(...)  __VA_ARGS__
 964
 965 #undef TP_locvar
 966 #define TP_locvar(...)  __VA_ARGS__
 967
 968 #undef TP_code_pre
 969 #define TP_code_pre(...)        __VA_ARGS__
 970
 971 #undef TP_code_post
 972 #define TP_code_post(...)       __VA_ARGS__
 973
 974 /*
 975  * For state dump, check that "session" argument (mandatory) matches the
 976  * session this event belongs to. Ensures that we write state dump data only
 977  * into the started session, not into all sessions.
 978  */
 979 #ifdef TP_SESSION_CHECK
 980 #define _TP_SESSION_CHECK(session, csession)    (session == csession)
 981 #else /* TP_SESSION_CHECK */
 982 #define _TP_SESSION_CHECK(session, csession)    1
 983 #endif /* TP_SESSION_CHECK */
 984
 985 /*
 986  * Using twice size for filter stack data to hold size and pointer for
 987  * each field (worse case). For integers, max size required is 64-bit.
 988  * Same for double-precision floats. Those fit within
 989  * 2*sizeof(unsigned long) for all supported architectures.
 990  * Perform UNION (||) of filter runtime list.
 991  */
 992 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 993 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 994 static void __event_probe__##_name(void *__data, _proto)                      \
 995 {                                                                             \
 996         struct probe_local_vars { _locvar };                                  \
 997         struct lttng_event *__event = __data;                                 \
 998         struct lttng_probe_ctx __lttng_probe_ctx = {                                  \
 999                 .event = __event,                                             \
1000                 .interruptible = !irqs_disabled(),                            \
1001         };                                                                    \
1002         struct lttng_channel *__chan = __event->chan;                         \
1003         struct lttng_session *__session = __chan->session;                    \
1004         struct lib_ring_buffer_ctx __ctx;                                     \
1005         ssize_t __event_len;                                                  \
1006         size_t __event_align;                                                 \
1007         size_t __orig_dynamic_len_offset, __dynamic_len_idx;                  \
1008         union {                                                               \
1009                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
1010                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1011         } __stackvar;                                                         \
1012         int __ret;                                                            \
1013         struct probe_local_vars __tp_locvar;                                  \
1014         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
1015                         &__tp_locvar;                                         \
1016         struct lttng_pid_tracker *__lpf;                                      \
1017                                                                               \
1018         if (!_TP_SESSION_CHECK(session, __session))                           \
1019                 return;                                                       \
1020         if (unlikely(!ACCESS_ONCE(__session->active)))                        \
1021                 return;                                                       \
1022         if (unlikely(!ACCESS_ONCE(__chan->enabled)))                          \
1023                 return;                                                       \
1024         if (unlikely(!ACCESS_ONCE(__event->enabled)))                         \
1025                 return;                                                       \
1026         __lpf = lttng_rcu_dereference(__session->pid_tracker);                \
1027         if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid)))  \
1028                 return;                                                       \
1029         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
1030         __dynamic_len_idx = __orig_dynamic_len_offset;                        \
1031         _code_pre                                                             \
1032         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
1033                 struct lttng_bytecode_runtime *bc_runtime;                    \
1034                 int __filter_record = __event->has_enablers_without_bytecode; \
1035                                                                               \
1036                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
1037                                 tp_locvar, _args);                                    \
1038                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
1039                         if (unlikely(bc_runtime->filter(bc_runtime, &__lttng_probe_ctx,       \
1040                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) \
1041                                 __filter_record = 1;                          \
1042                 }                                                             \
1043                 if (likely(!__filter_record))                                 \
1044                         goto __post;                                          \
1045         }                                                                     \
1046         __event_len = __event_get_size__##_name(tp_locvar, _args);            \
1047         if (unlikely(__event_len < 0)) {                                      \
1048                 lib_ring_buffer_lost_event_too_big(__chan->chan);             \
1049                 goto __post;                                                  \
1050         }                                                                     \
1051         __event_align = __event_get_align__##_name(tp_locvar, _args);         \
1052         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
1053                                  __event_align, -1);                          \
1054         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1055         if (__ret < 0)                                                        \
1056                 goto __post;                                                  \
1057         _fields                                                               \
1058         __chan->ops->event_commit(&__ctx);                                    \
1059 __post:                                                                       \
1060         _code_post                                                            \
1061         barrier();      /* use before un-reserve. */                          \
1062         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1063         return;                                                               \
1064 }
1065
1066 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
1067 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
1068 static void __event_probe__##_name(void *__data)                              \
1069 {                                                                             \
1070         struct probe_local_vars { _locvar };                                  \
1071         struct lttng_event *__event = __data;                                 \
1072         struct lttng_probe_ctx __lttng_probe_ctx = {                                  \
1073                 .event = __event,                                             \
1074                 .interruptible = !irqs_disabled(),                            \
1075         };                                                                    \
1076         struct lttng_channel *__chan = __event->chan;                         \
1077         struct lttng_session *__session = __chan->session;                    \
1078         struct lib_ring_buffer_ctx __ctx;                                     \
1079         ssize_t __event_len;                                                  \
1080         size_t __event_align;                                                 \
1081         size_t __orig_dynamic_len_offset, __dynamic_len_idx;                  \
1082         union {                                                               \
1083                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
1084                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1085         } __stackvar;                                                         \
1086         int __ret;                                                            \
1087         struct probe_local_vars __tp_locvar;                                  \
1088         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
1089                         &__tp_locvar;                                         \
1090         struct lttng_pid_tracker *__lpf;                                      \
1091                                                                               \
1092         if (!_TP_SESSION_CHECK(session, __session))                           \
1093                 return;                                                       \
1094         if (unlikely(!ACCESS_ONCE(__session->active)))                        \
1095                 return;                                                       \
1096         if (unlikely(!ACCESS_ONCE(__chan->enabled)))                          \
1097                 return;                                                       \
1098         if (unlikely(!ACCESS_ONCE(__event->enabled)))                         \
1099                 return;                                                       \
1100         __lpf = lttng_rcu_dereference(__session->pid_tracker);                \
1101         if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid)))  \
1102                 return;                                                       \
1103         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
1104         __dynamic_len_idx = __orig_dynamic_len_offset;                        \
1105         _code_pre                                                             \
1106         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
1107                 struct lttng_bytecode_runtime *bc_runtime;                    \
1108                 int __filter_record = __event->has_enablers_without_bytecode; \
1109                                                                               \
1110                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
1111                                 tp_locvar);                                   \
1112                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
1113                         if (unlikely(bc_runtime->filter(bc_runtime, &__lttng_probe_ctx, \
1114                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) \
1115                                 __filter_record = 1;                          \
1116                 }                                                             \
1117                 if (likely(!__filter_record))                                 \
1118                         goto __post;                                          \
1119         }                                                                     \
1120         __event_len = __event_get_size__##_name(tp_locvar);                   \
1121         if (unlikely(__event_len < 0)) {                                      \
1122                 lib_ring_buffer_lost_event_too_big(__chan->chan);             \
1123                 goto __post;                                                  \
1124         }                                                                     \
1125         __event_align = __event_get_align__##_name(tp_locvar);                \
1126         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
1127                                  __event_align, -1);                          \
1128         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1129         if (__ret < 0)                                                        \
1130                 goto __post;                                                  \
1131         _fields                                                               \
1132         __chan->ops->event_commit(&__ctx);                                    \
1133 __post:                                                                       \
1134         _code_post                                                            \
1135         barrier();      /* use before un-reserve. */                          \
1136         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1137         return;                                                               \
1138 }
1139
1140 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1141
1142 #undef __get_dynamic_len
1143
1144 /*
1145  * Stage 7 of the trace events.
1146  *
1147  * Create event descriptions.
1148  */
1149
1150 /* Named field types must be defined in lttng-types.h */
1151
1152 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1153
1154 #ifndef TP_PROBE_CB
1155 #define TP_PROBE_CB(_template)  &__event_probe__##_template
1156 #endif
1157
1158 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1159 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)      \
1160 static const struct lttng_event_desc __event_desc___##_map = {          \
1161         .fields = __event_fields___##_template,                         \
1162         .name = #_map,                                                  \
1163         .kname = #_name,                                                \
1164         .probe_callback = (void *) TP_PROBE_CB(_template),              \
1165         .nr_fields = ARRAY_SIZE(__event_fields___##_template),          \
1166         .owner = THIS_MODULE,                                           \
1167 };
1168
1169 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1170 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1171         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1172
1173 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1174
1175 /*
1176  * Stage 8 of the trace events.
1177  *
1178  * Create an array of event description pointers.
1179  */
1180
1181 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1182
1183 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1184 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
1185                 &__event_desc___##_map,
1186
1187 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1188 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1189         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1190
1191 #define TP_ID1(_token, _system) _token##_system
1192 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1193
1194 static const struct lttng_event_desc *TP_ID(__event_desc___, TRACE_SYSTEM)[] = {
1195 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1196 };
1197
1198 #undef TP_ID1
1199 #undef TP_ID
1200
1201 /*
1202  * Stage 9 of the trace events.
1203  *
1204  * Create a toplevel descriptor for the whole probe.
1205  */
1206
1207 #define TP_ID1(_token, _system) _token##_system
1208 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1209
1210 /* non-const because list head will be modified when registered. */
1211 static __used struct lttng_probe_desc TP_ID(__probe_desc___, TRACE_SYSTEM) = {
1212         .provider = __stringify(TRACE_SYSTEM),
1213         .event_desc = TP_ID(__event_desc___, TRACE_SYSTEM),
1214         .nr_events = ARRAY_SIZE(TP_ID(__event_desc___, TRACE_SYSTEM)),
1215         .head = { NULL, NULL },
1216         .lazy_init_head = { NULL, NULL },
1217         .lazy = 0,
1218 };
1219
1220 #undef TP_ID1
1221 #undef TP_ID
1222
1223 /*
1224  * Stage 10 of the trace events.
1225  *
1226  * Register/unregister probes at module load/unload.
1227  */
1228
1229 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1230
1231 #define TP_ID1(_token, _system) _token##_system
1232 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1233 #define module_init_eval1(_token, _system)      module_init(_token##_system)
1234 #define module_init_eval(_token, _system)       module_init_eval1(_token, _system)
1235 #define module_exit_eval1(_token, _system)      module_exit(_token##_system)
1236 #define module_exit_eval(_token, _system)       module_exit_eval1(_token, _system)
1237
1238 #ifndef TP_MODULE_NOINIT
1239 static int TP_ID(__lttng_events_init__, TRACE_SYSTEM)(void)
1240 {
1241         wrapper_vmalloc_sync_all();
1242         return lttng_probe_register(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1243 }
1244
1245 static void TP_ID(__lttng_events_exit__, TRACE_SYSTEM)(void)
1246 {
1247         lttng_probe_unregister(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1248 }
1249
1250 #ifndef TP_MODULE_NOAUTOLOAD
1251 module_init_eval(__lttng_events_init__, TRACE_SYSTEM);
1252 module_exit_eval(__lttng_events_exit__, TRACE_SYSTEM);
1253 #endif
1254
1255 #endif
1256
1257 #undef module_init_eval
1258 #undef module_exit_eval
1259 #undef TP_ID1
1260 #undef TP_ID
1261
1262 #undef TP_PROTO
1263 #undef TP_ARGS