probes/lttng-tracepoint-event-impl.h

   1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
   2  *
   3  * lttng-tracepoint-event-impl.h
   4  *
   5  * Copyright (C) 2009 Steven Rostedt <rostedt@goodmis.org>
   6  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   7  */
   8
   9 #include <linux/uaccess.h>
  10 #include <linux/debugfs.h>
  11 #include <linux/rculist.h>
  12 #include <asm/byteorder.h>
  13 #include <linux/swab.h>
  14
  15 #include <probes/lttng.h>
  16 #include <probes/lttng-types.h>
  17 #include <probes/lttng-probe-user.h>
  18 #include <include/ringbuffer/frontend_types.h>
  19 #include <include/ringbuffer/backend.h>
  20 #include <wrapper/rcu.h>
  21 #include <wrapper/user_namespace.h>
  22 #include <lttng-events.h>
  23 #include <lttng-tracer-core.h>
  24 #include <lttng-tp-mempool.h>
  25
  26 #define __LTTNG_NULL_STRING     "(null)"
  27
  28 /*
  29  * Macro declarations used for all stages.
  30  */
  31
  32 /*
  33  * LTTng name mapping macros. LTTng remaps some of the kernel events to
  34  * enforce name-spacing.
  35  */
  36 #undef LTTNG_TRACEPOINT_EVENT_MAP
  37 #define LTTNG_TRACEPOINT_EVENT_MAP(name, map, proto, args, fields) \
  38         LTTNG_TRACEPOINT_EVENT_CLASS(map,                               \
  39                              PARAMS(proto),                             \
  40                              PARAMS(args),                              \
  41                              PARAMS(fields))                            \
  42         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  43
  44 #undef LTTNG_TRACEPOINT_EVENT_MAP_NOARGS
  45 #define LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, map, fields)            \
  46         LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(map,                        \
  47                              PARAMS(fields))                            \
  48         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(map, name, map)
  49
  50 #undef LTTNG_TRACEPOINT_EVENT_CODE_MAP
  51 #define LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, map, proto, args, _locvar, _code_pre, fields, _code_post) \
  52         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(map,                          \
  53                              PARAMS(proto),                             \
  54                              PARAMS(args),                              \
  55                              PARAMS(_locvar),                           \
  56                              PARAMS(_code_pre),                         \
  57                              PARAMS(fields),                            \
  58                              PARAMS(_code_post))                        \
  59         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  60
  61 #undef LTTNG_TRACEPOINT_EVENT_CODE
  62 #define LTTNG_TRACEPOINT_EVENT_CODE(name, proto, args, _locvar, _code_pre, fields, _code_post) \
  63         LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, name,                     \
  64                              PARAMS(proto),                             \
  65                              PARAMS(args),                              \
  66                              PARAMS(_locvar),                           \
  67                              PARAMS(_code_pre),                         \
  68                              PARAMS(fields),                            \
  69                              PARAMS(_code_post))
  70
  71 /*
  72  * LTTNG_TRACEPOINT_EVENT_CLASS can be used to add a generic function
  73  * handlers for events. That is, if all events have the same parameters
  74  * and just have distinct trace points.  Each tracepoint can be defined
  75  * with LTTNG_TRACEPOINT_EVENT_INSTANCE and that will map the
  76  * LTTNG_TRACEPOINT_EVENT_CLASS to the tracepoint.
  77  *
  78  * LTTNG_TRACEPOINT_EVENT is a one to one mapping between tracepoint and
  79  * template.
  80  */
  81
  82 #undef LTTNG_TRACEPOINT_EVENT
  83 #define LTTNG_TRACEPOINT_EVENT(name, proto, args, fields)               \
  84         LTTNG_TRACEPOINT_EVENT_MAP(name, name,                          \
  85                         PARAMS(proto),                                  \
  86                         PARAMS(args),                                   \
  87                         PARAMS(fields))
  88
  89 #undef LTTNG_TRACEPOINT_EVENT_NOARGS
  90 #define LTTNG_TRACEPOINT_EVENT_NOARGS(name, fields)                     \
  91         LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, name, PARAMS(fields))
  92
  93 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE
  94 #define LTTNG_TRACEPOINT_EVENT_INSTANCE(template, name, proto, args)    \
  95         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(template, name, name, PARAMS(proto), PARAMS(args))
  96
  97 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS
  98 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS(template, name)  \
  99         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(template, name, name)
 100
 101 #undef LTTNG_TRACEPOINT_EVENT_CLASS
 102 #define LTTNG_TRACEPOINT_EVENT_CLASS(_name, _proto, _args, _fields) \
 103         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(_proto), PARAMS(_args), , , \
 104                 PARAMS(_fields), )
 105
 106 #undef LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS
 107 #define LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(_name, _fields) \
 108         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, , , PARAMS(_fields), )
 109
 110
 111 /*
 112  * Stage 1 of the trace events.
 113  *
 114  * Create dummy trace calls for each events, verifying that the LTTng module
 115  * instrumentation headers match the kernel arguments. Will be optimized
 116  * out by the compiler.
 117  */
 118
 119 /* Reset all macros within TRACEPOINT_EVENT */
 120 #include <probes/lttng-events-reset.h>
 121
 122 #undef TP_PROTO
 123 #define TP_PROTO(...)   __VA_ARGS__
 124
 125 #undef TP_ARGS
 126 #define TP_ARGS(...)    __VA_ARGS__
 127
 128 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 129 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 130 void trace_##_name(_proto);
 131
 132 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 133 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 134 void trace_##_name(void);
 135
 136 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 137
 138 /*
 139  * Stage 1.1 of the trace events.
 140  *
 141  * Create dummy trace prototypes for each event class, and for each used
 142  * template. This will allow checking whether the prototypes from the
 143  * class and the instance using the class actually match.
 144  */
 145
 146 #include <probes/lttng-events-reset.h>  /* Reset all macros within TRACE_EVENT */
 147
 148 #undef TP_PROTO
 149 #define TP_PROTO(...)   __VA_ARGS__
 150
 151 #undef TP_ARGS
 152 #define TP_ARGS(...)    __VA_ARGS__
 153
 154 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 155 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 156 void __event_template_proto___##_template(_proto);
 157
 158 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 159 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 160 void __event_template_proto___##_template(void);
 161
 162 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 163 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 164 void __event_template_proto___##_name(_proto);
 165
 166 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 167 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 168 void __event_template_proto___##_name(void);
 169
 170 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 171
 172 /*
 173  * Stage 1.2 of tracepoint event generation
 174  *
 175  * Unfolding the enums
 176  */
 177 #include <probes/lttng-events-reset.h>  /* Reset all macros within TRACE_EVENT */
 178
 179 /* Enumeration entry (single value) */
 180 #undef ctf_enum_value
 181 #define ctf_enum_value(_string, _value)                                 \
 182         {                                                               \
 183                 .start = {                                              \
 184                         .signedness = lttng_is_signed_type(__typeof__(_value)), \
 185                         .value = lttng_is_signed_type(__typeof__(_value)) ? \
 186                                 (long long) (_value) : (_value),        \
 187                 },                                                      \
 188                 .end = {                                                \
 189                         .signedness = lttng_is_signed_type(__typeof__(_value)), \
 190                         .value = lttng_is_signed_type(__typeof__(_value)) ? \
 191                                 (long long) (_value) : (_value),        \
 192                 },                                                      \
 193                 .string = (_string),                                    \
 194         },
 195
 196 /* Enumeration entry (range) */
 197 #undef ctf_enum_range
 198 #define ctf_enum_range(_string, _range_start, _range_end)               \
 199         {                                                               \
 200                 .start = {                                              \
 201                         .signedness = lttng_is_signed_type(__typeof__(_range_start)), \
 202                         .value = lttng_is_signed_type(__typeof__(_range_start)) ? \
 203                                 (long long) (_range_start) : (_range_start), \
 204                 },                                                      \
 205                 .end = {                                                \
 206                         .signedness = lttng_is_signed_type(__typeof__(_range_end)), \
 207                         .value = lttng_is_signed_type(__typeof__(_range_end)) ? \
 208                                 (long long) (_range_end) : (_range_end), \
 209                 },                                                      \
 210                 .string = (_string),                                    \
 211         },
 212
 213 /* Enumeration entry (automatic value; follows the rules of CTF) */
 214 #undef ctf_enum_auto
 215 #define ctf_enum_auto(_string)                                  \
 216         {                                                               \
 217                 .start = {                                              \
 218                         .signedness = -1,                               \
 219                         .value = -1,                                    \
 220                 },                                                      \
 221                 .end = {                                                \
 222                         .signedness = -1,                               \
 223                         .value = -1,                                    \
 224                 },                                                      \
 225                 .string = (_string),                                    \
 226                 .options = {                                            \
 227                         .is_auto = 1,                                   \
 228                 }                                                       \
 229         },
 230
 231 #undef TP_ENUM_VALUES
 232 #define TP_ENUM_VALUES(...)                                             \
 233         __VA_ARGS__
 234
 235 #undef LTTNG_TRACEPOINT_ENUM
 236 #define LTTNG_TRACEPOINT_ENUM(_name, _values)                           \
 237         const struct lttng_enum_entry __enum_values__##_name[] = { \
 238                 _values                                                 \
 239         };
 240
 241 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 242
 243 /*
 244  * Stage 2 of the trace events.
 245  *
 246  * Create event field type metadata section.
 247  * Each event produce an array of fields.
 248  */
 249
 250 /* Reset all macros within TRACEPOINT_EVENT */
 251 #include <probes/lttng-events-reset.h>
 252 #include <probes/lttng-events-write.h>
 253 #include <probes/lttng-events-nowrite.h>
 254
 255 #undef _ctf_integer_ext
 256 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 257         {                                                       \
 258           .name = #_item,                                       \
 259           .type = __type_integer(_type, 0, 0, -1, _byte_order, _base, none), \
 260           .nowrite = _nowrite,                                  \
 261           .user = _user,                                        \
 262           .nofilter = 0,                                        \
 263         },
 264
 265 #undef _ctf_array_encoded
 266 #define _ctf_array_encoded(_type, _item, _src, _length,         \
 267                 _encoding, _byte_order, _elem_type_base, _user, _nowrite) \
 268         {                                                       \
 269           .name = #_item,                                       \
 270           .type =                                               \
 271                 {                                               \
 272                   .atype = atype_array_nestable,                \
 273                   .u =                                          \
 274                         {                                       \
 275                           .array_nestable =                     \
 276                                 {                               \
 277                                   .elem_type = __LTTNG_COMPOUND_LITERAL(struct lttng_type, \
 278                                         __type_integer(_type, 0, 0, -1, _byte_order, _elem_type_base, _encoding)), \
 279                                   .length = _length,            \
 280                                   .alignment = 0,               \
 281                                 }                               \
 282                         }                                       \
 283                 },                                              \
 284           .nowrite = _nowrite,                                  \
 285           .user = _user,                                        \
 286           .nofilter = 0,                                        \
 287         },
 288
 289 #undef _ctf_array_bitfield
 290 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 291         {                                                       \
 292           .name = #_item,                                       \
 293           .type =                                               \
 294                 {                                               \
 295                   .atype = atype_array_nestable,                \
 296                   .u =                                          \
 297                         {                                       \
 298                           .array_nestable =                     \
 299                                 {                               \
 300                                   .elem_type = __LTTNG_COMPOUND_LITERAL(struct lttng_type, \
 301                                         __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none)), \
 302                                   .length = (_length) * sizeof(_type) * CHAR_BIT, \
 303                                   .alignment = lttng_alignof(_type), \
 304                                 }                               \
 305                         }                                       \
 306                 },                                              \
 307           .nowrite = _nowrite,                                  \
 308           .user = _user,                                        \
 309           .nofilter = 0,                                        \
 310         },
 311
 312
 313 #undef _ctf_sequence_encoded
 314 #define _ctf_sequence_encoded(_type, _item, _src,               \
 315                         _length_type, _src_length, _encoding,   \
 316                         _byte_order, _elem_type_base, _user, _nowrite) \
 317         {                                                       \
 318           .name = "_" #_item "_length",                         \
 319           .type = __type_integer(_length_type, 0, 0, -1, __BYTE_ORDER, 10, none), \
 320           .nowrite = _nowrite,                                  \
 321           .nofilter = 1,                                        \
 322         },                                                      \
 323         {                                                       \
 324           .name = #_item,                                       \
 325           .type =                                               \
 326                 {                                               \
 327                   .atype = atype_sequence_nestable,             \
 328                   .u =                                          \
 329                         {                                       \
 330                           .sequence_nestable =                  \
 331                                 {                               \
 332                                   .length_name = "_" #_item "_length", \
 333                                   .elem_type = __LTTNG_COMPOUND_LITERAL(struct lttng_type, \
 334                                         __type_integer(_type, 0, 0, -1, _byte_order, _elem_type_base, _encoding)), \
 335                                   .alignment = 0,               \
 336                                 },                              \
 337                         },                                      \
 338                 },                                              \
 339           .nowrite = _nowrite,                                  \
 340           .user = _user,                                        \
 341           .nofilter = 0,                                        \
 342         },
 343
 344 #undef _ctf_sequence_bitfield
 345 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 346                         _length_type, _src_length,              \
 347                         _user, _nowrite)                        \
 348         {                                                       \
 349           .name = "_" #_item "_length",                         \
 350           .type = __type_integer(_length_type, 0, 0, -1, __BYTE_ORDER, 10, none), \
 351           .nowrite = _nowrite,                                  \
 352           .nofilter = 1,                                        \
 353         },                                                      \
 354         {                                                       \
 355           .name = #_item,                                       \
 356           .type =                                               \
 357                 {                                               \
 358                   .atype = atype_sequence_nestable,             \
 359                   .u =                                          \
 360                         {                                       \
 361                           .sequence_nestable =                  \
 362                                 {                               \
 363                                   .length_name = "_" #_item "_length", \
 364                                   .elem_type = __LTTNG_COMPOUND_LITERAL(struct lttng_type, \
 365                                         __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none)), \
 366                                   .alignment = lttng_alignof(_type), \
 367                                 },                              \
 368                         },                                      \
 369                 },                                              \
 370           .nowrite = _nowrite,                                  \
 371           .user = _user,                                        \
 372           .nofilter = 0,                                        \
 373         },
 374
 375 #undef _ctf_string
 376 #define _ctf_string(_item, _src, _user, _nowrite)               \
 377         {                                                       \
 378           .name = #_item,                                       \
 379           .type =                                               \
 380                 {                                               \
 381                   .atype = atype_string,                        \
 382                   .u =                                          \
 383                         {                                       \
 384                           .string = { .encoding = lttng_encode_UTF8 }, \
 385                         },                                      \
 386                 },                                              \
 387           .nowrite = _nowrite,                                  \
 388           .user = _user,                                        \
 389           .nofilter = 0,                                        \
 390         },
 391
 392 #undef _ctf_enum
 393 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)   \
 394         {                                                       \
 395                 .name = #_item,                                 \
 396                 .type = {                                       \
 397                         .atype = atype_enum_nestable,           \
 398                         .u = {                                  \
 399                                 .enum_nestable = {              \
 400                                         .desc = &__enum_##_name, \
 401                                         .container_type = __LTTNG_COMPOUND_LITERAL(struct lttng_type, \
 402                                                 __type_integer(_type, 0, 0, -1, __BYTE_ORDER, 10, none)), \
 403                                 },                              \
 404                         },                                      \
 405                 },                                              \
 406                 .nowrite = _nowrite,                            \
 407                 .user = _user,                                  \
 408                 .nofilter = 0,                                  \
 409         },
 410
 411 #undef ctf_custom_field
 412 #define ctf_custom_field(_type, _item, _code)                   \
 413         {                                                       \
 414                 .name = #_item,                                 \
 415                 .type = _type,                                  \
 416                 .nowrite = 0,                                   \
 417                 .user = 0,                                      \
 418                 .nofilter = 1,                                  \
 419         },
 420
 421 #undef ctf_custom_type
 422 #define ctf_custom_type(...)    __VA_ARGS__
 423
 424 #undef TP_FIELDS
 425 #define TP_FIELDS(...)  __VA_ARGS__     /* Only one used in this phase */
 426
 427 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 428 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 429         static const struct lttng_event_field __event_fields___##_name[] = { \
 430                 _fields                                                      \
 431         };
 432
 433 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 434 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 435         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, PARAMS(_fields), _code_post)
 436
 437 #undef LTTNG_TRACEPOINT_ENUM
 438 #define LTTNG_TRACEPOINT_ENUM(_name, _values)                                           \
 439         static const struct lttng_enum_desc __enum_##_name = {                          \
 440                 .name = #_name,                                                         \
 441                 .entries = __enum_values__##_name,                                      \
 442                 .nr_entries = ARRAY_SIZE(__enum_values__##_name),                       \
 443         };
 444
 445 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 446
 447 /*
 448  * Stage 3 of the trace events.
 449  *
 450  * Create probe callback prototypes.
 451  */
 452
 453 /* Reset all macros within TRACEPOINT_EVENT */
 454 #include <probes/lttng-events-reset.h>
 455
 456 #undef TP_PROTO
 457 #define TP_PROTO(...)   __VA_ARGS__
 458
 459 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 460 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 461 static void __event_probe__##_name(void *__data, _proto);
 462
 463 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 464 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 465 static void __event_probe__##_name(void *__data);
 466
 467 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 468
 469 /*
 470  * Stage 4 of the trace events.
 471  *
 472  * Create static inline function that calculates event size.
 473  */
 474
 475 /* Reset all macros within TRACEPOINT_EVENT */
 476 #include <probes/lttng-events-reset.h>
 477 #include <probes/lttng-events-write.h>
 478
 479 #undef _ctf_integer_ext
 480 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 481         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 482         __event_len += sizeof(_type);
 483
 484 #undef _ctf_array_encoded
 485 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 486         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 487         __event_len += sizeof(_type) * (_length);
 488
 489 #undef _ctf_array_bitfield
 490 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 491         _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
 492
 493 #undef _ctf_sequence_encoded
 494 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 495                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 496         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_length_type)); \
 497         __event_len += sizeof(_length_type);                                   \
 498         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 499         {                                                                               \
 500                 size_t __seqlen = (_src_length);                                        \
 501                                                                                         \
 502                 if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 503                         goto error;                                                     \
 504                 barrier();      /* reserve before use. */                               \
 505                 this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = __seqlen; \
 506                 __event_len += sizeof(_type) * __seqlen;                                \
 507         }
 508
 509 #undef _ctf_sequence_bitfield
 510 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 511                         _length_type, _src_length,              \
 512                         _user, _nowrite)                        \
 513         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 514                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 515
 516 /*
 517  * ctf_user_string includes \0. If returns 0, it faulted, so we set size to
 518  * 1 (\0 only).
 519  */
 520 #undef _ctf_string
 521 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 522         if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 523                 goto error;                                                    \
 524         barrier();      /* reserve before use. */                              \
 525         if (_user) {                                                           \
 526                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 527                         max_t(size_t, lttng_strlen_user_inatomic(_src), 1);    \
 528         } else {                                                               \
 529                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 530                         strlen((_src) ? (_src) : __LTTNG_NULL_STRING) + 1; \
 531         }
 532
 533 #undef _ctf_enum
 534 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)                  \
 535         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 536
 537 #undef ctf_align
 538 #define ctf_align(_type)                                                \
 539         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type));
 540
 541 #undef ctf_custom_field
 542 #define ctf_custom_field(_type, _item, _code)                           \
 543         {                                                               \
 544                 _code                                                   \
 545         }
 546
 547 #undef ctf_custom_code
 548 #define ctf_custom_code(...)            __VA_ARGS__
 549
 550 #undef TP_PROTO
 551 #define TP_PROTO(...)   __VA_ARGS__
 552
 553 #undef TP_FIELDS
 554 #define TP_FIELDS(...)  __VA_ARGS__
 555
 556 #undef TP_locvar
 557 #define TP_locvar(...)  __VA_ARGS__
 558
 559 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 560 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 561 static inline ssize_t __event_get_size__##_name(void *__tp_locvar, _proto)     \
 562 {                                                                             \
 563         size_t __event_len = 0;                                               \
 564         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 565         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 566                                                                               \
 567         _fields                                                               \
 568         return __event_len;                                                   \
 569                                                                               \
 570 error:                                                                        \
 571         __attribute__((unused));                                              \
 572         return -1;                                                            \
 573 }
 574
 575 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 576 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 577 static inline ssize_t __event_get_size__##_name(void *__tp_locvar)            \
 578 {                                                                             \
 579         size_t __event_len = 0;                                               \
 580         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 581         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 582                                                                               \
 583         _fields                                                               \
 584         return __event_len;                                                   \
 585                                                                               \
 586 error:                                                                        \
 587         __attribute__((unused));                                              \
 588         return -1;                                                            \
 589 }
 590
 591 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 592
 593
 594 /*
 595  * Stage 4.1 of tracepoint event generation.
 596  *
 597  * Create static inline function that layout the filter stack data.
 598  * We make both write and nowrite data available to the filter.
 599  */
 600
 601 /* Reset all macros within TRACEPOINT_EVENT */
 602 #include <probes/lttng-events-reset.h>
 603 #include <probes/lttng-events-write.h>
 604 #include <probes/lttng-events-nowrite.h>
 605
 606 #undef _ctf_integer_ext_fetched
 607 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 608         if (lttng_is_signed_type(_type)) {                                     \
 609                 int64_t __ctf_tmp_int64;                                       \
 610                 switch (sizeof(_type)) {                                       \
 611                 case 1:                                                        \
 612                 {                                                              \
 613                         union { _type t; int8_t v; } __tmp = { (_type) (_src) }; \
 614                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 615                         break;                                                 \
 616                 }                                                              \
 617                 case 2:                                                        \
 618                 {                                                              \
 619                         union { _type t; int16_t v; } __tmp = { (_type) (_src) }; \
 620                         if (_byte_order != __BYTE_ORDER)                       \
 621                                 __swab16s(&__tmp.v);                           \
 622                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 623                         break;                                                 \
 624                 }                                                              \
 625                 case 4:                                                        \
 626                 {                                                              \
 627                         union { _type t; int32_t v; } __tmp = { (_type) (_src) }; \
 628                         if (_byte_order != __BYTE_ORDER)                       \
 629                                 __swab32s(&__tmp.v);                           \
 630                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 631                         break;                                                 \
 632                 }                                                              \
 633                 case 8:                                                        \
 634                 {                                                              \
 635                         union { _type t; int64_t v; } __tmp = { (_type) (_src) }; \
 636                         if (_byte_order != __BYTE_ORDER)                       \
 637                                 __swab64s(&__tmp.v);                           \
 638                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 639                         break;                                                 \
 640                 }                                                              \
 641                 default:                                                       \
 642                         BUG_ON(1);                                             \
 643                 };                                                             \
 644                 memcpy(__stack_data, &__ctf_tmp_int64, sizeof(int64_t));       \
 645         } else {                                                               \
 646                 uint64_t __ctf_tmp_uint64;                                     \
 647                 switch (sizeof(_type)) {                                       \
 648                 case 1:                                                        \
 649                 {                                                              \
 650                         union { _type t; uint8_t v; } __tmp = { (_type) (_src) }; \
 651                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 652                         break;                                                 \
 653                 }                                                              \
 654                 case 2:                                                        \
 655                 {                                                              \
 656                         union { _type t; uint16_t v; } __tmp = { (_type) (_src) }; \
 657                         if (_byte_order != __BYTE_ORDER)                       \
 658                                 __swab16s(&__tmp.v);                           \
 659                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 660                         break;                                                 \
 661                 }                                                              \
 662                 case 4:                                                        \
 663                 {                                                              \
 664                         union { _type t; uint32_t v; } __tmp = { (_type) (_src) }; \
 665                         if (_byte_order != __BYTE_ORDER)                       \
 666                                 __swab32s(&__tmp.v);                           \
 667                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 668                         break;                                                 \
 669                 }                                                              \
 670                 case 8:                                                        \
 671                 {                                                              \
 672                         union { _type t; uint64_t v; } __tmp = { (_type) (_src) }; \
 673                         if (_byte_order != __BYTE_ORDER)                       \
 674                                 __swab64s(&__tmp.v);                           \
 675                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 676                         break;                                                 \
 677                 }                                                              \
 678                 default:                                                       \
 679                         BUG_ON(1);                                             \
 680                 };                                                             \
 681                 memcpy(__stack_data, &__ctf_tmp_uint64, sizeof(uint64_t));     \
 682         }                                                                      \
 683         __stack_data += sizeof(int64_t);
 684
 685 #undef _ctf_integer_ext_isuser0
 686 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 687         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 688
 689 #undef _ctf_integer_ext_isuser1
 690 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 691 {                                                                                       \
 692         union {                                                                         \
 693                 char __array[sizeof(_user_src)];                                        \
 694                 __typeof__(_user_src) __v;                                              \
 695         } __tmp_fetch;                                                                  \
 696         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 697                                 &(_user_src), sizeof(_user_src)))                       \
 698                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 699         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 700 }
 701
 702 #undef _ctf_integer_ext
 703 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 704         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 705
 706 #undef _ctf_array_encoded
 707 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 708         {                                                                      \
 709                 unsigned long __ctf_tmp_ulong = (unsigned long) (_length);     \
 710                 const void *__ctf_tmp_ptr = (_src);                            \
 711                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 712                 __stack_data += sizeof(unsigned long);                         \
 713                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 714                 __stack_data += sizeof(void *);                                \
 715         }
 716
 717 #undef _ctf_array_bitfield
 718 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 719         _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
 720
 721 #undef _ctf_sequence_encoded
 722 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                \
 723                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 724         {                                                                      \
 725                 unsigned long __ctf_tmp_ulong = (unsigned long) (_src_length); \
 726                 const void *__ctf_tmp_ptr = (_src);                            \
 727                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 728                 __stack_data += sizeof(unsigned long);                         \
 729                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 730                 __stack_data += sizeof(void *);                                \
 731         }
 732
 733 #undef _ctf_sequence_bitfield
 734 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 735                         _length_type, _src_length,              \
 736                         _user, _nowrite)                        \
 737         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 738                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 739
 740 #undef _ctf_string
 741 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 742         {                                                                      \
 743                 const void *__ctf_tmp_ptr =                                    \
 744                         ((_src) ? (_src) : __LTTNG_NULL_STRING);               \
 745                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 746                 __stack_data += sizeof(void *);                                \
 747         }
 748
 749 #undef _ctf_enum
 750 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)                  \
 751         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 752
 753 #undef TP_PROTO
 754 #define TP_PROTO(...) __VA_ARGS__
 755
 756 #undef TP_FIELDS
 757 #define TP_FIELDS(...) __VA_ARGS__
 758
 759 #undef TP_locvar
 760 #define TP_locvar(...)  __VA_ARGS__
 761
 762 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 763 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 764 static inline                                                                 \
 765 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 766                 void *__tp_locvar)                                            \
 767 {                                                                             \
 768         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 769                                                                               \
 770         _fields                                                               \
 771 }
 772
 773 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 774 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 775 static inline                                                                 \
 776 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 777                 void *__tp_locvar, _proto)                                    \
 778 {                                                                             \
 779         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 780                                                                               \
 781         _fields                                                               \
 782 }
 783
 784 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 785
 786 /*
 787  * Stage 5 of the trace events.
 788  *
 789  * Create static inline function that calculates event payload alignment.
 790  */
 791
 792 /* Reset all macros within TRACEPOINT_EVENT */
 793 #include <probes/lttng-events-reset.h>
 794 #include <probes/lttng-events-write.h>
 795
 796 #undef _ctf_integer_ext
 797 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 798         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 799
 800 #undef _ctf_array_encoded
 801 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 802         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 803
 804 #undef _ctf_array_bitfield
 805 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 806         _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
 807
 808 #undef _ctf_sequence_encoded
 809 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 810                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 811         __event_align = max_t(size_t, __event_align, lttng_alignof(_length_type)); \
 812         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 813
 814 #undef _ctf_sequence_bitfield
 815 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 816                         _length_type, _src_length,              \
 817                         _user, _nowrite)                        \
 818         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 819                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 820
 821 #undef _ctf_string
 822 #define _ctf_string(_item, _src, _user, _nowrite)
 823
 824 #undef _ctf_enum
 825 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)   \
 826         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 827
 828 #undef ctf_align
 829 #define ctf_align(_type)                                                \
 830         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 831
 832 #undef TP_PROTO
 833 #define TP_PROTO(...)   __VA_ARGS__
 834
 835 #undef TP_FIELDS
 836 #define TP_FIELDS(...)  __VA_ARGS__
 837
 838 #undef TP_locvar
 839 #define TP_locvar(...)  __VA_ARGS__
 840
 841 #undef ctf_custom_field
 842 #define ctf_custom_field(_type, _item, _code)           _code
 843
 844 #undef ctf_custom_code
 845 #define ctf_custom_code(...)                                            \
 846         {                                                               \
 847                 __VA_ARGS__                                             \
 848         }
 849
 850 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 851 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 852 static inline size_t __event_get_align__##_name(void *__tp_locvar, _proto)    \
 853 {                                                                             \
 854         size_t __event_align = 1;                                             \
 855         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 856                                                                               \
 857         _fields                                                               \
 858         return __event_align;                                                 \
 859 }
 860
 861 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 862 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 863 static inline size_t __event_get_align__##_name(void *__tp_locvar)            \
 864 {                                                                             \
 865         size_t __event_align = 1;                                             \
 866         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 867                                                                               \
 868         _fields                                                               \
 869         return __event_align;                                                 \
 870 }
 871
 872 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 873
 874 /*
 875  * Stage 6 of tracepoint event generation.
 876  *
 877  * Create the probe function. This function calls event size calculation
 878  * and writes event data into the buffer.
 879  */
 880
 881 /* Reset all macros within TRACEPOINT_EVENT */
 882 #include <probes/lttng-events-reset.h>
 883 #include <probes/lttng-events-write.h>
 884
 885 #undef _ctf_integer_ext_fetched
 886 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 887         {                                                               \
 888                 _type __tmp = _src;                                     \
 889                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(__tmp));\
 890                 __chan->ops->event_write(&__ctx, &__tmp, sizeof(__tmp));\
 891         }
 892
 893 #undef _ctf_integer_ext_isuser0
 894 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 895         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 896
 897 #undef _ctf_integer_ext_isuser1
 898 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 899 {                                                                              \
 900         union {                                                                         \
 901                 char __array[sizeof(_user_src)];                                        \
 902                 __typeof__(_user_src) __v;                                              \
 903         } __tmp_fetch;                                                                  \
 904         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 905                                 &(_user_src), sizeof(_user_src)))                       \
 906                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 907         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 908 }
 909
 910 #undef _ctf_integer_ext
 911 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 912         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 913
 914 #undef _ctf_array_encoded
 915 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 916         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 917         if (_user) {                                                    \
 918                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 919         } else {                                                        \
 920                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 921         }
 922
 923 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 924 #undef _ctf_array_bitfield
 925 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 926         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 927         if (_user) {                                                    \
 928                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 929         } else {                                                        \
 930                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 931         }
 932 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 933 /*
 934  * For big endian, we need to byteswap into little endian.
 935  */
 936 #undef _ctf_array_bitfield
 937 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 938         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 939         {                                                               \
 940                 size_t _i;                                              \
 941                                                                         \
 942                 for (_i = 0; _i < (_length); _i++) {                    \
 943                         _type _tmp;                                     \
 944                                                                         \
 945                         if (_user) {                                    \
 946                                 if (get_user(_tmp, (_type *) _src + _i)) \
 947                                         _tmp = 0;                       \
 948                         } else {                                        \
 949                                 _tmp = ((_type *) _src)[_i];            \
 950                         }                                               \
 951                         switch (sizeof(_type)) {                        \
 952                         case 1:                                         \
 953                                 break;                                  \
 954                         case 2:                                         \
 955                                 _tmp = cpu_to_le16(_tmp);               \
 956                                 break;                                  \
 957                         case 4:                                         \
 958                                 _tmp = cpu_to_le32(_tmp);               \
 959                                 break;                                  \
 960                         case 8:                                         \
 961                                 _tmp = cpu_to_le64(_tmp);               \
 962                                 break;                                  \
 963                         default:                                        \
 964                                 BUG_ON(1);                              \
 965                         }                                               \
 966                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
 967                 }                                                       \
 968         }
 969 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 970
 971 #undef _ctf_sequence_encoded
 972 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,         \
 973                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 974         {                                                               \
 975                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx]; \
 976                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 977                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 978         }                                                               \
 979         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 980         if (_user) {                                                    \
 981                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 982                         sizeof(_type) * __get_dynamic_len(dest));       \
 983         } else {                                                        \
 984                 __chan->ops->event_write(&__ctx, _src,                  \
 985                         sizeof(_type) * __get_dynamic_len(dest));       \
 986         }
 987
 988 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 989 #undef _ctf_sequence_bitfield
 990 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 991                         _length_type, _src_length,              \
 992                         _user, _nowrite)                        \
 993         {                                                               \
 994                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 995                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 996                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 997         }                                                               \
 998         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 999         if (_user) {                                                    \
1000                 __chan->ops->event_write_from_user(&__ctx, _src,        \
1001                         sizeof(_type) * __get_dynamic_len(dest));       \
1002         } else {                                                        \
1003                 __chan->ops->event_write(&__ctx, _src,                  \
1004                         sizeof(_type) * __get_dynamic_len(dest));       \
1005         }
1006 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
1007 /*
1008  * For big endian, we need to byteswap into little endian.
1009  */
1010 #undef _ctf_sequence_bitfield
1011 #define _ctf_sequence_bitfield(_type, _item, _src,              \
1012                         _length_type, _src_length,              \
1013                         _user, _nowrite)                        \
1014         {                                                       \
1015                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
1016                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
1017                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
1018         }                                                               \
1019         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
1020         {                                                               \
1021                 size_t _i, _length;                                     \
1022                                                                         \
1023                 _length = __get_dynamic_len(dest);                      \
1024                 for (_i = 0; _i < _length; _i++) {                      \
1025                         _type _tmp;                                     \
1026                                                                         \
1027                         if (_user) {                                    \
1028                                 if (get_user(_tmp, (_type *) _src + _i)) \
1029                                         _tmp = 0;                       \
1030                         } else {                                        \
1031                                 _tmp = ((_type *) _src)[_i];            \
1032                         }                                               \
1033                         switch (sizeof(_type)) {                        \
1034                         case 1:                                         \
1035                                 break;                                  \
1036                         case 2:                                         \
1037                                 _tmp = cpu_to_le16(_tmp);               \
1038                                 break;                                  \
1039                         case 4:                                         \
1040                                 _tmp = cpu_to_le32(_tmp);               \
1041                                 break;                                  \
1042                         case 8:                                         \
1043                                 _tmp = cpu_to_le64(_tmp);               \
1044                                 break;                                  \
1045                         default:                                        \
1046                                 BUG_ON(1);                              \
1047                         }                                               \
1048                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
1049                 }                                                       \
1050         }
1051 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
1052
1053 #undef _ctf_string
1054 #define _ctf_string(_item, _src, _user, _nowrite)                       \
1055         if (_user) {                                                    \
1056                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(*(_src))); \
1057                 __chan->ops->event_strcpy_from_user(&__ctx, _src,       \
1058                         __get_dynamic_len(dest));                       \
1059         } else {                                                        \
1060                 const char *__ctf_tmp_string =                          \
1061                         ((_src) ? (_src) : __LTTNG_NULL_STRING);        \
1062                 lib_ring_buffer_align_ctx(&__ctx,                       \
1063                         lttng_alignof(*__ctf_tmp_string));              \
1064                 __chan->ops->event_strcpy(&__ctx, __ctf_tmp_string,     \
1065                         __get_dynamic_len(dest));                       \
1066         }
1067
1068 #undef _ctf_enum
1069 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)           \
1070         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
1071
1072 #undef ctf_align
1073 #define ctf_align(_type)                                                \
1074         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));
1075
1076 #undef ctf_custom_field
1077 #define ctf_custom_field(_type, _item, _code)           _code
1078
1079 #undef ctf_custom_code
1080 #define ctf_custom_code(...)                                            \
1081         {                                                               \
1082                 __VA_ARGS__                                             \
1083         }
1084
1085 /* Beware: this get len actually consumes the len value */
1086 #undef __get_dynamic_len
1087 #define __get_dynamic_len(field)        this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx++]
1088
1089 #undef TP_PROTO
1090 #define TP_PROTO(...)   __VA_ARGS__
1091
1092 #undef TP_ARGS
1093 #define TP_ARGS(...)    __VA_ARGS__
1094
1095 #undef TP_FIELDS
1096 #define TP_FIELDS(...)  __VA_ARGS__
1097
1098 #undef TP_locvar
1099 #define TP_locvar(...)  __VA_ARGS__
1100
1101 #undef TP_code_pre
1102 #define TP_code_pre(...)        __VA_ARGS__
1103
1104 #undef TP_code_post
1105 #define TP_code_post(...)       __VA_ARGS__
1106
1107 /*
1108  * For state dump, check that "session" argument (mandatory) matches the
1109  * session this event belongs to. Ensures that we write state dump data only
1110  * into the started session, not into all sessions.
1111  */
1112 #ifdef TP_SESSION_CHECK
1113 #define _TP_SESSION_CHECK(session, csession)    (session == csession)
1114 #else /* TP_SESSION_CHECK */
1115 #define _TP_SESSION_CHECK(session, csession)    1
1116 #endif /* TP_SESSION_CHECK */
1117
1118 /*
1119  * Using twice size for filter stack data to hold size and pointer for
1120  * each field (worse case). For integers, max size required is 64-bit.
1121  * Same for double-precision floats. Those fit within
1122  * 2*sizeof(unsigned long) for all supported architectures.
1123  * Perform UNION (||) of filter runtime list.
1124  */
1125 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
1126 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
1127 static void __event_probe__##_name(void *__data, _proto)                      \
1128 {                                                                             \
1129         struct probe_local_vars { _locvar };                                  \
1130         struct lttng_event *__event = __data;                                 \
1131         struct lttng_probe_ctx __lttng_probe_ctx = {                                  \
1132                 .event = __event,                                             \
1133                 .interruptible = !irqs_disabled(),                            \
1134         };                                                                    \
1135         struct lttng_channel *__chan = __event->chan;                         \
1136         struct lttng_session *__session = __chan->session;                    \
1137         struct lib_ring_buffer_ctx __ctx;                                     \
1138         ssize_t __event_len;                                                  \
1139         size_t __event_align;                                                 \
1140         size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \
1141         union {                                                               \
1142                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
1143                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1144         } __stackvar;                                                         \
1145         int __ret;                                                            \
1146         struct probe_local_vars __tp_locvar;                                  \
1147         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
1148                         &__tp_locvar;                                         \
1149         struct lttng_id_tracker_rcu *__lf;                                    \
1150                                                                               \
1151         if (!_TP_SESSION_CHECK(session, __session))                           \
1152                 return;                                                       \
1153         if (unlikely(!READ_ONCE(__session->active)))                          \
1154                 return;                                                       \
1155         if (unlikely(!READ_ONCE(__chan->enabled)))                            \
1156                 return;                                                       \
1157         if (unlikely(!READ_ONCE(__event->enabled)))                           \
1158                 return;                                                       \
1159         __lf = lttng_rcu_dereference(__session->pid_tracker.p);               \
1160         if (__lf && likely(!lttng_id_tracker_lookup(__lf, current->tgid)))    \
1161                 return;                                                       \
1162         __lf = lttng_rcu_dereference(__session->vpid_tracker.p);              \
1163         if (__lf && likely(!lttng_id_tracker_lookup(__lf, task_tgid_vnr(current)))) \
1164                 return;                                                       \
1165         __lf = lttng_rcu_dereference(__session->uid_tracker.p);               \
1166         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1167                         lttng_current_uid())))                                \
1168                 return;                                                       \
1169         __lf = lttng_rcu_dereference(__session->vuid_tracker.p);              \
1170         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1171                         lttng_current_vuid())))                               \
1172                 return;                                                       \
1173         __lf = lttng_rcu_dereference(__session->gid_tracker.p);               \
1174         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1175                         lttng_current_gid())))                                \
1176                 return;                                                       \
1177         __lf = lttng_rcu_dereference(__session->vgid_tracker.p);              \
1178         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1179                         lttng_current_vgid())))                               \
1180                 return;                                                       \
1181         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
1182         __dynamic_len_idx = __orig_dynamic_len_offset;                        \
1183         _code_pre                                                             \
1184         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
1185                 struct lttng_bytecode_runtime *bc_runtime;                    \
1186                 int __filter_record = __event->has_enablers_without_bytecode; \
1187                                                                               \
1188                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
1189                                 tp_locvar, _args);                                    \
1190                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
1191                         if (unlikely(bc_runtime->filter(bc_runtime, &__lttng_probe_ctx,       \
1192                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) { \
1193                                 __filter_record = 1;                          \
1194                                 break;                                        \
1195                         }                                                     \
1196                 }                                                             \
1197                 if (likely(!__filter_record))                                 \
1198                         goto __post;                                          \
1199         }                                                                     \
1200         __event_len = __event_get_size__##_name(tp_locvar, _args);            \
1201         if (unlikely(__event_len < 0)) {                                      \
1202                 lib_ring_buffer_lost_event_too_big(__chan->chan);             \
1203                 goto __post;                                                  \
1204         }                                                                     \
1205         __event_align = __event_get_align__##_name(tp_locvar, _args);         \
1206         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
1207                                  __event_align, -1);                          \
1208         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1209         if (__ret < 0)                                                        \
1210                 goto __post;                                                  \
1211         _fields                                                               \
1212         __chan->ops->event_commit(&__ctx);                                    \
1213 __post:                                                                       \
1214         _code_post                                                            \
1215         barrier();      /* use before un-reserve. */                          \
1216         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1217         return;                                                               \
1218 }
1219
1220 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
1221 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
1222 static void __event_probe__##_name(void *__data)                              \
1223 {                                                                             \
1224         struct probe_local_vars { _locvar };                                  \
1225         struct lttng_event *__event = __data;                                 \
1226         struct lttng_probe_ctx __lttng_probe_ctx = {                                  \
1227                 .event = __event,                                             \
1228                 .interruptible = !irqs_disabled(),                            \
1229         };                                                                    \
1230         struct lttng_channel *__chan = __event->chan;                         \
1231         struct lttng_session *__session = __chan->session;                    \
1232         struct lib_ring_buffer_ctx __ctx;                                     \
1233         ssize_t __event_len;                                                  \
1234         size_t __event_align;                                                 \
1235         size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \
1236         union {                                                               \
1237                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
1238                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1239         } __stackvar;                                                         \
1240         int __ret;                                                            \
1241         struct probe_local_vars __tp_locvar;                                  \
1242         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
1243                         &__tp_locvar;                                         \
1244         struct lttng_id_tracker_rcu *__lf;                                    \
1245                                                                               \
1246         if (!_TP_SESSION_CHECK(session, __session))                           \
1247                 return;                                                       \
1248         if (unlikely(!READ_ONCE(__session->active)))                          \
1249                 return;                                                       \
1250         if (unlikely(!READ_ONCE(__chan->enabled)))                            \
1251                 return;                                                       \
1252         if (unlikely(!READ_ONCE(__event->enabled)))                           \
1253                 return;                                                       \
1254         __lf = lttng_rcu_dereference(__session->pid_tracker.p);               \
1255         if (__lf && likely(!lttng_id_tracker_lookup(__lf, current->tgid)))    \
1256                 return;                                                       \
1257         __lf = lttng_rcu_dereference(__session->vpid_tracker.p);              \
1258         if (__lf && likely(!lttng_id_tracker_lookup(__lf, task_tgid_vnr(current)))) \
1259                 return;                                                       \
1260         __lf = lttng_rcu_dereference(__session->uid_tracker.p);               \
1261         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1262                         lttng_current_uid())))                                \
1263                 return;                                                       \
1264         __lf = lttng_rcu_dereference(__session->vuid_tracker.p);              \
1265         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1266                         lttng_current_vuid())))                               \
1267                 return;                                                       \
1268         __lf = lttng_rcu_dereference(__session->gid_tracker.p);               \
1269         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1270                         lttng_current_gid())))                                \
1271                 return;                                                       \
1272         __lf = lttng_rcu_dereference(__session->vgid_tracker.p);              \
1273         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1274                         lttng_current_vgid())))                               \
1275                 return;                                                       \
1276         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
1277         __dynamic_len_idx = __orig_dynamic_len_offset;                        \
1278         _code_pre                                                             \
1279         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
1280                 struct lttng_bytecode_runtime *bc_runtime;                    \
1281                 int __filter_record = __event->has_enablers_without_bytecode; \
1282                                                                               \
1283                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
1284                                 tp_locvar);                                   \
1285                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
1286                         if (unlikely(bc_runtime->filter(bc_runtime, &__lttng_probe_ctx, \
1287                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) { \
1288                                 __filter_record = 1;                          \
1289                                 break;                                        \
1290                         }                                                     \
1291                 }                                                             \
1292                 if (likely(!__filter_record))                                 \
1293                         goto __post;                                          \
1294         }                                                                     \
1295         __event_len = __event_get_size__##_name(tp_locvar);                   \
1296         if (unlikely(__event_len < 0)) {                                      \
1297                 lib_ring_buffer_lost_event_too_big(__chan->chan);             \
1298                 goto __post;                                                  \
1299         }                                                                     \
1300         __event_align = __event_get_align__##_name(tp_locvar);                \
1301         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
1302                                  __event_align, -1);                          \
1303         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1304         if (__ret < 0)                                                        \
1305                 goto __post;                                                  \
1306         _fields                                                               \
1307         __chan->ops->event_commit(&__ctx);                                    \
1308 __post:                                                                       \
1309         _code_post                                                            \
1310         barrier();      /* use before un-reserve. */                          \
1311         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1312         return;                                                               \
1313 }
1314
1315 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1316
1317 #undef __get_dynamic_len
1318
1319 /*
1320  * Stage 7 of the trace events.
1321  *
1322  * Create event descriptions.
1323  */
1324
1325 /* Named field types must be defined in lttng-types.h */
1326
1327 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1328
1329 #ifndef TP_PROBE_CB
1330 #define TP_PROBE_CB(_template)  &__event_probe__##_template
1331 #endif
1332
1333 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1334 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)      \
1335 static const struct lttng_event_desc __event_desc___##_map = {          \
1336         .fields = __event_fields___##_template,                         \
1337         .name = #_map,                                                  \
1338         .kname = #_name,                                                \
1339         .probe_callback = (void *) TP_PROBE_CB(_template),              \
1340         .nr_fields = ARRAY_SIZE(__event_fields___##_template),          \
1341         .owner = THIS_MODULE,                                           \
1342 };
1343
1344 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1345 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1346         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1347
1348 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1349
1350 /*
1351  * Stage 8 of the trace events.
1352  *
1353  * Create an array of event description pointers.
1354  */
1355
1356 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1357
1358 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1359 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
1360                 &__event_desc___##_map,
1361
1362 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1363 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1364         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1365
1366 #define TP_ID1(_token, _system) _token##_system
1367 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1368
1369 static const struct lttng_event_desc *TP_ID(__event_desc___, TRACE_SYSTEM)[] = {
1370 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1371 };
1372
1373 #undef TP_ID1
1374 #undef TP_ID
1375
1376 /*
1377  * Stage 9 of the trace events.
1378  *
1379  * Create a toplevel descriptor for the whole probe.
1380  */
1381
1382 #define TP_ID1(_token, _system) _token##_system
1383 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1384
1385 /* non-const because list head will be modified when registered. */
1386 static __used struct lttng_probe_desc TP_ID(__probe_desc___, TRACE_SYSTEM) = {
1387         .provider = __stringify(TRACE_SYSTEM),
1388         .event_desc = TP_ID(__event_desc___, TRACE_SYSTEM),
1389         .nr_events = ARRAY_SIZE(TP_ID(__event_desc___, TRACE_SYSTEM)),
1390         .head = { NULL, NULL },
1391         .lazy_init_head = { NULL, NULL },
1392         .lazy = 0,
1393 };
1394
1395 #undef TP_ID1
1396 #undef TP_ID
1397
1398 /*
1399  * Stage 10 of the trace events.
1400  *
1401  * Register/unregister probes at module load/unload.
1402  */
1403
1404 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1405
1406 #define TP_ID1(_token, _system) _token##_system
1407 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1408 #define module_init_eval1(_token, _system)      module_init(_token##_system)
1409 #define module_init_eval(_token, _system)       module_init_eval1(_token, _system)
1410 #define module_exit_eval1(_token, _system)      module_exit(_token##_system)
1411 #define module_exit_eval(_token, _system)       module_exit_eval1(_token, _system)
1412
1413 #ifndef TP_MODULE_NOINIT
1414 static int TP_ID(__lttng_events_init__, TRACE_SYSTEM)(void)
1415 {
1416         return lttng_probe_register(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1417 }
1418
1419 static void TP_ID(__lttng_events_exit__, TRACE_SYSTEM)(void)
1420 {
1421         lttng_probe_unregister(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1422 }
1423
1424 #ifndef TP_MODULE_NOAUTOLOAD
1425 module_init_eval(__lttng_events_init__, TRACE_SYSTEM);
1426 module_exit_eval(__lttng_events_exit__, TRACE_SYSTEM);
1427 #endif
1428
1429 #endif
1430
1431 #undef module_init_eval
1432 #undef module_exit_eval
1433 #undef TP_ID1
1434 #undef TP_ID
1435
1436 #undef TP_PROTO
1437 #undef TP_ARGS