probes/lttng-tracepoint-event-impl.h

   1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
   2  *
   3  * lttng-tracepoint-event-impl.h
   4  *
   5  * Copyright (C) 2009 Steven Rostedt <rostedt@goodmis.org>
   6  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   7  */
   8
   9 #include <linux/uaccess.h>
  10 #include <linux/debugfs.h>
  11 #include <linux/rculist.h>
  12 #include <asm/byteorder.h>
  13 #include <linux/swab.h>
  14
  15 #include <probes/lttng.h>
  16 #include <probes/lttng-types.h>
  17 #include <probes/lttng-probe-user.h>
  18 #include <wrapper/vmalloc.h>    /* for wrapper_vmalloc_sync_mappings() */
  19 #include <ringbuffer/frontend_types.h>
  20 #include <ringbuffer/backend.h>
  21 #include <wrapper/rcu.h>
  22 #include <wrapper/user_namespace.h>
  23 #include <lttng-events.h>
  24 #include <lttng-tracer-core.h>
  25 #include <lttng-tp-mempool.h>
  26
  27 #define __LTTNG_NULL_STRING     "(null)"
  28
  29 /*
  30  * Macro declarations used for all stages.
  31  */
  32
  33 /*
  34  * LTTng name mapping macros. LTTng remaps some of the kernel events to
  35  * enforce name-spacing.
  36  */
  37 #undef LTTNG_TRACEPOINT_EVENT_MAP
  38 #define LTTNG_TRACEPOINT_EVENT_MAP(name, map, proto, args, fields) \
  39         LTTNG_TRACEPOINT_EVENT_CLASS(map,                               \
  40                              PARAMS(proto),                             \
  41                              PARAMS(args),                              \
  42                              PARAMS(fields))                            \
  43         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  44
  45 #undef LTTNG_TRACEPOINT_EVENT_MAP_NOARGS
  46 #define LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, map, fields)            \
  47         LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(map,                        \
  48                              PARAMS(fields))                            \
  49         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(map, name, map)
  50
  51 #undef LTTNG_TRACEPOINT_EVENT_CODE_MAP
  52 #define LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, map, proto, args, _locvar, _code_pre, fields, _code_post) \
  53         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(map,                          \
  54                              PARAMS(proto),                             \
  55                              PARAMS(args),                              \
  56                              PARAMS(_locvar),                           \
  57                              PARAMS(_code_pre),                         \
  58                              PARAMS(fields),                            \
  59                              PARAMS(_code_post))                        \
  60         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  61
  62 #undef LTTNG_TRACEPOINT_EVENT_CODE
  63 #define LTTNG_TRACEPOINT_EVENT_CODE(name, proto, args, _locvar, _code_pre, fields, _code_post) \
  64         LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, name,                     \
  65                              PARAMS(proto),                             \
  66                              PARAMS(args),                              \
  67                              PARAMS(_locvar),                           \
  68                              PARAMS(_code_pre),                         \
  69                              PARAMS(fields),                            \
  70                              PARAMS(_code_post))
  71
  72 /*
  73  * LTTNG_TRACEPOINT_EVENT_CLASS can be used to add a generic function
  74  * handlers for events. That is, if all events have the same parameters
  75  * and just have distinct trace points.  Each tracepoint can be defined
  76  * with LTTNG_TRACEPOINT_EVENT_INSTANCE and that will map the
  77  * LTTNG_TRACEPOINT_EVENT_CLASS to the tracepoint.
  78  *
  79  * LTTNG_TRACEPOINT_EVENT is a one to one mapping between tracepoint and
  80  * template.
  81  */
  82
  83 #undef LTTNG_TRACEPOINT_EVENT
  84 #define LTTNG_TRACEPOINT_EVENT(name, proto, args, fields)               \
  85         LTTNG_TRACEPOINT_EVENT_MAP(name, name,                          \
  86                         PARAMS(proto),                                  \
  87                         PARAMS(args),                                   \
  88                         PARAMS(fields))
  89
  90 #undef LTTNG_TRACEPOINT_EVENT_NOARGS
  91 #define LTTNG_TRACEPOINT_EVENT_NOARGS(name, fields)                     \
  92         LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, name, PARAMS(fields))
  93
  94 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE
  95 #define LTTNG_TRACEPOINT_EVENT_INSTANCE(template, name, proto, args)    \
  96         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(template, name, name, PARAMS(proto), PARAMS(args))
  97
  98 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS
  99 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS(template, name)  \
 100         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(template, name, name)
 101
 102 #undef LTTNG_TRACEPOINT_EVENT_CLASS
 103 #define LTTNG_TRACEPOINT_EVENT_CLASS(_name, _proto, _args, _fields) \
 104         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(_proto), PARAMS(_args), , , \
 105                 PARAMS(_fields), )
 106
 107 #undef LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS
 108 #define LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(_name, _fields) \
 109         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, , , PARAMS(_fields), )
 110
 111
 112 /*
 113  * Stage 1 of the trace events.
 114  *
 115  * Create dummy trace calls for each events, verifying that the LTTng module
 116  * instrumentation headers match the kernel arguments. Will be optimized
 117  * out by the compiler.
 118  */
 119
 120 /* Reset all macros within TRACEPOINT_EVENT */
 121 #include <probes/lttng-events-reset.h>
 122
 123 #undef TP_PROTO
 124 #define TP_PROTO(...)   __VA_ARGS__
 125
 126 #undef TP_ARGS
 127 #define TP_ARGS(...)    __VA_ARGS__
 128
 129 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 130 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 131 void trace_##_name(_proto);
 132
 133 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 134 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 135 void trace_##_name(void);
 136
 137 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 138
 139 /*
 140  * Stage 1.1 of the trace events.
 141  *
 142  * Create dummy trace prototypes for each event class, and for each used
 143  * template. This will allow checking whether the prototypes from the
 144  * class and the instance using the class actually match.
 145  */
 146
 147 #include <probes/lttng-events-reset.h>  /* Reset all macros within TRACE_EVENT */
 148
 149 #undef TP_PROTO
 150 #define TP_PROTO(...)   __VA_ARGS__
 151
 152 #undef TP_ARGS
 153 #define TP_ARGS(...)    __VA_ARGS__
 154
 155 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 156 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 157 void __event_template_proto___##_template(_proto);
 158
 159 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 160 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 161 void __event_template_proto___##_template(void);
 162
 163 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 164 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 165 void __event_template_proto___##_name(_proto);
 166
 167 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 168 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 169 void __event_template_proto___##_name(void);
 170
 171 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 172
 173 /*
 174  * Stage 1.2 of tracepoint event generation
 175  *
 176  * Unfolding the enums
 177  */
 178 #include <probes/lttng-events-reset.h>  /* Reset all macros within TRACE_EVENT */
 179
 180 /* Enumeration entry (single value) */
 181 #undef ctf_enum_value
 182 #define ctf_enum_value(_string, _value)                                 \
 183         {                                                               \
 184                 .start = {                                              \
 185                         .signedness = lttng_is_signed_type(__typeof__(_value)), \
 186                         .value = lttng_is_signed_type(__typeof__(_value)) ? \
 187                                 (long long) (_value) : (_value),        \
 188                 },                                                      \
 189                 .end = {                                                \
 190                         .signedness = lttng_is_signed_type(__typeof__(_value)), \
 191                         .value = lttng_is_signed_type(__typeof__(_value)) ? \
 192                                 (long long) (_value) : (_value),        \
 193                 },                                                      \
 194                 .string = (_string),                                    \
 195         },
 196
 197 /* Enumeration entry (range) */
 198 #undef ctf_enum_range
 199 #define ctf_enum_range(_string, _range_start, _range_end)               \
 200         {                                                               \
 201                 .start = {                                              \
 202                         .signedness = lttng_is_signed_type(__typeof__(_range_start)), \
 203                         .value = lttng_is_signed_type(__typeof__(_range_start)) ? \
 204                                 (long long) (_range_start) : (_range_start), \
 205                 },                                                      \
 206                 .end = {                                                \
 207                         .signedness = lttng_is_signed_type(__typeof__(_range_end)), \
 208                         .value = lttng_is_signed_type(__typeof__(_range_end)) ? \
 209                                 (long long) (_range_end) : (_range_end), \
 210                 },                                                      \
 211                 .string = (_string),                                    \
 212         },
 213
 214 /* Enumeration entry (automatic value; follows the rules of CTF) */
 215 #undef ctf_enum_auto
 216 #define ctf_enum_auto(_string)                                  \
 217         {                                                               \
 218                 .start = {                                              \
 219                         .signedness = -1,                               \
 220                         .value = -1,                                    \
 221                 },                                                      \
 222                 .end = {                                                \
 223                         .signedness = -1,                               \
 224                         .value = -1,                                    \
 225                 },                                                      \
 226                 .string = (_string),                                    \
 227                 .options = {                                            \
 228                         .is_auto = 1,                                   \
 229                 }                                                       \
 230         },
 231
 232 #undef TP_ENUM_VALUES
 233 #define TP_ENUM_VALUES(...)                                             \
 234         __VA_ARGS__
 235
 236 #undef LTTNG_TRACEPOINT_ENUM
 237 #define LTTNG_TRACEPOINT_ENUM(_name, _values)                           \
 238         const struct lttng_enum_entry __enum_values__##_name[] = { \
 239                 _values                                                 \
 240         };
 241
 242 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 243
 244 /*
 245  * Stage 2 of the trace events.
 246  *
 247  * Create event field type metadata section.
 248  * Each event produce an array of fields.
 249  */
 250
 251 /* Reset all macros within TRACEPOINT_EVENT */
 252 #include <probes/lttng-events-reset.h>
 253 #include <probes/lttng-events-write.h>
 254 #include <probes/lttng-events-nowrite.h>
 255
 256 #undef _ctf_integer_ext
 257 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 258         {                                                       \
 259           .name = #_item,                                       \
 260           .type = __type_integer(_type, 0, 0, -1, _byte_order, _base, none), \
 261           .nowrite = _nowrite,                                  \
 262           .user = _user,                                        \
 263           .nofilter = 0,                                        \
 264         },
 265
 266 #undef _ctf_array_encoded
 267 #define _ctf_array_encoded(_type, _item, _src, _length,         \
 268                 _encoding, _byte_order, _elem_type_base, _user, _nowrite) \
 269         {                                                       \
 270           .name = #_item,                                       \
 271           .type =                                               \
 272                 {                                               \
 273                   .atype = atype_array_nestable,                \
 274                   .u =                                          \
 275                         {                                       \
 276                           .array_nestable =                     \
 277                                 {                               \
 278                                   .elem_type = __LTTNG_COMPOUND_LITERAL(struct lttng_type, \
 279                                         __type_integer(_type, 0, 0, -1, _byte_order, _elem_type_base, _encoding)), \
 280                                   .length = _length,            \
 281                                   .alignment = 0,               \
 282                                 }                               \
 283                         }                                       \
 284                 },                                              \
 285           .nowrite = _nowrite,                                  \
 286           .user = _user,                                        \
 287           .nofilter = 0,                                        \
 288         },
 289
 290 #undef _ctf_array_bitfield
 291 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 292         {                                                       \
 293           .name = #_item,                                       \
 294           .type =                                               \
 295                 {                                               \
 296                   .atype = atype_array_nestable,                \
 297                   .u =                                          \
 298                         {                                       \
 299                           .array_nestable =                     \
 300                                 {                               \
 301                                   .elem_type = __LTTNG_COMPOUND_LITERAL(struct lttng_type, \
 302                                         __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none)), \
 303                                   .length = (_length) * sizeof(_type) * CHAR_BIT, \
 304                                   .alignment = lttng_alignof(_type), \
 305                                 }                               \
 306                         }                                       \
 307                 },                                              \
 308           .nowrite = _nowrite,                                  \
 309           .user = _user,                                        \
 310           .nofilter = 0,                                        \
 311         },
 312
 313
 314 #undef _ctf_sequence_encoded
 315 #define _ctf_sequence_encoded(_type, _item, _src,               \
 316                         _length_type, _src_length, _encoding,   \
 317                         _byte_order, _elem_type_base, _user, _nowrite) \
 318         {                                                       \
 319           .name = "_" #_item "_length",                         \
 320           .type = __type_integer(_length_type, 0, 0, -1, __BYTE_ORDER, 10, none), \
 321           .nowrite = _nowrite,                                  \
 322           .nofilter = 1,                                        \
 323         },                                                      \
 324         {                                                       \
 325           .name = #_item,                                       \
 326           .type =                                               \
 327                 {                                               \
 328                   .atype = atype_sequence_nestable,             \
 329                   .u =                                          \
 330                         {                                       \
 331                           .sequence_nestable =                  \
 332                                 {                               \
 333                                   .length_name = "_" #_item "_length", \
 334                                   .elem_type = __LTTNG_COMPOUND_LITERAL(struct lttng_type, \
 335                                         __type_integer(_type, 0, 0, -1, _byte_order, _elem_type_base, _encoding)), \
 336                                   .alignment = 0,               \
 337                                 },                              \
 338                         },                                      \
 339                 },                                              \
 340           .nowrite = _nowrite,                                  \
 341           .user = _user,                                        \
 342           .nofilter = 0,                                        \
 343         },
 344
 345 #undef _ctf_sequence_bitfield
 346 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 347                         _length_type, _src_length,              \
 348                         _user, _nowrite)                        \
 349         {                                                       \
 350           .name = "_" #_item "_length",                         \
 351           .type = __type_integer(_length_type, 0, 0, -1, __BYTE_ORDER, 10, none), \
 352           .nowrite = _nowrite,                                  \
 353           .nofilter = 1,                                        \
 354         },                                                      \
 355         {                                                       \
 356           .name = #_item,                                       \
 357           .type =                                               \
 358                 {                                               \
 359                   .atype = atype_sequence_nestable,             \
 360                   .u =                                          \
 361                         {                                       \
 362                           .sequence_nestable =                  \
 363                                 {                               \
 364                                   .length_name = "_" #_item "_length", \
 365                                   .elem_type = __LTTNG_COMPOUND_LITERAL(struct lttng_type, \
 366                                         __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none)), \
 367                                   .alignment = lttng_alignof(_type), \
 368                                 },                              \
 369                         },                                      \
 370                 },                                              \
 371           .nowrite = _nowrite,                                  \
 372           .user = _user,                                        \
 373           .nofilter = 0,                                        \
 374         },
 375
 376 #undef _ctf_string
 377 #define _ctf_string(_item, _src, _user, _nowrite)               \
 378         {                                                       \
 379           .name = #_item,                                       \
 380           .type =                                               \
 381                 {                                               \
 382                   .atype = atype_string,                        \
 383                   .u =                                          \
 384                         {                                       \
 385                           .string = { .encoding = lttng_encode_UTF8 }, \
 386                         },                                      \
 387                 },                                              \
 388           .nowrite = _nowrite,                                  \
 389           .user = _user,                                        \
 390           .nofilter = 0,                                        \
 391         },
 392
 393 #undef _ctf_enum
 394 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)   \
 395         {                                                       \
 396                 .name = #_item,                                 \
 397                 .type = {                                       \
 398                         .atype = atype_enum_nestable,           \
 399                         .u = {                                  \
 400                                 .enum_nestable = {              \
 401                                         .desc = &__enum_##_name, \
 402                                         .container_type = __LTTNG_COMPOUND_LITERAL(struct lttng_type, \
 403                                                 __type_integer(_type, 0, 0, -1, __BYTE_ORDER, 10, none)), \
 404                                 },                              \
 405                         },                                      \
 406                 },                                              \
 407                 .nowrite = _nowrite,                            \
 408                 .user = _user,                                  \
 409                 .nofilter = 0,                                  \
 410         },
 411
 412 #undef ctf_custom_field
 413 #define ctf_custom_field(_type, _item, _code)                   \
 414         {                                                       \
 415                 .name = #_item,                                 \
 416                 .type = _type,                                  \
 417                 .nowrite = 0,                                   \
 418                 .user = 0,                                      \
 419                 .nofilter = 1,                                  \
 420         },
 421
 422 #undef ctf_custom_type
 423 #define ctf_custom_type(...)    __VA_ARGS__
 424
 425 #undef TP_FIELDS
 426 #define TP_FIELDS(...)  __VA_ARGS__     /* Only one used in this phase */
 427
 428 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 429 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 430         static const struct lttng_event_field __event_fields___##_name[] = { \
 431                 _fields                                                      \
 432         };
 433
 434 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 435 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 436         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, PARAMS(_fields), _code_post)
 437
 438 #undef LTTNG_TRACEPOINT_ENUM
 439 #define LTTNG_TRACEPOINT_ENUM(_name, _values)                                           \
 440         static const struct lttng_enum_desc __enum_##_name = {                          \
 441                 .name = #_name,                                                         \
 442                 .entries = __enum_values__##_name,                                      \
 443                 .nr_entries = ARRAY_SIZE(__enum_values__##_name),                       \
 444         };
 445
 446 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 447
 448 /*
 449  * Stage 3 of the trace events.
 450  *
 451  * Create probe callback prototypes.
 452  */
 453
 454 /* Reset all macros within TRACEPOINT_EVENT */
 455 #include <probes/lttng-events-reset.h>
 456
 457 #undef TP_PROTO
 458 #define TP_PROTO(...)   __VA_ARGS__
 459
 460 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 461 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 462 static void __event_probe__##_name(void *__data, _proto);
 463
 464 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 465 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 466 static void __event_probe__##_name(void *__data);
 467
 468 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 469
 470 /*
 471  * Stage 4 of the trace events.
 472  *
 473  * Create static inline function that calculates event size.
 474  */
 475
 476 /* Reset all macros within TRACEPOINT_EVENT */
 477 #include <probes/lttng-events-reset.h>
 478 #include <probes/lttng-events-write.h>
 479
 480 #undef _ctf_integer_ext
 481 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 482         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 483         __event_len += sizeof(_type);
 484
 485 #undef _ctf_array_encoded
 486 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 487         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 488         __event_len += sizeof(_type) * (_length);
 489
 490 #undef _ctf_array_bitfield
 491 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 492         _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
 493
 494 #undef _ctf_sequence_encoded
 495 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 496                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 497         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_length_type)); \
 498         __event_len += sizeof(_length_type);                                   \
 499         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 500         {                                                                               \
 501                 size_t __seqlen = (_src_length);                                        \
 502                                                                                         \
 503                 if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 504                         goto error;                                                     \
 505                 barrier();      /* reserve before use. */                               \
 506                 this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = __seqlen; \
 507                 __event_len += sizeof(_type) * __seqlen;                                \
 508         }
 509
 510 #undef _ctf_sequence_bitfield
 511 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 512                         _length_type, _src_length,              \
 513                         _user, _nowrite)                        \
 514         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 515                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 516
 517 /*
 518  * ctf_user_string includes \0. If returns 0, it faulted, so we set size to
 519  * 1 (\0 only).
 520  */
 521 #undef _ctf_string
 522 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 523         if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 524                 goto error;                                                    \
 525         barrier();      /* reserve before use. */                              \
 526         if (_user) {                                                           \
 527                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 528                         max_t(size_t, lttng_strlen_user_inatomic(_src), 1);    \
 529         } else {                                                               \
 530                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 531                         strlen((_src) ? (_src) : __LTTNG_NULL_STRING) + 1; \
 532         }
 533
 534 #undef _ctf_enum
 535 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)                  \
 536         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 537
 538 #undef ctf_align
 539 #define ctf_align(_type)                                                \
 540         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type));
 541
 542 #undef ctf_custom_field
 543 #define ctf_custom_field(_type, _item, _code)                           \
 544         {                                                               \
 545                 _code                                                   \
 546         }
 547
 548 #undef ctf_custom_code
 549 #define ctf_custom_code(...)            __VA_ARGS__
 550
 551 #undef TP_PROTO
 552 #define TP_PROTO(...)   __VA_ARGS__
 553
 554 #undef TP_FIELDS
 555 #define TP_FIELDS(...)  __VA_ARGS__
 556
 557 #undef TP_locvar
 558 #define TP_locvar(...)  __VA_ARGS__
 559
 560 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 561 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 562 static inline ssize_t __event_get_size__##_name(void *__tp_locvar, _proto)     \
 563 {                                                                             \
 564         size_t __event_len = 0;                                               \
 565         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 566         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 567                                                                               \
 568         _fields                                                               \
 569         return __event_len;                                                   \
 570                                                                               \
 571 error:                                                                        \
 572         __attribute__((unused));                                              \
 573         return -1;                                                            \
 574 }
 575
 576 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 577 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 578 static inline ssize_t __event_get_size__##_name(void *__tp_locvar)            \
 579 {                                                                             \
 580         size_t __event_len = 0;                                               \
 581         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 582         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 583                                                                               \
 584         _fields                                                               \
 585         return __event_len;                                                   \
 586                                                                               \
 587 error:                                                                        \
 588         __attribute__((unused));                                              \
 589         return -1;                                                            \
 590 }
 591
 592 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 593
 594
 595 /*
 596  * Stage 4.1 of tracepoint event generation.
 597  *
 598  * Create static inline function that layout the filter stack data.
 599  * We make both write and nowrite data available to the filter.
 600  */
 601
 602 /* Reset all macros within TRACEPOINT_EVENT */
 603 #include <probes/lttng-events-reset.h>
 604 #include <probes/lttng-events-write.h>
 605 #include <probes/lttng-events-nowrite.h>
 606
 607 #undef _ctf_integer_ext_fetched
 608 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 609         if (lttng_is_signed_type(_type)) {                                     \
 610                 int64_t __ctf_tmp_int64;                                       \
 611                 switch (sizeof(_type)) {                                       \
 612                 case 1:                                                        \
 613                 {                                                              \
 614                         union { _type t; int8_t v; } __tmp = { (_type) (_src) }; \
 615                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 616                         break;                                                 \
 617                 }                                                              \
 618                 case 2:                                                        \
 619                 {                                                              \
 620                         union { _type t; int16_t v; } __tmp = { (_type) (_src) }; \
 621                         if (_byte_order != __BYTE_ORDER)                       \
 622                                 __swab16s(&__tmp.v);                           \
 623                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 624                         break;                                                 \
 625                 }                                                              \
 626                 case 4:                                                        \
 627                 {                                                              \
 628                         union { _type t; int32_t v; } __tmp = { (_type) (_src) }; \
 629                         if (_byte_order != __BYTE_ORDER)                       \
 630                                 __swab32s(&__tmp.v);                           \
 631                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 632                         break;                                                 \
 633                 }                                                              \
 634                 case 8:                                                        \
 635                 {                                                              \
 636                         union { _type t; int64_t v; } __tmp = { (_type) (_src) }; \
 637                         if (_byte_order != __BYTE_ORDER)                       \
 638                                 __swab64s(&__tmp.v);                           \
 639                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 640                         break;                                                 \
 641                 }                                                              \
 642                 default:                                                       \
 643                         BUG_ON(1);                                             \
 644                 };                                                             \
 645                 memcpy(__stack_data, &__ctf_tmp_int64, sizeof(int64_t));       \
 646         } else {                                                               \
 647                 uint64_t __ctf_tmp_uint64;                                     \
 648                 switch (sizeof(_type)) {                                       \
 649                 case 1:                                                        \
 650                 {                                                              \
 651                         union { _type t; uint8_t v; } __tmp = { (_type) (_src) }; \
 652                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 653                         break;                                                 \
 654                 }                                                              \
 655                 case 2:                                                        \
 656                 {                                                              \
 657                         union { _type t; uint16_t v; } __tmp = { (_type) (_src) }; \
 658                         if (_byte_order != __BYTE_ORDER)                       \
 659                                 __swab16s(&__tmp.v);                           \
 660                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 661                         break;                                                 \
 662                 }                                                              \
 663                 case 4:                                                        \
 664                 {                                                              \
 665                         union { _type t; uint32_t v; } __tmp = { (_type) (_src) }; \
 666                         if (_byte_order != __BYTE_ORDER)                       \
 667                                 __swab32s(&__tmp.v);                           \
 668                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 669                         break;                                                 \
 670                 }                                                              \
 671                 case 8:                                                        \
 672                 {                                                              \
 673                         union { _type t; uint64_t v; } __tmp = { (_type) (_src) }; \
 674                         if (_byte_order != __BYTE_ORDER)                       \
 675                                 __swab64s(&__tmp.v);                           \
 676                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 677                         break;                                                 \
 678                 }                                                              \
 679                 default:                                                       \
 680                         BUG_ON(1);                                             \
 681                 };                                                             \
 682                 memcpy(__stack_data, &__ctf_tmp_uint64, sizeof(uint64_t));     \
 683         }                                                                      \
 684         __stack_data += sizeof(int64_t);
 685
 686 #undef _ctf_integer_ext_isuser0
 687 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 688         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 689
 690 #undef _ctf_integer_ext_isuser1
 691 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 692 {                                                                                       \
 693         union {                                                                         \
 694                 char __array[sizeof(_user_src)];                                        \
 695                 __typeof__(_user_src) __v;                                              \
 696         } __tmp_fetch;                                                                  \
 697         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 698                                 &(_user_src), sizeof(_user_src)))                       \
 699                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 700         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 701 }
 702
 703 #undef _ctf_integer_ext
 704 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 705         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 706
 707 #undef _ctf_array_encoded
 708 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 709         {                                                                      \
 710                 unsigned long __ctf_tmp_ulong = (unsigned long) (_length);     \
 711                 const void *__ctf_tmp_ptr = (_src);                            \
 712                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 713                 __stack_data += sizeof(unsigned long);                         \
 714                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 715                 __stack_data += sizeof(void *);                                \
 716         }
 717
 718 #undef _ctf_array_bitfield
 719 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 720         _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
 721
 722 #undef _ctf_sequence_encoded
 723 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                \
 724                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 725         {                                                                      \
 726                 unsigned long __ctf_tmp_ulong = (unsigned long) (_src_length); \
 727                 const void *__ctf_tmp_ptr = (_src);                            \
 728                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 729                 __stack_data += sizeof(unsigned long);                         \
 730                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 731                 __stack_data += sizeof(void *);                                \
 732         }
 733
 734 #undef _ctf_sequence_bitfield
 735 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 736                         _length_type, _src_length,              \
 737                         _user, _nowrite)                        \
 738         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 739                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 740
 741 #undef _ctf_string
 742 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 743         {                                                                      \
 744                 const void *__ctf_tmp_ptr =                                    \
 745                         ((_src) ? (_src) : __LTTNG_NULL_STRING);               \
 746                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 747                 __stack_data += sizeof(void *);                                \
 748         }
 749
 750 #undef _ctf_enum
 751 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)                  \
 752         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 753
 754 #undef TP_PROTO
 755 #define TP_PROTO(...) __VA_ARGS__
 756
 757 #undef TP_FIELDS
 758 #define TP_FIELDS(...) __VA_ARGS__
 759
 760 #undef TP_locvar
 761 #define TP_locvar(...)  __VA_ARGS__
 762
 763 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 764 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 765 static inline                                                                 \
 766 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 767                 void *__tp_locvar)                                            \
 768 {                                                                             \
 769         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 770                                                                               \
 771         _fields                                                               \
 772 }
 773
 774 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 775 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 776 static inline                                                                 \
 777 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 778                 void *__tp_locvar, _proto)                                    \
 779 {                                                                             \
 780         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 781                                                                               \
 782         _fields                                                               \
 783 }
 784
 785 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 786
 787 /*
 788  * Stage 5 of the trace events.
 789  *
 790  * Create static inline function that calculates event payload alignment.
 791  */
 792
 793 /* Reset all macros within TRACEPOINT_EVENT */
 794 #include <probes/lttng-events-reset.h>
 795 #include <probes/lttng-events-write.h>
 796
 797 #undef _ctf_integer_ext
 798 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 799         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 800
 801 #undef _ctf_array_encoded
 802 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 803         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 804
 805 #undef _ctf_array_bitfield
 806 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 807         _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
 808
 809 #undef _ctf_sequence_encoded
 810 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 811                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 812         __event_align = max_t(size_t, __event_align, lttng_alignof(_length_type)); \
 813         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 814
 815 #undef _ctf_sequence_bitfield
 816 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 817                         _length_type, _src_length,              \
 818                         _user, _nowrite)                        \
 819         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 820                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 821
 822 #undef _ctf_string
 823 #define _ctf_string(_item, _src, _user, _nowrite)
 824
 825 #undef _ctf_enum
 826 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)   \
 827         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 828
 829 #undef ctf_align
 830 #define ctf_align(_type)                                                \
 831         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 832
 833 #undef TP_PROTO
 834 #define TP_PROTO(...)   __VA_ARGS__
 835
 836 #undef TP_FIELDS
 837 #define TP_FIELDS(...)  __VA_ARGS__
 838
 839 #undef TP_locvar
 840 #define TP_locvar(...)  __VA_ARGS__
 841
 842 #undef ctf_custom_field
 843 #define ctf_custom_field(_type, _item, _code)           _code
 844
 845 #undef ctf_custom_code
 846 #define ctf_custom_code(...)                                            \
 847         {                                                               \
 848                 __VA_ARGS__                                             \
 849         }
 850
 851 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 852 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 853 static inline size_t __event_get_align__##_name(void *__tp_locvar, _proto)    \
 854 {                                                                             \
 855         size_t __event_align = 1;                                             \
 856         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 857                                                                               \
 858         _fields                                                               \
 859         return __event_align;                                                 \
 860 }
 861
 862 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 863 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 864 static inline size_t __event_get_align__##_name(void *__tp_locvar)            \
 865 {                                                                             \
 866         size_t __event_align = 1;                                             \
 867         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 868                                                                               \
 869         _fields                                                               \
 870         return __event_align;                                                 \
 871 }
 872
 873 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 874
 875 /*
 876  * Stage 6 of tracepoint event generation.
 877  *
 878  * Create the probe function. This function calls event size calculation
 879  * and writes event data into the buffer.
 880  */
 881
 882 /* Reset all macros within TRACEPOINT_EVENT */
 883 #include <probes/lttng-events-reset.h>
 884 #include <probes/lttng-events-write.h>
 885
 886 #undef _ctf_integer_ext_fetched
 887 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 888         {                                                               \
 889                 _type __tmp = _src;                                     \
 890                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(__tmp));\
 891                 __chan->ops->event_write(&__ctx, &__tmp, sizeof(__tmp));\
 892         }
 893
 894 #undef _ctf_integer_ext_isuser0
 895 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 896         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 897
 898 #undef _ctf_integer_ext_isuser1
 899 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 900 {                                                                              \
 901         union {                                                                         \
 902                 char __array[sizeof(_user_src)];                                        \
 903                 __typeof__(_user_src) __v;                                              \
 904         } __tmp_fetch;                                                                  \
 905         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 906                                 &(_user_src), sizeof(_user_src)))                       \
 907                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 908         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 909 }
 910
 911 #undef _ctf_integer_ext
 912 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 913         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 914
 915 #undef _ctf_array_encoded
 916 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 917         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 918         if (_user) {                                                    \
 919                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 920         } else {                                                        \
 921                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 922         }
 923
 924 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 925 #undef _ctf_array_bitfield
 926 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 927         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 928         if (_user) {                                                    \
 929                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 930         } else {                                                        \
 931                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 932         }
 933 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 934 /*
 935  * For big endian, we need to byteswap into little endian.
 936  */
 937 #undef _ctf_array_bitfield
 938 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 939         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 940         {                                                               \
 941                 size_t _i;                                              \
 942                                                                         \
 943                 for (_i = 0; _i < (_length); _i++) {                    \
 944                         _type _tmp;                                     \
 945                                                                         \
 946                         if (_user) {                                    \
 947                                 if (get_user(_tmp, (_type *) _src + _i)) \
 948                                         _tmp = 0;                       \
 949                         } else {                                        \
 950                                 _tmp = ((_type *) _src)[_i];            \
 951                         }                                               \
 952                         switch (sizeof(_type)) {                        \
 953                         case 1:                                         \
 954                                 break;                                  \
 955                         case 2:                                         \
 956                                 _tmp = cpu_to_le16(_tmp);               \
 957                                 break;                                  \
 958                         case 4:                                         \
 959                                 _tmp = cpu_to_le32(_tmp);               \
 960                                 break;                                  \
 961                         case 8:                                         \
 962                                 _tmp = cpu_to_le64(_tmp);               \
 963                                 break;                                  \
 964                         default:                                        \
 965                                 BUG_ON(1);                              \
 966                         }                                               \
 967                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
 968                 }                                                       \
 969         }
 970 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 971
 972 #undef _ctf_sequence_encoded
 973 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,         \
 974                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 975         {                                                               \
 976                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx]; \
 977                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 978                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 979         }                                                               \
 980         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 981         if (_user) {                                                    \
 982                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 983                         sizeof(_type) * __get_dynamic_len(dest));       \
 984         } else {                                                        \
 985                 __chan->ops->event_write(&__ctx, _src,                  \
 986                         sizeof(_type) * __get_dynamic_len(dest));       \
 987         }
 988
 989 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 990 #undef _ctf_sequence_bitfield
 991 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 992                         _length_type, _src_length,              \
 993                         _user, _nowrite)                        \
 994         {                                                               \
 995                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 996                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 997                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 998         }                                                               \
 999         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
1000         if (_user) {                                                    \
1001                 __chan->ops->event_write_from_user(&__ctx, _src,        \
1002                         sizeof(_type) * __get_dynamic_len(dest));       \
1003         } else {                                                        \
1004                 __chan->ops->event_write(&__ctx, _src,                  \
1005                         sizeof(_type) * __get_dynamic_len(dest));       \
1006         }
1007 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
1008 /*
1009  * For big endian, we need to byteswap into little endian.
1010  */
1011 #undef _ctf_sequence_bitfield
1012 #define _ctf_sequence_bitfield(_type, _item, _src,              \
1013                         _length_type, _src_length,              \
1014                         _user, _nowrite)                        \
1015         {                                                       \
1016                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
1017                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
1018                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
1019         }                                                               \
1020         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
1021         {                                                               \
1022                 size_t _i, _length;                                     \
1023                                                                         \
1024                 _length = __get_dynamic_len(dest);                      \
1025                 for (_i = 0; _i < _length; _i++) {                      \
1026                         _type _tmp;                                     \
1027                                                                         \
1028                         if (_user) {                                    \
1029                                 if (get_user(_tmp, (_type *) _src + _i)) \
1030                                         _tmp = 0;                       \
1031                         } else {                                        \
1032                                 _tmp = ((_type *) _src)[_i];            \
1033                         }                                               \
1034                         switch (sizeof(_type)) {                        \
1035                         case 1:                                         \
1036                                 break;                                  \
1037                         case 2:                                         \
1038                                 _tmp = cpu_to_le16(_tmp);               \
1039                                 break;                                  \
1040                         case 4:                                         \
1041                                 _tmp = cpu_to_le32(_tmp);               \
1042                                 break;                                  \
1043                         case 8:                                         \
1044                                 _tmp = cpu_to_le64(_tmp);               \
1045                                 break;                                  \
1046                         default:                                        \
1047                                 BUG_ON(1);                              \
1048                         }                                               \
1049                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
1050                 }                                                       \
1051         }
1052 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
1053
1054 #undef _ctf_string
1055 #define _ctf_string(_item, _src, _user, _nowrite)                       \
1056         if (_user) {                                                    \
1057                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(*(_src))); \
1058                 __chan->ops->event_strcpy_from_user(&__ctx, _src,       \
1059                         __get_dynamic_len(dest));                       \
1060         } else {                                                        \
1061                 const char *__ctf_tmp_string =                          \
1062                         ((_src) ? (_src) : __LTTNG_NULL_STRING);        \
1063                 lib_ring_buffer_align_ctx(&__ctx,                       \
1064                         lttng_alignof(*__ctf_tmp_string));              \
1065                 __chan->ops->event_strcpy(&__ctx, __ctf_tmp_string,     \
1066                         __get_dynamic_len(dest));                       \
1067         }
1068
1069 #undef _ctf_enum
1070 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)           \
1071         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
1072
1073 #undef ctf_align
1074 #define ctf_align(_type)                                                \
1075         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));
1076
1077 #undef ctf_custom_field
1078 #define ctf_custom_field(_type, _item, _code)           _code
1079
1080 #undef ctf_custom_code
1081 #define ctf_custom_code(...)                                            \
1082         {                                                               \
1083                 __VA_ARGS__                                             \
1084         }
1085
1086 /* Beware: this get len actually consumes the len value */
1087 #undef __get_dynamic_len
1088 #define __get_dynamic_len(field)        this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx++]
1089
1090 #undef TP_PROTO
1091 #define TP_PROTO(...)   __VA_ARGS__
1092
1093 #undef TP_ARGS
1094 #define TP_ARGS(...)    __VA_ARGS__
1095
1096 #undef TP_FIELDS
1097 #define TP_FIELDS(...)  __VA_ARGS__
1098
1099 #undef TP_locvar
1100 #define TP_locvar(...)  __VA_ARGS__
1101
1102 #undef TP_code_pre
1103 #define TP_code_pre(...)        __VA_ARGS__
1104
1105 #undef TP_code_post
1106 #define TP_code_post(...)       __VA_ARGS__
1107
1108 /*
1109  * For state dump, check that "session" argument (mandatory) matches the
1110  * session this event belongs to. Ensures that we write state dump data only
1111  * into the started session, not into all sessions.
1112  */
1113 #ifdef TP_SESSION_CHECK
1114 #define _TP_SESSION_CHECK(session, csession)    (session == csession)
1115 #else /* TP_SESSION_CHECK */
1116 #define _TP_SESSION_CHECK(session, csession)    1
1117 #endif /* TP_SESSION_CHECK */
1118
1119 /*
1120  * Using twice size for filter stack data to hold size and pointer for
1121  * each field (worse case). For integers, max size required is 64-bit.
1122  * Same for double-precision floats. Those fit within
1123  * 2*sizeof(unsigned long) for all supported architectures.
1124  * Perform UNION (||) of filter runtime list.
1125  */
1126 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
1127 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
1128 static void __event_probe__##_name(void *__data, _proto)                      \
1129 {                                                                             \
1130         struct probe_local_vars { _locvar };                                  \
1131         struct lttng_event *__event = __data;                                 \
1132         struct lttng_probe_ctx __lttng_probe_ctx = {                                  \
1133                 .event = __event,                                             \
1134                 .interruptible = !irqs_disabled(),                            \
1135         };                                                                    \
1136         struct lttng_channel *__chan = __event->chan;                         \
1137         struct lttng_session *__session = __chan->session;                    \
1138         struct lib_ring_buffer_ctx __ctx;                                     \
1139         ssize_t __event_len;                                                  \
1140         size_t __event_align;                                                 \
1141         size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \
1142         union {                                                               \
1143                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
1144                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1145         } __stackvar;                                                         \
1146         int __ret;                                                            \
1147         struct probe_local_vars __tp_locvar;                                  \
1148         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
1149                         &__tp_locvar;                                         \
1150         struct lttng_id_tracker_rcu *__lf;                                    \
1151                                                                               \
1152         if (!_TP_SESSION_CHECK(session, __session))                           \
1153                 return;                                                       \
1154         if (unlikely(!READ_ONCE(__session->active)))                          \
1155                 return;                                                       \
1156         if (unlikely(!READ_ONCE(__chan->enabled)))                            \
1157                 return;                                                       \
1158         if (unlikely(!READ_ONCE(__event->enabled)))                           \
1159                 return;                                                       \
1160         __lf = lttng_rcu_dereference(__session->pid_tracker.p);               \
1161         if (__lf && likely(!lttng_id_tracker_lookup(__lf, current->tgid)))    \
1162                 return;                                                       \
1163         __lf = lttng_rcu_dereference(__session->vpid_tracker.p);              \
1164         if (__lf && likely(!lttng_id_tracker_lookup(__lf, task_tgid_vnr(current)))) \
1165                 return;                                                       \
1166         __lf = lttng_rcu_dereference(__session->uid_tracker.p);               \
1167         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1168                         lttng_current_uid())))                                \
1169                 return;                                                       \
1170         __lf = lttng_rcu_dereference(__session->vuid_tracker.p);              \
1171         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1172                         lttng_current_vuid())))                               \
1173                 return;                                                       \
1174         __lf = lttng_rcu_dereference(__session->gid_tracker.p);               \
1175         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1176                         lttng_current_gid())))                                \
1177                 return;                                                       \
1178         __lf = lttng_rcu_dereference(__session->vgid_tracker.p);              \
1179         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1180                         lttng_current_vgid())))                               \
1181                 return;                                                       \
1182         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
1183         __dynamic_len_idx = __orig_dynamic_len_offset;                        \
1184         _code_pre                                                             \
1185         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
1186                 struct lttng_bytecode_runtime *bc_runtime;                    \
1187                 int __filter_record = __event->has_enablers_without_bytecode; \
1188                                                                               \
1189                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
1190                                 tp_locvar, _args);                                    \
1191                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
1192                         if (unlikely(bc_runtime->filter(bc_runtime, &__lttng_probe_ctx,       \
1193                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) { \
1194                                 __filter_record = 1;                          \
1195                                 break;                                        \
1196                         }                                                     \
1197                 }                                                             \
1198                 if (likely(!__filter_record))                                 \
1199                         goto __post;                                          \
1200         }                                                                     \
1201         __event_len = __event_get_size__##_name(tp_locvar, _args);            \
1202         if (unlikely(__event_len < 0)) {                                      \
1203                 lib_ring_buffer_lost_event_too_big(__chan->chan);             \
1204                 goto __post;                                                  \
1205         }                                                                     \
1206         __event_align = __event_get_align__##_name(tp_locvar, _args);         \
1207         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
1208                                  __event_align, -1);                          \
1209         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1210         if (__ret < 0)                                                        \
1211                 goto __post;                                                  \
1212         _fields                                                               \
1213         __chan->ops->event_commit(&__ctx);                                    \
1214 __post:                                                                       \
1215         _code_post                                                            \
1216         barrier();      /* use before un-reserve. */                          \
1217         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1218         return;                                                               \
1219 }
1220
1221 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
1222 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
1223 static void __event_probe__##_name(void *__data)                              \
1224 {                                                                             \
1225         struct probe_local_vars { _locvar };                                  \
1226         struct lttng_event *__event = __data;                                 \
1227         struct lttng_probe_ctx __lttng_probe_ctx = {                                  \
1228                 .event = __event,                                             \
1229                 .interruptible = !irqs_disabled(),                            \
1230         };                                                                    \
1231         struct lttng_channel *__chan = __event->chan;                         \
1232         struct lttng_session *__session = __chan->session;                    \
1233         struct lib_ring_buffer_ctx __ctx;                                     \
1234         ssize_t __event_len;                                                  \
1235         size_t __event_align;                                                 \
1236         size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \
1237         union {                                                               \
1238                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
1239                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1240         } __stackvar;                                                         \
1241         int __ret;                                                            \
1242         struct probe_local_vars __tp_locvar;                                  \
1243         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
1244                         &__tp_locvar;                                         \
1245         struct lttng_id_tracker_rcu *__lf;                                    \
1246                                                                               \
1247         if (!_TP_SESSION_CHECK(session, __session))                           \
1248                 return;                                                       \
1249         if (unlikely(!READ_ONCE(__session->active)))                          \
1250                 return;                                                       \
1251         if (unlikely(!READ_ONCE(__chan->enabled)))                            \
1252                 return;                                                       \
1253         if (unlikely(!READ_ONCE(__event->enabled)))                           \
1254                 return;                                                       \
1255         __lf = lttng_rcu_dereference(__session->pid_tracker.p);               \
1256         if (__lf && likely(!lttng_id_tracker_lookup(__lf, current->tgid)))    \
1257                 return;                                                       \
1258         __lf = lttng_rcu_dereference(__session->vpid_tracker.p);              \
1259         if (__lf && likely(!lttng_id_tracker_lookup(__lf, task_tgid_vnr(current)))) \
1260                 return;                                                       \
1261         __lf = lttng_rcu_dereference(__session->uid_tracker.p);               \
1262         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1263                         lttng_current_uid())))                                \
1264                 return;                                                       \
1265         __lf = lttng_rcu_dereference(__session->vuid_tracker.p);              \
1266         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1267                         lttng_current_vuid())))                               \
1268                 return;                                                       \
1269         __lf = lttng_rcu_dereference(__session->gid_tracker.p);               \
1270         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1271                         lttng_current_gid())))                                \
1272                 return;                                                       \
1273         __lf = lttng_rcu_dereference(__session->vgid_tracker.p);              \
1274         if (__lf && likely(!lttng_id_tracker_lookup(__lf,                     \
1275                         lttng_current_vgid())))                               \
1276                 return;                                                       \
1277         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
1278         __dynamic_len_idx = __orig_dynamic_len_offset;                        \
1279         _code_pre                                                             \
1280         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
1281                 struct lttng_bytecode_runtime *bc_runtime;                    \
1282                 int __filter_record = __event->has_enablers_without_bytecode; \
1283                                                                               \
1284                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
1285                                 tp_locvar);                                   \
1286                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
1287                         if (unlikely(bc_runtime->filter(bc_runtime, &__lttng_probe_ctx, \
1288                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) { \
1289                                 __filter_record = 1;                          \
1290                                 break;                                        \
1291                         }                                                     \
1292                 }                                                             \
1293                 if (likely(!__filter_record))                                 \
1294                         goto __post;                                          \
1295         }                                                                     \
1296         __event_len = __event_get_size__##_name(tp_locvar);                   \
1297         if (unlikely(__event_len < 0)) {                                      \
1298                 lib_ring_buffer_lost_event_too_big(__chan->chan);             \
1299                 goto __post;                                                  \
1300         }                                                                     \
1301         __event_align = __event_get_align__##_name(tp_locvar);                \
1302         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
1303                                  __event_align, -1);                          \
1304         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1305         if (__ret < 0)                                                        \
1306                 goto __post;                                                  \
1307         _fields                                                               \
1308         __chan->ops->event_commit(&__ctx);                                    \
1309 __post:                                                                       \
1310         _code_post                                                            \
1311         barrier();      /* use before un-reserve. */                          \
1312         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1313         return;                                                               \
1314 }
1315
1316 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1317
1318 #undef __get_dynamic_len
1319
1320 /*
1321  * Stage 7 of the trace events.
1322  *
1323  * Create event descriptions.
1324  */
1325
1326 /* Named field types must be defined in lttng-types.h */
1327
1328 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1329
1330 #ifndef TP_PROBE_CB
1331 #define TP_PROBE_CB(_template)  &__event_probe__##_template
1332 #endif
1333
1334 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1335 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)      \
1336 static const struct lttng_event_desc __event_desc___##_map = {          \
1337         .fields = __event_fields___##_template,                         \
1338         .name = #_map,                                                  \
1339         .kname = #_name,                                                \
1340         .probe_callback = (void *) TP_PROBE_CB(_template),              \
1341         .nr_fields = ARRAY_SIZE(__event_fields___##_template),          \
1342         .owner = THIS_MODULE,                                           \
1343 };
1344
1345 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1346 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1347         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1348
1349 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1350
1351 /*
1352  * Stage 8 of the trace events.
1353  *
1354  * Create an array of event description pointers.
1355  */
1356
1357 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1358
1359 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1360 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
1361                 &__event_desc___##_map,
1362
1363 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1364 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1365         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1366
1367 #define TP_ID1(_token, _system) _token##_system
1368 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1369
1370 static const struct lttng_event_desc *TP_ID(__event_desc___, TRACE_SYSTEM)[] = {
1371 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1372 };
1373
1374 #undef TP_ID1
1375 #undef TP_ID
1376
1377 /*
1378  * Stage 9 of the trace events.
1379  *
1380  * Create a toplevel descriptor for the whole probe.
1381  */
1382
1383 #define TP_ID1(_token, _system) _token##_system
1384 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1385
1386 /* non-const because list head will be modified when registered. */
1387 static __used struct lttng_probe_desc TP_ID(__probe_desc___, TRACE_SYSTEM) = {
1388         .provider = __stringify(TRACE_SYSTEM),
1389         .event_desc = TP_ID(__event_desc___, TRACE_SYSTEM),
1390         .nr_events = ARRAY_SIZE(TP_ID(__event_desc___, TRACE_SYSTEM)),
1391         .head = { NULL, NULL },
1392         .lazy_init_head = { NULL, NULL },
1393         .lazy = 0,
1394 };
1395
1396 #undef TP_ID1
1397 #undef TP_ID
1398
1399 /*
1400  * Stage 10 of the trace events.
1401  *
1402  * Register/unregister probes at module load/unload.
1403  */
1404
1405 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1406
1407 #define TP_ID1(_token, _system) _token##_system
1408 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1409 #define module_init_eval1(_token, _system)      module_init(_token##_system)
1410 #define module_init_eval(_token, _system)       module_init_eval1(_token, _system)
1411 #define module_exit_eval1(_token, _system)      module_exit(_token##_system)
1412 #define module_exit_eval(_token, _system)       module_exit_eval1(_token, _system)
1413
1414 #ifndef TP_MODULE_NOINIT
1415 static int TP_ID(__lttng_events_init__, TRACE_SYSTEM)(void)
1416 {
1417         wrapper_vmalloc_sync_mappings();
1418         return lttng_probe_register(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1419 }
1420
1421 static void TP_ID(__lttng_events_exit__, TRACE_SYSTEM)(void)
1422 {
1423         lttng_probe_unregister(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1424 }
1425
1426 #ifndef TP_MODULE_NOAUTOLOAD
1427 module_init_eval(__lttng_events_init__, TRACE_SYSTEM);
1428 module_exit_eval(__lttng_events_exit__, TRACE_SYSTEM);
1429 #endif
1430
1431 #endif
1432
1433 #undef module_init_eval
1434 #undef module_exit_eval
1435 #undef TP_ID1
1436 #undef TP_ID
1437
1438 #undef TP_PROTO
1439 #undef TP_ARGS