probes/lttng-tracepoint-event-impl.h

   1 /* SPDX-License-Identifier: (GPL-2.0 or LGPL-2.1)
   2  *
   3  * lttng-tracepoint-event-impl.h
   4  *
   5  * Copyright (C) 2009 Steven Rostedt <rostedt@goodmis.org>
   6  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   7  */
   8
   9 #include <linux/uaccess.h>
  10 #include <linux/debugfs.h>
  11 #include <linux/rculist.h>
  12 #include <asm/byteorder.h>
  13 #include <linux/swab.h>
  14
  15 #include <probes/lttng.h>
  16 #include <probes/lttng-types.h>
  17 #include <probes/lttng-probe-user.h>
  18 #include <wrapper/vmalloc.h>    /* for wrapper_vmalloc_sync_all() */
  19 #include <wrapper/ringbuffer/frontend_types.h>
  20 #include <wrapper/ringbuffer/backend.h>
  21 #include <wrapper/rcu.h>
  22 #include <lttng-events.h>
  23 #include <lttng-tracer-core.h>
  24 #include <lttng-tp-mempool.h>
  25
  26 #define __LTTNG_NULL_STRING     "(null)"
  27
  28 /*
  29  * Macro declarations used for all stages.
  30  */
  31
  32 /*
  33  * LTTng name mapping macros. LTTng remaps some of the kernel events to
  34  * enforce name-spacing.
  35  */
  36 #undef LTTNG_TRACEPOINT_EVENT_MAP
  37 #define LTTNG_TRACEPOINT_EVENT_MAP(name, map, proto, args, fields) \
  38         LTTNG_TRACEPOINT_EVENT_CLASS(map,                               \
  39                              PARAMS(proto),                             \
  40                              PARAMS(args),                              \
  41                              PARAMS(fields))                            \
  42         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  43
  44 #undef LTTNG_TRACEPOINT_EVENT_MAP_NOARGS
  45 #define LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, map, fields)            \
  46         LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(map,                        \
  47                              PARAMS(fields))                            \
  48         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(map, name, map)
  49
  50 #undef LTTNG_TRACEPOINT_EVENT_CODE_MAP
  51 #define LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, map, proto, args, _locvar, _code_pre, fields, _code_post) \
  52         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(map,                          \
  53                              PARAMS(proto),                             \
  54                              PARAMS(args),                              \
  55                              PARAMS(_locvar),                           \
  56                              PARAMS(_code_pre),                         \
  57                              PARAMS(fields),                            \
  58                              PARAMS(_code_post))                        \
  59         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  60
  61 #undef LTTNG_TRACEPOINT_EVENT_CODE
  62 #define LTTNG_TRACEPOINT_EVENT_CODE(name, proto, args, _locvar, _code_pre, fields, _code_post) \
  63         LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, name,                     \
  64                              PARAMS(proto),                             \
  65                              PARAMS(args),                              \
  66                              PARAMS(_locvar),                           \
  67                              PARAMS(_code_pre),                         \
  68                              PARAMS(fields),                            \
  69                              PARAMS(_code_post))
  70
  71 /*
  72  * LTTNG_TRACEPOINT_EVENT_CLASS can be used to add a generic function
  73  * handlers for events. That is, if all events have the same parameters
  74  * and just have distinct trace points.  Each tracepoint can be defined
  75  * with LTTNG_TRACEPOINT_EVENT_INSTANCE and that will map the
  76  * LTTNG_TRACEPOINT_EVENT_CLASS to the tracepoint.
  77  *
  78  * LTTNG_TRACEPOINT_EVENT is a one to one mapping between tracepoint and
  79  * template.
  80  */
  81
  82 #undef LTTNG_TRACEPOINT_EVENT
  83 #define LTTNG_TRACEPOINT_EVENT(name, proto, args, fields)               \
  84         LTTNG_TRACEPOINT_EVENT_MAP(name, name,                          \
  85                         PARAMS(proto),                                  \
  86                         PARAMS(args),                                   \
  87                         PARAMS(fields))
  88
  89 #undef LTTNG_TRACEPOINT_EVENT_NOARGS
  90 #define LTTNG_TRACEPOINT_EVENT_NOARGS(name, fields)                     \
  91         LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, name, PARAMS(fields))
  92
  93 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE
  94 #define LTTNG_TRACEPOINT_EVENT_INSTANCE(template, name, proto, args)    \
  95         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(template, name, name, PARAMS(proto), PARAMS(args))
  96
  97 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS
  98 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS(template, name)  \
  99         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(template, name, name)
 100
 101 #undef LTTNG_TRACEPOINT_EVENT_CLASS
 102 #define LTTNG_TRACEPOINT_EVENT_CLASS(_name, _proto, _args, _fields) \
 103         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(_proto), PARAMS(_args), , , \
 104                 PARAMS(_fields), )
 105
 106 #undef LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS
 107 #define LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(_name, _fields) \
 108         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, , , PARAMS(_fields), )
 109
 110
 111 /*
 112  * Stage 1 of the trace events.
 113  *
 114  * Create dummy trace calls for each events, verifying that the LTTng module
 115  * instrumentation headers match the kernel arguments. Will be optimized
 116  * out by the compiler.
 117  */
 118
 119 /* Reset all macros within TRACEPOINT_EVENT */
 120 #include <probes/lttng-events-reset.h>
 121
 122 #undef TP_PROTO
 123 #define TP_PROTO(...)   __VA_ARGS__
 124
 125 #undef TP_ARGS
 126 #define TP_ARGS(...)    __VA_ARGS__
 127
 128 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 129 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 130 void trace_##_name(_proto);
 131
 132 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 133 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 134 void trace_##_name(void);
 135
 136 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 137
 138 /*
 139  * Stage 1.1 of the trace events.
 140  *
 141  * Create dummy trace prototypes for each event class, and for each used
 142  * template. This will allow checking whether the prototypes from the
 143  * class and the instance using the class actually match.
 144  */
 145
 146 #include <probes/lttng-events-reset.h>  /* Reset all macros within TRACE_EVENT */
 147
 148 #undef TP_PROTO
 149 #define TP_PROTO(...)   __VA_ARGS__
 150
 151 #undef TP_ARGS
 152 #define TP_ARGS(...)    __VA_ARGS__
 153
 154 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 155 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 156 void __event_template_proto___##_template(_proto);
 157
 158 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 159 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 160 void __event_template_proto___##_template(void);
 161
 162 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 163 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 164 void __event_template_proto___##_name(_proto);
 165
 166 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 167 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 168 void __event_template_proto___##_name(void);
 169
 170 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 171
 172 /*
 173  * Stage 1.2 of tracepoint event generation
 174  *
 175  * Unfolding the enums
 176  */
 177 #include <probes/lttng-events-reset.h>  /* Reset all macros within TRACE_EVENT */
 178
 179 /* Enumeration entry (single value) */
 180 #undef ctf_enum_value
 181 #define ctf_enum_value(_string, _value)                                 \
 182         {                                                               \
 183                 .start = {                                              \
 184                         .signedness = lttng_is_signed_type(__typeof__(_value)), \
 185                         .value = lttng_is_signed_type(__typeof__(_value)) ? \
 186                                 (long long) (_value) : (_value),        \
 187                 },                                                      \
 188                 .end = {                                                \
 189                         .signedness = lttng_is_signed_type(__typeof__(_value)), \
 190                         .value = lttng_is_signed_type(__typeof__(_value)) ? \
 191                                 (long long) (_value) : (_value),        \
 192                 },                                                      \
 193                 .string = (_string),                                    \
 194         },
 195
 196 /* Enumeration entry (range) */
 197 #undef ctf_enum_range
 198 #define ctf_enum_range(_string, _range_start, _range_end)               \
 199         {                                                               \
 200                 .start = {                                              \
 201                         .signedness = lttng_is_signed_type(__typeof__(_range_start)), \
 202                         .value = lttng_is_signed_type(__typeof__(_range_start)) ? \
 203                                 (long long) (_range_start) : (_range_start), \
 204                 },                                                      \
 205                 .end = {                                                \
 206                         .signedness = lttng_is_signed_type(__typeof__(_range_end)), \
 207                         .value = lttng_is_signed_type(__typeof__(_range_end)) ? \
 208                                 (long long) (_range_end) : (_range_end), \
 209                 },                                                      \
 210                 .string = (_string),                                    \
 211         },
 212
 213 /* Enumeration entry (automatic value; follows the rules of CTF) */
 214 #undef ctf_enum_auto
 215 #define ctf_enum_auto(_string)                                  \
 216         {                                                               \
 217                 .start = {                                              \
 218                         .signedness = -1,                               \
 219                         .value = -1,                                    \
 220                 },                                                      \
 221                 .end = {                                                \
 222                         .signedness = -1,                               \
 223                         .value = -1,                                    \
 224                 },                                                      \
 225                 .string = (_string),                                    \
 226                 .options = {                                            \
 227                         .is_auto = 1,                                   \
 228                 }                                                       \
 229         },
 230
 231 #undef TP_ENUM_VALUES
 232 #define TP_ENUM_VALUES(...)                                             \
 233         __VA_ARGS__
 234
 235 #undef LTTNG_TRACEPOINT_ENUM
 236 #define LTTNG_TRACEPOINT_ENUM(_name, _values)                           \
 237         const struct lttng_enum_entry __enum_values__##_name[] = { \
 238                 _values                                                 \
 239         };
 240
 241 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 242
 243 /*
 244  * Stage 2 of the trace events.
 245  *
 246  * Create event field type metadata section.
 247  * Each event produce an array of fields.
 248  */
 249
 250 /* Reset all macros within TRACEPOINT_EVENT */
 251 #include <probes/lttng-events-reset.h>
 252 #include <probes/lttng-events-write.h>
 253 #include <probes/lttng-events-nowrite.h>
 254
 255 #undef _ctf_integer_ext
 256 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 257         {                                                       \
 258           .name = #_item,                                       \
 259           .type = __type_integer(_type, 0, 0, -1, _byte_order, _base, none),\
 260           .nowrite = _nowrite,                                  \
 261           .user = _user,                                        \
 262         },
 263
 264 #undef _ctf_array_encoded
 265 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 266         {                                                       \
 267           .name = #_item,                                       \
 268           .type =                                               \
 269                 {                                               \
 270                   .atype = atype_array,                         \
 271                   .u =                                          \
 272                         {                                       \
 273                           .array =                              \
 274                                 {                               \
 275                                   .elem_type = __type_integer(_type, 0, 0, 0, _byte_order, _base, _encoding), \
 276                                   .length = _length,            \
 277                                 }                               \
 278                         }                                       \
 279                 },                                              \
 280           .nowrite = _nowrite,                                  \
 281           .user = _user,                                        \
 282         },
 283
 284 #undef _ctf_array_bitfield
 285 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 286         {                                                       \
 287           .name = #_item,                                       \
 288           .type =                                               \
 289                 {                                               \
 290                   .atype = atype_array_bitfield,                \
 291                   .u =                                          \
 292                         {                                       \
 293                           .array =                              \
 294                                 {                               \
 295                                   .elem_type = __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none), \
 296                                   .length = (_length) * sizeof(_type) * CHAR_BIT, \
 297                                   .elem_alignment = lttng_alignof(_type), \
 298                                 }                               \
 299                         }                                       \
 300                 },                                              \
 301           .nowrite = _nowrite,                                  \
 302           .user = _user,                                        \
 303         },
 304
 305
 306 #undef _ctf_sequence_encoded
 307 #define _ctf_sequence_encoded(_type, _item, _src,               \
 308                         _length_type, _src_length, _encoding,   \
 309                         _byte_order, _base, _user, _nowrite)    \
 310         {                                                       \
 311           .name = #_item,                                       \
 312           .type =                                               \
 313                 {                                               \
 314                   .atype = atype_sequence,                      \
 315                   .u =                                          \
 316                         {                                       \
 317                           .sequence =                           \
 318                                 {                               \
 319                                   .length_type = __type_integer(_length_type, 0, 0, 0, __BYTE_ORDER, 10, none), \
 320                                   .elem_type = __type_integer(_type, 0, 0, -1, _byte_order, _base, _encoding), \
 321                                 },                              \
 322                         },                                      \
 323                 },                                              \
 324           .nowrite = _nowrite,                                  \
 325           .user = _user,                                        \
 326         },
 327
 328 #undef _ctf_sequence_bitfield
 329 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 330                         _length_type, _src_length,              \
 331                         _user, _nowrite)                        \
 332         {                                                       \
 333           .name = #_item,                                       \
 334           .type =                                               \
 335                 {                                               \
 336                   .atype = atype_sequence_bitfield,             \
 337                   .u =                                          \
 338                         {                                       \
 339                           .sequence =                           \
 340                                 {                               \
 341                                   .length_type = __type_integer(_length_type, 0, 0, 0, __BYTE_ORDER, 10, none), \
 342                                   .elem_type = __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none), \
 343                                   .elem_alignment = lttng_alignof(_type), \
 344                                 },                              \
 345                         },                                      \
 346                 },                                              \
 347           .nowrite = _nowrite,                                  \
 348           .user = _user,                                        \
 349         },
 350
 351 #undef _ctf_string
 352 #define _ctf_string(_item, _src, _user, _nowrite)               \
 353         {                                                       \
 354           .name = #_item,                                       \
 355           .type =                                               \
 356                 {                                               \
 357                   .atype = atype_string,                        \
 358                   .u =                                          \
 359                         {                                       \
 360                           .basic = { .string = { .encoding = lttng_encode_UTF8 } } \
 361                         },                                      \
 362                 },                                              \
 363           .nowrite = _nowrite,                                  \
 364           .user = _user,                                        \
 365         },
 366
 367 #undef _ctf_enum
 368 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)   \
 369         {                                                       \
 370                 .name = #_item,                                 \
 371                 .type = {                                       \
 372                         .atype = atype_enum,                    \
 373                         .u = {                                  \
 374                                 .basic = {                      \
 375                                         .enumeration = {        \
 376                                                 .desc = &__enum_##_name, \
 377                                                 .container_type = { \
 378                                                         .size = sizeof(_type) * CHAR_BIT, \
 379                                                         .alignment = lttng_alignof(_type) * CHAR_BIT, \
 380                                                         .signedness = lttng_is_signed_type(_type), \
 381                                                         .reverse_byte_order = 0, \
 382                                                         .base = 10, \
 383                                                         .encoding = lttng_encode_none, \
 384                                                 },              \
 385                                         },                      \
 386                                  },                             \
 387                         },                                      \
 388                 },                                              \
 389                 .nowrite = _nowrite,                            \
 390                 .user = _user,                                  \
 391         },
 392
 393 #undef ctf_custom_field
 394 #define ctf_custom_field(_type, _item, _code)                   \
 395         {                                                       \
 396                 .name = #_item,                                 \
 397                 .type = { _type },                              \
 398                 .nowrite = 0,                                   \
 399                 .user = 0,                                      \
 400         },
 401
 402 #undef ctf_custom_type
 403 #define ctf_custom_type(...)    __VA_ARGS__
 404
 405 #undef TP_FIELDS
 406 #define TP_FIELDS(...)  __VA_ARGS__     /* Only one used in this phase */
 407
 408 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 409 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 410         static const struct lttng_event_field __event_fields___##_name[] = { \
 411                 _fields                                                      \
 412         };
 413
 414 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 415 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 416         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, PARAMS(_fields), _code_post)
 417
 418 #undef LTTNG_TRACEPOINT_ENUM
 419 #define LTTNG_TRACEPOINT_ENUM(_name, _values)                                           \
 420         static const struct lttng_enum_desc __enum_##_name = {                          \
 421                 .name = #_name,                                                         \
 422                 .entries = __enum_values__##_name,                                      \
 423                 .nr_entries = ARRAY_SIZE(__enum_values__##_name),                       \
 424         };
 425
 426 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 427
 428 /*
 429  * Stage 3 of the trace events.
 430  *
 431  * Create probe callback prototypes.
 432  */
 433
 434 /* Reset all macros within TRACEPOINT_EVENT */
 435 #include <probes/lttng-events-reset.h>
 436
 437 #undef TP_PROTO
 438 #define TP_PROTO(...)   __VA_ARGS__
 439
 440 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 441 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 442 static void __event_probe__##_name(void *__data, _proto);
 443
 444 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 445 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 446 static void __event_probe__##_name(void *__data);
 447
 448 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 449
 450 /*
 451  * Stage 4 of the trace events.
 452  *
 453  * Create static inline function that calculates event size.
 454  */
 455
 456 /* Reset all macros within TRACEPOINT_EVENT */
 457 #include <probes/lttng-events-reset.h>
 458 #include <probes/lttng-events-write.h>
 459
 460 #undef _ctf_integer_ext
 461 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 462         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 463         __event_len += sizeof(_type);
 464
 465 #undef _ctf_array_encoded
 466 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 467         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 468         __event_len += sizeof(_type) * (_length);
 469
 470 #undef _ctf_array_bitfield
 471 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 472         _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
 473
 474 #undef _ctf_sequence_encoded
 475 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 476                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 477         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_length_type)); \
 478         __event_len += sizeof(_length_type);                                   \
 479         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 480         {                                                                               \
 481                 size_t __seqlen = (_src_length);                                        \
 482                                                                                         \
 483                 if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 484                         goto error;                                                     \
 485                 barrier();      /* reserve before use. */                               \
 486                 this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = __seqlen; \
 487                 __event_len += sizeof(_type) * __seqlen;                                \
 488         }
 489
 490 #undef _ctf_sequence_bitfield
 491 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 492                         _length_type, _src_length,              \
 493                         _user, _nowrite)                        \
 494         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 495                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 496
 497 /*
 498  * ctf_user_string includes \0. If returns 0, it faulted, so we set size to
 499  * 1 (\0 only).
 500  */
 501 #undef _ctf_string
 502 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 503         if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 504                 goto error;                                                    \
 505         barrier();      /* reserve before use. */                              \
 506         if (_user) {                                                           \
 507                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 508                         max_t(size_t, lttng_strlen_user_inatomic(_src), 1);    \
 509         } else {                                                               \
 510                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 511                         strlen((_src) ? (_src) : __LTTNG_NULL_STRING) + 1; \
 512         }
 513
 514 #undef _ctf_enum
 515 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)                  \
 516         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 517
 518 #undef ctf_align
 519 #define ctf_align(_type)                                                \
 520         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type));
 521
 522 #undef ctf_custom_field
 523 #define ctf_custom_field(_type, _item, _code)                           \
 524         {                                                               \
 525                 _code                                                   \
 526         }
 527
 528 #undef ctf_custom_code
 529 #define ctf_custom_code(...)            __VA_ARGS__
 530
 531 #undef TP_PROTO
 532 #define TP_PROTO(...)   __VA_ARGS__
 533
 534 #undef TP_FIELDS
 535 #define TP_FIELDS(...)  __VA_ARGS__
 536
 537 #undef TP_locvar
 538 #define TP_locvar(...)  __VA_ARGS__
 539
 540 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 541 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 542 static inline ssize_t __event_get_size__##_name(void *__tp_locvar, _proto)     \
 543 {                                                                             \
 544         size_t __event_len = 0;                                               \
 545         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 546         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 547                                                                               \
 548         _fields                                                               \
 549         return __event_len;                                                   \
 550                                                                               \
 551 error:                                                                        \
 552         __attribute__((unused));                                              \
 553         return -1;                                                            \
 554 }
 555
 556 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 557 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 558 static inline ssize_t __event_get_size__##_name(void *__tp_locvar)            \
 559 {                                                                             \
 560         size_t __event_len = 0;                                               \
 561         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 562         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 563                                                                               \
 564         _fields                                                               \
 565         return __event_len;                                                   \
 566                                                                               \
 567 error:                                                                        \
 568         __attribute__((unused));                                              \
 569         return -1;                                                            \
 570 }
 571
 572 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 573
 574
 575 /*
 576  * Stage 4.1 of tracepoint event generation.
 577  *
 578  * Create static inline function that layout the filter stack data.
 579  * We make both write and nowrite data available to the filter.
 580  */
 581
 582 /* Reset all macros within TRACEPOINT_EVENT */
 583 #include <probes/lttng-events-reset.h>
 584 #include <probes/lttng-events-write.h>
 585 #include <probes/lttng-events-nowrite.h>
 586
 587 #undef _ctf_integer_ext_fetched
 588 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 589         if (lttng_is_signed_type(_type)) {                                     \
 590                 int64_t __ctf_tmp_int64;                                       \
 591                 switch (sizeof(_type)) {                                       \
 592                 case 1:                                                        \
 593                 {                                                              \
 594                         union { _type t; int8_t v; } __tmp = { (_type) (_src) }; \
 595                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 596                         break;                                                 \
 597                 }                                                              \
 598                 case 2:                                                        \
 599                 {                                                              \
 600                         union { _type t; int16_t v; } __tmp = { (_type) (_src) }; \
 601                         if (_byte_order != __BYTE_ORDER)                       \
 602                                 __swab16s(&__tmp.v);                           \
 603                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 604                         break;                                                 \
 605                 }                                                              \
 606                 case 4:                                                        \
 607                 {                                                              \
 608                         union { _type t; int32_t v; } __tmp = { (_type) (_src) }; \
 609                         if (_byte_order != __BYTE_ORDER)                       \
 610                                 __swab32s(&__tmp.v);                           \
 611                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 612                         break;                                                 \
 613                 }                                                              \
 614                 case 8:                                                        \
 615                 {                                                              \
 616                         union { _type t; int64_t v; } __tmp = { (_type) (_src) }; \
 617                         if (_byte_order != __BYTE_ORDER)                       \
 618                                 __swab64s(&__tmp.v);                           \
 619                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 620                         break;                                                 \
 621                 }                                                              \
 622                 default:                                                       \
 623                         BUG_ON(1);                                             \
 624                 };                                                             \
 625                 memcpy(__stack_data, &__ctf_tmp_int64, sizeof(int64_t));       \
 626         } else {                                                               \
 627                 uint64_t __ctf_tmp_uint64;                                     \
 628                 switch (sizeof(_type)) {                                       \
 629                 case 1:                                                        \
 630                 {                                                              \
 631                         union { _type t; uint8_t v; } __tmp = { (_type) (_src) }; \
 632                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 633                         break;                                                 \
 634                 }                                                              \
 635                 case 2:                                                        \
 636                 {                                                              \
 637                         union { _type t; uint16_t v; } __tmp = { (_type) (_src) }; \
 638                         if (_byte_order != __BYTE_ORDER)                       \
 639                                 __swab16s(&__tmp.v);                           \
 640                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 641                         break;                                                 \
 642                 }                                                              \
 643                 case 4:                                                        \
 644                 {                                                              \
 645                         union { _type t; uint32_t v; } __tmp = { (_type) (_src) }; \
 646                         if (_byte_order != __BYTE_ORDER)                       \
 647                                 __swab32s(&__tmp.v);                           \
 648                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 649                         break;                                                 \
 650                 }                                                              \
 651                 case 8:                                                        \
 652                 {                                                              \
 653                         union { _type t; uint64_t v; } __tmp = { (_type) (_src) }; \
 654                         if (_byte_order != __BYTE_ORDER)                       \
 655                                 __swab64s(&__tmp.v);                           \
 656                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 657                         break;                                                 \
 658                 }                                                              \
 659                 default:                                                       \
 660                         BUG_ON(1);                                             \
 661                 };                                                             \
 662                 memcpy(__stack_data, &__ctf_tmp_uint64, sizeof(uint64_t));     \
 663         }                                                                      \
 664         __stack_data += sizeof(int64_t);
 665
 666 #undef _ctf_integer_ext_isuser0
 667 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 668         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 669
 670 #undef _ctf_integer_ext_isuser1
 671 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 672 {                                                                                       \
 673         union {                                                                         \
 674                 char __array[sizeof(_user_src)];                                        \
 675                 __typeof__(_user_src) __v;                                              \
 676         } __tmp_fetch;                                                                  \
 677         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 678                                 &(_user_src), sizeof(_user_src)))                       \
 679                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 680         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 681 }
 682
 683 #undef _ctf_integer_ext
 684 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 685         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 686
 687 #undef _ctf_array_encoded
 688 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 689         {                                                                      \
 690                 unsigned long __ctf_tmp_ulong = (unsigned long) (_length);     \
 691                 const void *__ctf_tmp_ptr = (_src);                            \
 692                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 693                 __stack_data += sizeof(unsigned long);                         \
 694                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 695                 __stack_data += sizeof(void *);                                \
 696         }
 697
 698 #undef _ctf_array_bitfield
 699 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 700         _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
 701
 702 #undef _ctf_sequence_encoded
 703 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                \
 704                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 705         {                                                                      \
 706                 unsigned long __ctf_tmp_ulong = (unsigned long) (_src_length); \
 707                 const void *__ctf_tmp_ptr = (_src);                            \
 708                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 709                 __stack_data += sizeof(unsigned long);                         \
 710                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 711                 __stack_data += sizeof(void *);                                \
 712         }
 713
 714 #undef _ctf_sequence_bitfield
 715 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 716                         _length_type, _src_length,              \
 717                         _user, _nowrite)                        \
 718         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 719                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 720
 721 #undef _ctf_string
 722 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 723         {                                                                      \
 724                 const void *__ctf_tmp_ptr =                                    \
 725                         ((_src) ? (_src) : __LTTNG_NULL_STRING);               \
 726                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 727                 __stack_data += sizeof(void *);                                \
 728         }
 729
 730 #undef _ctf_enum
 731 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)                  \
 732         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 733
 734 #undef TP_PROTO
 735 #define TP_PROTO(...) __VA_ARGS__
 736
 737 #undef TP_FIELDS
 738 #define TP_FIELDS(...) __VA_ARGS__
 739
 740 #undef TP_locvar
 741 #define TP_locvar(...)  __VA_ARGS__
 742
 743 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 744 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 745 static inline                                                                 \
 746 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 747                 void *__tp_locvar)                                            \
 748 {                                                                             \
 749         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 750                                                                               \
 751         _fields                                                               \
 752 }
 753
 754 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 755 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 756 static inline                                                                 \
 757 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 758                 void *__tp_locvar, _proto)                                    \
 759 {                                                                             \
 760         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 761                                                                               \
 762         _fields                                                               \
 763 }
 764
 765 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 766
 767 /*
 768  * Stage 5 of the trace events.
 769  *
 770  * Create static inline function that calculates event payload alignment.
 771  */
 772
 773 /* Reset all macros within TRACEPOINT_EVENT */
 774 #include <probes/lttng-events-reset.h>
 775 #include <probes/lttng-events-write.h>
 776
 777 #undef _ctf_integer_ext
 778 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 779         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 780
 781 #undef _ctf_array_encoded
 782 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 783         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 784
 785 #undef _ctf_array_bitfield
 786 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 787         _ctf_array_encoded(_type, _item, _src, _length, none, __LITTLE_ENDIAN, 0, _user, _nowrite)
 788
 789 #undef _ctf_sequence_encoded
 790 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 791                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 792         __event_align = max_t(size_t, __event_align, lttng_alignof(_length_type)); \
 793         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 794
 795 #undef _ctf_sequence_bitfield
 796 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 797                         _length_type, _src_length,              \
 798                         _user, _nowrite)                        \
 799         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 800                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 801
 802 #undef _ctf_string
 803 #define _ctf_string(_item, _src, _user, _nowrite)
 804
 805 #undef _ctf_enum
 806 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)   \
 807         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 808
 809 #undef ctf_align
 810 #define ctf_align(_type)                                                \
 811         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 812
 813 #undef TP_PROTO
 814 #define TP_PROTO(...)   __VA_ARGS__
 815
 816 #undef TP_FIELDS
 817 #define TP_FIELDS(...)  __VA_ARGS__
 818
 819 #undef TP_locvar
 820 #define TP_locvar(...)  __VA_ARGS__
 821
 822 #undef ctf_custom_field
 823 #define ctf_custom_field(_type, _item, _code)           _code
 824
 825 #undef ctf_custom_code
 826 #define ctf_custom_code(...)                                            \
 827         {                                                               \
 828                 __VA_ARGS__                                             \
 829         }
 830
 831 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 832 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 833 static inline size_t __event_get_align__##_name(void *__tp_locvar, _proto)    \
 834 {                                                                             \
 835         size_t __event_align = 1;                                             \
 836         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 837                                                                               \
 838         _fields                                                               \
 839         return __event_align;                                                 \
 840 }
 841
 842 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 843 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 844 static inline size_t __event_get_align__##_name(void *__tp_locvar)            \
 845 {                                                                             \
 846         size_t __event_align = 1;                                             \
 847         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 848                                                                               \
 849         _fields                                                               \
 850         return __event_align;                                                 \
 851 }
 852
 853 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 854
 855 /*
 856  * Stage 6 of tracepoint event generation.
 857  *
 858  * Create the probe function. This function calls event size calculation
 859  * and writes event data into the buffer.
 860  */
 861
 862 /* Reset all macros within TRACEPOINT_EVENT */
 863 #include <probes/lttng-events-reset.h>
 864 #include <probes/lttng-events-write.h>
 865
 866 #undef _ctf_integer_ext_fetched
 867 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 868         {                                                               \
 869                 _type __tmp = _src;                                     \
 870                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(__tmp));\
 871                 __chan->ops->event_write(&__ctx, &__tmp, sizeof(__tmp));\
 872         }
 873
 874 #undef _ctf_integer_ext_isuser0
 875 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 876         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 877
 878 #undef _ctf_integer_ext_isuser1
 879 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 880 {                                                                              \
 881         union {                                                                         \
 882                 char __array[sizeof(_user_src)];                                        \
 883                 __typeof__(_user_src) __v;                                              \
 884         } __tmp_fetch;                                                                  \
 885         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 886                                 &(_user_src), sizeof(_user_src)))                       \
 887                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 888         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 889 }
 890
 891 #undef _ctf_integer_ext
 892 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 893         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 894
 895 #undef _ctf_array_encoded
 896 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _byte_order, _base, _user, _nowrite) \
 897         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 898         if (_user) {                                                    \
 899                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 900         } else {                                                        \
 901                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 902         }
 903
 904 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 905 #undef _ctf_array_bitfield
 906 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 907         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 908         if (_user) {                                                    \
 909                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 910         } else {                                                        \
 911                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 912         }
 913 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 914 /*
 915  * For big endian, we need to byteswap into little endian.
 916  */
 917 #undef _ctf_array_bitfield
 918 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 919         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 920         {                                                               \
 921                 size_t _i;                                              \
 922                                                                         \
 923                 for (_i = 0; _i < (_length); _i++) {                    \
 924                         _type _tmp;                                     \
 925                                                                         \
 926                         if (_user) {                                    \
 927                                 if (get_user(_tmp, (_type *) _src + _i)) \
 928                                         _tmp = 0;                       \
 929                         } else {                                        \
 930                                 _tmp = ((_type *) _src)[_i];            \
 931                         }                                               \
 932                         switch (sizeof(_type)) {                        \
 933                         case 1:                                         \
 934                                 break;                                  \
 935                         case 2:                                         \
 936                                 _tmp = cpu_to_le16(_tmp);               \
 937                                 break;                                  \
 938                         case 4:                                         \
 939                                 _tmp = cpu_to_le32(_tmp);               \
 940                                 break;                                  \
 941                         case 8:                                         \
 942                                 _tmp = cpu_to_le64(_tmp);               \
 943                                 break;                                  \
 944                         default:                                        \
 945                                 BUG_ON(1);                              \
 946                         }                                               \
 947                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
 948                 }                                                       \
 949         }
 950 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 951
 952 #undef _ctf_sequence_encoded
 953 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,         \
 954                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 955         {                                                               \
 956                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx]; \
 957                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 958                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 959         }                                                               \
 960         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 961         if (_user) {                                                    \
 962                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 963                         sizeof(_type) * __get_dynamic_len(dest));       \
 964         } else {                                                        \
 965                 __chan->ops->event_write(&__ctx, _src,                  \
 966                         sizeof(_type) * __get_dynamic_len(dest));       \
 967         }
 968
 969 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 970 #undef _ctf_sequence_bitfield
 971 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 972                         _length_type, _src_length,              \
 973                         _user, _nowrite)                        \
 974         {                                                               \
 975                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 976                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 977                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 978         }                                                               \
 979         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 980         if (_user) {                                                    \
 981                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 982                         sizeof(_type) * __get_dynamic_len(dest));       \
 983         } else {                                                        \
 984                 __chan->ops->event_write(&__ctx, _src,                  \
 985                         sizeof(_type) * __get_dynamic_len(dest));       \
 986         }
 987 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 988 /*
 989  * For big endian, we need to byteswap into little endian.
 990  */
 991 #undef _ctf_sequence_bitfield
 992 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 993                         _length_type, _src_length,              \
 994                         _user, _nowrite)                        \
 995         {                                                       \
 996                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 997                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 998                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 999         }                                                               \
1000         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
1001         {                                                               \
1002                 size_t _i, _length;                                     \
1003                                                                         \
1004                 _length = __get_dynamic_len(dest);                      \
1005                 for (_i = 0; _i < _length; _i++) {                      \
1006                         _type _tmp;                                     \
1007                                                                         \
1008                         if (_user) {                                    \
1009                                 if (get_user(_tmp, (_type *) _src + _i)) \
1010                                         _tmp = 0;                       \
1011                         } else {                                        \
1012                                 _tmp = ((_type *) _src)[_i];            \
1013                         }                                               \
1014                         switch (sizeof(_type)) {                        \
1015                         case 1:                                         \
1016                                 break;                                  \
1017                         case 2:                                         \
1018                                 _tmp = cpu_to_le16(_tmp);               \
1019                                 break;                                  \
1020                         case 4:                                         \
1021                                 _tmp = cpu_to_le32(_tmp);               \
1022                                 break;                                  \
1023                         case 8:                                         \
1024                                 _tmp = cpu_to_le64(_tmp);               \
1025                                 break;                                  \
1026                         default:                                        \
1027                                 BUG_ON(1);                              \
1028                         }                                               \
1029                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
1030                 }                                                       \
1031         }
1032 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
1033
1034 #undef _ctf_string
1035 #define _ctf_string(_item, _src, _user, _nowrite)                       \
1036         if (_user) {                                                    \
1037                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(*(_src))); \
1038                 __chan->ops->event_strcpy_from_user(&__ctx, _src,       \
1039                         __get_dynamic_len(dest));                       \
1040         } else {                                                        \
1041                 const char *__ctf_tmp_string =                          \
1042                         ((_src) ? (_src) : __LTTNG_NULL_STRING);        \
1043                 lib_ring_buffer_align_ctx(&__ctx,                       \
1044                         lttng_alignof(*__ctf_tmp_string));              \
1045                 __chan->ops->event_strcpy(&__ctx, __ctf_tmp_string,     \
1046                         __get_dynamic_len(dest));                       \
1047         }
1048
1049 #undef _ctf_enum
1050 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)           \
1051         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
1052
1053 #undef ctf_align
1054 #define ctf_align(_type)                                                \
1055         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));
1056
1057 #undef ctf_custom_field
1058 #define ctf_custom_field(_type, _item, _code)           _code
1059
1060 #undef ctf_custom_code
1061 #define ctf_custom_code(...)                                            \
1062         {                                                               \
1063                 __VA_ARGS__                                             \
1064         }
1065
1066 /* Beware: this get len actually consumes the len value */
1067 #undef __get_dynamic_len
1068 #define __get_dynamic_len(field)        this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx++]
1069
1070 #undef TP_PROTO
1071 #define TP_PROTO(...)   __VA_ARGS__
1072
1073 #undef TP_ARGS
1074 #define TP_ARGS(...)    __VA_ARGS__
1075
1076 #undef TP_FIELDS
1077 #define TP_FIELDS(...)  __VA_ARGS__
1078
1079 #undef TP_locvar
1080 #define TP_locvar(...)  __VA_ARGS__
1081
1082 #undef TP_code_pre
1083 #define TP_code_pre(...)        __VA_ARGS__
1084
1085 #undef TP_code_post
1086 #define TP_code_post(...)       __VA_ARGS__
1087
1088 /*
1089  * For state dump, check that "session" argument (mandatory) matches the
1090  * session this event belongs to. Ensures that we write state dump data only
1091  * into the started session, not into all sessions.
1092  */
1093 #ifdef TP_SESSION_CHECK
1094 #define _TP_SESSION_CHECK(session, csession)    (session == csession)
1095 #else /* TP_SESSION_CHECK */
1096 #define _TP_SESSION_CHECK(session, csession)    1
1097 #endif /* TP_SESSION_CHECK */
1098
1099 /*
1100  * Using twice size for filter stack data to hold size and pointer for
1101  * each field (worse case). For integers, max size required is 64-bit.
1102  * Same for double-precision floats. Those fit within
1103  * 2*sizeof(unsigned long) for all supported architectures.
1104  * Perform UNION (||) of filter runtime list.
1105  */
1106 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
1107 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
1108 static void __event_probe__##_name(void *__data, _proto)                      \
1109 {                                                                             \
1110         struct probe_local_vars { _locvar };                                  \
1111         struct lttng_event *__event = __data;                                 \
1112         struct lttng_probe_ctx __lttng_probe_ctx = {                                  \
1113                 .event = __event,                                             \
1114                 .interruptible = !irqs_disabled(),                            \
1115         };                                                                    \
1116         struct lttng_channel *__chan = __event->chan;                         \
1117         struct lttng_session *__session = __chan->session;                    \
1118         struct lib_ring_buffer_ctx __ctx;                                     \
1119         ssize_t __event_len;                                                  \
1120         size_t __event_align;                                                 \
1121         size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \
1122         union {                                                               \
1123                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
1124                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1125         } __stackvar;                                                         \
1126         int __ret;                                                            \
1127         struct probe_local_vars __tp_locvar;                                  \
1128         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
1129                         &__tp_locvar;                                         \
1130         struct lttng_pid_tracker *__lpf;                                      \
1131                                                                               \
1132         if (!_TP_SESSION_CHECK(session, __session))                           \
1133                 return;                                                       \
1134         if (unlikely(!READ_ONCE(__session->active)))                          \
1135                 return;                                                       \
1136         if (unlikely(!READ_ONCE(__chan->enabled)))                            \
1137                 return;                                                       \
1138         if (unlikely(!READ_ONCE(__event->enabled)))                           \
1139                 return;                                                       \
1140         __lpf = lttng_rcu_dereference(__session->pid_tracker);                \
1141         if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->tgid))) \
1142                 return;                                                       \
1143         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
1144         __dynamic_len_idx = __orig_dynamic_len_offset;                        \
1145         _code_pre                                                             \
1146         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
1147                 struct lttng_bytecode_runtime *bc_runtime;                    \
1148                 int __filter_record = __event->has_enablers_without_bytecode; \
1149                                                                               \
1150                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
1151                                 tp_locvar, _args);                                    \
1152                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
1153                         if (unlikely(bc_runtime->filter(bc_runtime, &__lttng_probe_ctx,       \
1154                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) \
1155                                 __filter_record = 1;                          \
1156                 }                                                             \
1157                 if (likely(!__filter_record))                                 \
1158                         goto __post;                                          \
1159         }                                                                     \
1160         __event_len = __event_get_size__##_name(tp_locvar, _args);            \
1161         if (unlikely(__event_len < 0)) {                                      \
1162                 lib_ring_buffer_lost_event_too_big(__chan->chan);             \
1163                 goto __post;                                                  \
1164         }                                                                     \
1165         __event_align = __event_get_align__##_name(tp_locvar, _args);         \
1166         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
1167                                  __event_align, -1);                          \
1168         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1169         if (__ret < 0)                                                        \
1170                 goto __post;                                                  \
1171         _fields                                                               \
1172         __chan->ops->event_commit(&__ctx);                                    \
1173 __post:                                                                       \
1174         _code_post                                                            \
1175         barrier();      /* use before un-reserve. */                          \
1176         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1177         return;                                                               \
1178 }
1179
1180 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
1181 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
1182 static void __event_probe__##_name(void *__data)                              \
1183 {                                                                             \
1184         struct probe_local_vars { _locvar };                                  \
1185         struct lttng_event *__event = __data;                                 \
1186         struct lttng_probe_ctx __lttng_probe_ctx = {                                  \
1187                 .event = __event,                                             \
1188                 .interruptible = !irqs_disabled(),                            \
1189         };                                                                    \
1190         struct lttng_channel *__chan = __event->chan;                         \
1191         struct lttng_session *__session = __chan->session;                    \
1192         struct lib_ring_buffer_ctx __ctx;                                     \
1193         ssize_t __event_len;                                                  \
1194         size_t __event_align;                                                 \
1195         size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \
1196         union {                                                               \
1197                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
1198                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1199         } __stackvar;                                                         \
1200         int __ret;                                                            \
1201         struct probe_local_vars __tp_locvar;                                  \
1202         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
1203                         &__tp_locvar;                                         \
1204         struct lttng_pid_tracker *__lpf;                                      \
1205                                                                               \
1206         if (!_TP_SESSION_CHECK(session, __session))                           \
1207                 return;                                                       \
1208         if (unlikely(!READ_ONCE(__session->active)))                          \
1209                 return;                                                       \
1210         if (unlikely(!READ_ONCE(__chan->enabled)))                            \
1211                 return;                                                       \
1212         if (unlikely(!READ_ONCE(__event->enabled)))                           \
1213                 return;                                                       \
1214         __lpf = lttng_rcu_dereference(__session->pid_tracker);                \
1215         if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->tgid)))  \
1216                 return;                                                       \
1217         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
1218         __dynamic_len_idx = __orig_dynamic_len_offset;                        \
1219         _code_pre                                                             \
1220         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
1221                 struct lttng_bytecode_runtime *bc_runtime;                    \
1222                 int __filter_record = __event->has_enablers_without_bytecode; \
1223                                                                               \
1224                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
1225                                 tp_locvar);                                   \
1226                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
1227                         if (unlikely(bc_runtime->filter(bc_runtime, &__lttng_probe_ctx, \
1228                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) \
1229                                 __filter_record = 1;                          \
1230                 }                                                             \
1231                 if (likely(!__filter_record))                                 \
1232                         goto __post;                                          \
1233         }                                                                     \
1234         __event_len = __event_get_size__##_name(tp_locvar);                   \
1235         if (unlikely(__event_len < 0)) {                                      \
1236                 lib_ring_buffer_lost_event_too_big(__chan->chan);             \
1237                 goto __post;                                                  \
1238         }                                                                     \
1239         __event_align = __event_get_align__##_name(tp_locvar);                \
1240         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
1241                                  __event_align, -1);                          \
1242         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1243         if (__ret < 0)                                                        \
1244                 goto __post;                                                  \
1245         _fields                                                               \
1246         __chan->ops->event_commit(&__ctx);                                    \
1247 __post:                                                                       \
1248         _code_post                                                            \
1249         barrier();      /* use before un-reserve. */                          \
1250         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1251         return;                                                               \
1252 }
1253
1254 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1255
1256 #undef __get_dynamic_len
1257
1258 /*
1259  * Stage 7 of the trace events.
1260  *
1261  * Create event descriptions.
1262  */
1263
1264 /* Named field types must be defined in lttng-types.h */
1265
1266 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1267
1268 #ifndef TP_PROBE_CB
1269 #define TP_PROBE_CB(_template)  &__event_probe__##_template
1270 #endif
1271
1272 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1273 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)      \
1274 static const struct lttng_event_desc __event_desc___##_map = {          \
1275         .fields = __event_fields___##_template,                         \
1276         .name = #_map,                                                  \
1277         .kname = #_name,                                                \
1278         .probe_callback = (void *) TP_PROBE_CB(_template),              \
1279         .nr_fields = ARRAY_SIZE(__event_fields___##_template),          \
1280         .owner = THIS_MODULE,                                           \
1281 };
1282
1283 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1284 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1285         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1286
1287 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1288
1289 /*
1290  * Stage 8 of the trace events.
1291  *
1292  * Create an array of event description pointers.
1293  */
1294
1295 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1296
1297 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1298 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
1299                 &__event_desc___##_map,
1300
1301 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1302 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1303         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1304
1305 #define TP_ID1(_token, _system) _token##_system
1306 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1307
1308 static const struct lttng_event_desc *TP_ID(__event_desc___, TRACE_SYSTEM)[] = {
1309 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1310 };
1311
1312 #undef TP_ID1
1313 #undef TP_ID
1314
1315 /*
1316  * Stage 9 of the trace events.
1317  *
1318  * Create a toplevel descriptor for the whole probe.
1319  */
1320
1321 #define TP_ID1(_token, _system) _token##_system
1322 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1323
1324 /* non-const because list head will be modified when registered. */
1325 static __used struct lttng_probe_desc TP_ID(__probe_desc___, TRACE_SYSTEM) = {
1326         .provider = __stringify(TRACE_SYSTEM),
1327         .event_desc = TP_ID(__event_desc___, TRACE_SYSTEM),
1328         .nr_events = ARRAY_SIZE(TP_ID(__event_desc___, TRACE_SYSTEM)),
1329         .head = { NULL, NULL },
1330         .lazy_init_head = { NULL, NULL },
1331         .lazy = 0,
1332 };
1333
1334 #undef TP_ID1
1335 #undef TP_ID
1336
1337 /*
1338  * Stage 10 of the trace events.
1339  *
1340  * Register/unregister probes at module load/unload.
1341  */
1342
1343 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1344
1345 #define TP_ID1(_token, _system) _token##_system
1346 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1347 #define module_init_eval1(_token, _system)      module_init(_token##_system)
1348 #define module_init_eval(_token, _system)       module_init_eval1(_token, _system)
1349 #define module_exit_eval1(_token, _system)      module_exit(_token##_system)
1350 #define module_exit_eval(_token, _system)       module_exit_eval1(_token, _system)
1351
1352 #ifndef TP_MODULE_NOINIT
1353 static int TP_ID(__lttng_events_init__, TRACE_SYSTEM)(void)
1354 {
1355         wrapper_vmalloc_sync_all();
1356         return lttng_probe_register(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1357 }
1358
1359 static void TP_ID(__lttng_events_exit__, TRACE_SYSTEM)(void)
1360 {
1361         lttng_probe_unregister(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1362 }
1363
1364 #ifndef TP_MODULE_NOAUTOLOAD
1365 module_init_eval(__lttng_events_init__, TRACE_SYSTEM);
1366 module_exit_eval(__lttng_events_exit__, TRACE_SYSTEM);
1367 #endif
1368
1369 #endif
1370
1371 #undef module_init_eval
1372 #undef module_exit_eval
1373 #undef TP_ID1
1374 #undef TP_ID
1375
1376 #undef TP_PROTO
1377 #undef TP_ARGS