probes/lttng-tracepoint-event-impl.h

   1 /*
   2  * lttng-tracepoint-event-impl.h
   3  *
   4  * Copyright (C) 2009 Steven Rostedt <rostedt@goodmis.org>
   5  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; only
  10  * version 2.1 of the License.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20  */
  21
  22 #include <linux/uaccess.h>
  23 #include <linux/debugfs.h>
  24 #include <linux/rculist.h>
  25 #include <asm/byteorder.h>
  26 #include <linux/swab.h>
  27
  28 #include <probes/lttng.h>
  29 #include <probes/lttng-types.h>
  30 #include <probes/lttng-probe-user.h>
  31 #include <wrapper/vmalloc.h>    /* for wrapper_vmalloc_sync_all() */
  32 #include <wrapper/ringbuffer/frontend_types.h>
  33 #include <wrapper/ringbuffer/backend.h>
  34 #include <wrapper/rcu.h>
  35 #include <lttng-events.h>
  36 #include <lttng-tracer-core.h>
  37
  38 #define __LTTNG_NULL_STRING     "(null)"
  39
  40 /*
  41  * Macro declarations used for all stages.
  42  */
  43
  44 /*
  45  * LTTng name mapping macros. LTTng remaps some of the kernel events to
  46  * enforce name-spacing.
  47  */
  48 #undef LTTNG_TRACEPOINT_EVENT_MAP
  49 #define LTTNG_TRACEPOINT_EVENT_MAP(name, map, proto, args, fields) \
  50         LTTNG_TRACEPOINT_EVENT_CLASS(map,                               \
  51                              PARAMS(proto),                             \
  52                              PARAMS(args),                              \
  53                              PARAMS(fields))                            \
  54         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  55
  56 #undef LTTNG_TRACEPOINT_EVENT_MAP_NOARGS
  57 #define LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, map, fields)            \
  58         LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(map,                        \
  59                              PARAMS(fields))                            \
  60         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(map, name, map)
  61
  62 #undef LTTNG_TRACEPOINT_EVENT_CODE_MAP
  63 #define LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, map, proto, args, _locvar, _code_pre, fields, _code_post) \
  64         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(map,                          \
  65                              PARAMS(proto),                             \
  66                              PARAMS(args),                              \
  67                              PARAMS(_locvar),                           \
  68                              PARAMS(_code_pre),                         \
  69                              PARAMS(fields),                            \
  70                              PARAMS(_code_post))                        \
  71         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(map, name, map, PARAMS(proto), PARAMS(args))
  72
  73 #undef LTTNG_TRACEPOINT_EVENT_CODE
  74 #define LTTNG_TRACEPOINT_EVENT_CODE(name, proto, args, _locvar, _code_pre, fields, _code_post) \
  75         LTTNG_TRACEPOINT_EVENT_CODE_MAP(name, name,                     \
  76                              PARAMS(proto),                             \
  77                              PARAMS(args),                              \
  78                              PARAMS(_locvar),                           \
  79                              PARAMS(_code_pre),                         \
  80                              PARAMS(fields),                            \
  81                              PARAMS(_code_post))
  82
  83 /*
  84  * LTTNG_TRACEPOINT_EVENT_CLASS can be used to add a generic function
  85  * handlers for events. That is, if all events have the same parameters
  86  * and just have distinct trace points.  Each tracepoint can be defined
  87  * with LTTNG_TRACEPOINT_EVENT_INSTANCE and that will map the
  88  * LTTNG_TRACEPOINT_EVENT_CLASS to the tracepoint.
  89  *
  90  * LTTNG_TRACEPOINT_EVENT is a one to one mapping between tracepoint and
  91  * template.
  92  */
  93
  94 #undef LTTNG_TRACEPOINT_EVENT
  95 #define LTTNG_TRACEPOINT_EVENT(name, proto, args, fields)               \
  96         LTTNG_TRACEPOINT_EVENT_MAP(name, name,                          \
  97                         PARAMS(proto),                                  \
  98                         PARAMS(args),                                   \
  99                         PARAMS(fields))
 100
 101 #undef LTTNG_TRACEPOINT_EVENT_NOARGS
 102 #define LTTNG_TRACEPOINT_EVENT_NOARGS(name, fields)                     \
 103         LTTNG_TRACEPOINT_EVENT_MAP_NOARGS(name, name, PARAMS(fields))
 104
 105 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE
 106 #define LTTNG_TRACEPOINT_EVENT_INSTANCE(template, name, proto, args)    \
 107         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(template, name, name, PARAMS(proto), PARAMS(args))
 108
 109 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS
 110 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_NOARGS(template, name)  \
 111         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(template, name, name)
 112
 113 #undef LTTNG_TRACEPOINT_EVENT_CLASS
 114 #define LTTNG_TRACEPOINT_EVENT_CLASS(_name, _proto, _args, _fields) \
 115         LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, PARAMS(_proto), PARAMS(_args), , , \
 116                 PARAMS(_fields), )
 117
 118 #undef LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS
 119 #define LTTNG_TRACEPOINT_EVENT_CLASS_NOARGS(_name, _fields) \
 120         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, , , PARAMS(_fields), )
 121
 122
 123 /*
 124  * Stage 1 of the trace events.
 125  *
 126  * Create dummy trace calls for each events, verifying that the LTTng module
 127  * instrumentation headers match the kernel arguments. Will be optimized
 128  * out by the compiler.
 129  */
 130
 131 /* Reset all macros within TRACEPOINT_EVENT */
 132 #include <probes/lttng-events-reset.h>
 133
 134 #undef TP_PROTO
 135 #define TP_PROTO(...)   __VA_ARGS__
 136
 137 #undef TP_ARGS
 138 #define TP_ARGS(...)    __VA_ARGS__
 139
 140 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 141 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 142 void trace_##_name(_proto);
 143
 144 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 145 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 146 void trace_##_name(void);
 147
 148 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 149
 150 /*
 151  * Stage 1.1 of the trace events.
 152  *
 153  * Create dummy trace prototypes for each event class, and for each used
 154  * template. This will allow checking whether the prototypes from the
 155  * class and the instance using the class actually match.
 156  */
 157
 158 #include <probes/lttng-events-reset.h>  /* Reset all macros within TRACE_EVENT */
 159
 160 #undef TP_PROTO
 161 #define TP_PROTO(...)   __VA_ARGS__
 162
 163 #undef TP_ARGS
 164 #define TP_ARGS(...)    __VA_ARGS__
 165
 166 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
 167 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
 168 void __event_template_proto___##_template(_proto);
 169
 170 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
 171 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
 172 void __event_template_proto___##_template(void);
 173
 174 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 175 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 176 void __event_template_proto___##_name(_proto);
 177
 178 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 179 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 180 void __event_template_proto___##_name(void);
 181
 182 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 183
 184 /*
 185  * Stage 1.2 of tracepoint event generation
 186  *
 187  * Unfolding the enums
 188  */
 189 #include <probes/lttng-events-reset.h>  /* Reset all macros within TRACE_EVENT */
 190
 191 /* Enumeration entry (single value) */
 192 #undef ctf_enum_value
 193 #define ctf_enum_value(_string, _value)                                 \
 194         {                                                               \
 195                 .start = {                                              \
 196                         .signedness = lttng_is_signed_type(__typeof__(_value)), \
 197                         .value = lttng_is_signed_type(__typeof__(_value)) ? \
 198                                 (long long) (_value) : (_value),        \
 199                 },                                                      \
 200                 .end = {                                                \
 201                         .signedness = lttng_is_signed_type(__typeof__(_value)), \
 202                         .value = lttng_is_signed_type(__typeof__(_value)) ? \
 203                                 (long long) (_value) : (_value),        \
 204                 },                                                      \
 205                 .string = (_string),                                    \
 206         },
 207
 208 /* Enumeration entry (range) */
 209 #undef ctf_enum_range
 210 #define ctf_enum_range(_string, _range_start, _range_end)               \
 211         {                                                               \
 212                 .start = {                                              \
 213                         .signedness = lttng_is_signed_type(__typeof__(_range_start)), \
 214                         .value = lttng_is_signed_type(__typeof__(_range_start)) ? \
 215                                 (long long) (_range_start) : (_range_start), \
 216                 },                                                      \
 217                 .end = {                                                \
 218                         .signedness = lttng_is_signed_type(__typeof__(_range_end)), \
 219                         .value = lttng_is_signed_type(__typeof__(_range_end)) ? \
 220                                 (long long) (_range_end) : (_range_end), \
 221                 },                                                      \
 222                 .string = (_string),                                    \
 223         },
 224
 225 #undef TP_ENUM_VALUES
 226 #define TP_ENUM_VALUES(...)                                             \
 227         __VA_ARGS__
 228
 229 #undef LTTNG_TRACEPOINT_ENUM
 230 #define LTTNG_TRACEPOINT_ENUM(_name, _values)                           \
 231         const struct lttng_enum_entry __enum_values__##_name[] = { \
 232                 _values                                                 \
 233         };
 234
 235 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 236
 237 /*
 238  * Stage 2 of the trace events.
 239  *
 240  * Create event field type metadata section.
 241  * Each event produce an array of fields.
 242  */
 243
 244 /* Reset all macros within TRACEPOINT_EVENT */
 245 #include <probes/lttng-events-reset.h>
 246 #include <probes/lttng-events-write.h>
 247 #include <probes/lttng-events-nowrite.h>
 248
 249 #undef _ctf_integer_ext
 250 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 251         {                                                       \
 252           .name = #_item,                                       \
 253           .type = __type_integer(_type, 0, 0, -1, _byte_order, _base, none),\
 254           .nowrite = _nowrite,                                  \
 255           .user = _user,                                        \
 256         },
 257
 258 #undef _ctf_array_encoded
 259 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 260         {                                                       \
 261           .name = #_item,                                       \
 262           .type =                                               \
 263                 {                                               \
 264                   .atype = atype_array,                         \
 265                   .u =                                          \
 266                         {                                       \
 267                           .array =                              \
 268                                 {                               \
 269                                   .elem_type = __type_integer(_type, 0, 0, 0, __BYTE_ORDER, 10, _encoding), \
 270                                   .length = _length,            \
 271                                 }                               \
 272                         }                                       \
 273                 },                                              \
 274           .nowrite = _nowrite,                                  \
 275           .user = _user,                                        \
 276         },
 277
 278 #undef _ctf_array_bitfield
 279 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 280         {                                                       \
 281           .name = #_item,                                       \
 282           .type =                                               \
 283                 {                                               \
 284                   .atype = atype_array,                         \
 285                   .u =                                          \
 286                         {                                       \
 287                           .array =                              \
 288                                 {                               \
 289                                   .elem_type = __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none), \
 290                                   .length = (_length) * sizeof(_type) * CHAR_BIT, \
 291                                   .elem_alignment = lttng_alignof(_type), \
 292                                 }                               \
 293                         }                                       \
 294                 },                                              \
 295           .nowrite = _nowrite,                                  \
 296           .user = _user,                                        \
 297         },
 298
 299
 300 #undef _ctf_sequence_encoded
 301 #define _ctf_sequence_encoded(_type, _item, _src,               \
 302                         _length_type, _src_length, _encoding,   \
 303                         _byte_order, _base, _user, _nowrite)    \
 304         {                                                       \
 305           .name = #_item,                                       \
 306           .type =                                               \
 307                 {                                               \
 308                   .atype = atype_sequence,                      \
 309                   .u =                                          \
 310                         {                                       \
 311                           .sequence =                           \
 312                                 {                               \
 313                                   .length_type = __type_integer(_length_type, 0, 0, 0, __BYTE_ORDER, 10, none), \
 314                                   .elem_type = __type_integer(_type, 0, 0, -1, _byte_order, _base, _encoding), \
 315                                 },                              \
 316                         },                                      \
 317                 },                                              \
 318           .nowrite = _nowrite,                                  \
 319           .user = _user,                                        \
 320         },
 321
 322 #undef _ctf_sequence_bitfield
 323 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 324                         _length_type, _src_length,              \
 325                         _user, _nowrite)                        \
 326         {                                                       \
 327           .name = #_item,                                       \
 328           .type =                                               \
 329                 {                                               \
 330                   .atype = atype_sequence,                      \
 331                   .u =                                          \
 332                         {                                       \
 333                           .sequence =                           \
 334                                 {                               \
 335                                   .length_type = __type_integer(_length_type, 0, 0, 0, __BYTE_ORDER, 10, none), \
 336                                   .elem_type = __type_integer(_type, 1, 1, 0, __LITTLE_ENDIAN, 10, none), \
 337                                   .elem_alignment = lttng_alignof(_type), \
 338                                 },                              \
 339                         },                                      \
 340                 },                                              \
 341           .nowrite = _nowrite,                                  \
 342           .user = _user,                                        \
 343         },
 344
 345 #undef _ctf_string
 346 #define _ctf_string(_item, _src, _user, _nowrite)               \
 347         {                                                       \
 348           .name = #_item,                                       \
 349           .type =                                               \
 350                 {                                               \
 351                   .atype = atype_string,                        \
 352                   .u =                                          \
 353                         {                                       \
 354                           .basic = { .string = { .encoding = lttng_encode_UTF8 } } \
 355                         },                                      \
 356                 },                                              \
 357           .nowrite = _nowrite,                                  \
 358           .user = _user,                                        \
 359         },
 360
 361 #undef _ctf_enum
 362 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)   \
 363         {                                                       \
 364                 .name = #_item,                                 \
 365                 .type = {                                       \
 366                         .atype = atype_enum,                    \
 367                         .u = {                                  \
 368                                 .basic = {                      \
 369                                         .enumeration = {        \
 370                                                 .desc = &__enum_##_name, \
 371                                                 .container_type = { \
 372                                                         .size = sizeof(_type) * CHAR_BIT, \
 373                                                         .alignment = lttng_alignof(_type) * CHAR_BIT, \
 374                                                         .signedness = lttng_is_signed_type(_type), \
 375                                                         .reverse_byte_order = 0, \
 376                                                         .base = 10, \
 377                                                         .encoding = lttng_encode_none, \
 378                                                 },              \
 379                                         },                      \
 380                                  },                             \
 381                         },                                      \
 382                 },                                              \
 383                 .nowrite = _nowrite,                            \
 384                 .user = _user,                                  \
 385         },
 386
 387 #undef ctf_custom_field
 388 #define ctf_custom_field(_type, _item, _code)                   \
 389         {                                                       \
 390                 .name = #_item,                                 \
 391                 .type = { _type },                              \
 392                 .nowrite = 0,                                   \
 393                 .user = 0,                                      \
 394         },
 395
 396 #undef ctf_custom_type
 397 #define ctf_custom_type(...)    __VA_ARGS__
 398
 399 #undef TP_FIELDS
 400 #define TP_FIELDS(...)  __VA_ARGS__     /* Only one used in this phase */
 401
 402 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 403 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 404         static const struct lttng_event_field __event_fields___##_name[] = { \
 405                 _fields                                                      \
 406         };
 407
 408 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 409 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 410         LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, PARAMS(_fields), _code_post)
 411
 412 #undef LTTNG_TRACEPOINT_ENUM
 413 #define LTTNG_TRACEPOINT_ENUM(_name, _values)                                           \
 414         static const struct lttng_enum_desc __enum_##_name = {                          \
 415                 .name = #_name,                                                         \
 416                 .entries = __enum_values__##_name,                                      \
 417                 .nr_entries = ARRAY_SIZE(__enum_values__##_name),                       \
 418         };
 419
 420 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 421
 422 /*
 423  * Stage 3 of the trace events.
 424  *
 425  * Create probe callback prototypes.
 426  */
 427
 428 /* Reset all macros within TRACEPOINT_EVENT */
 429 #include <probes/lttng-events-reset.h>
 430
 431 #undef TP_PROTO
 432 #define TP_PROTO(...)   __VA_ARGS__
 433
 434 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 435 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 436 static void __event_probe__##_name(void *__data, _proto);
 437
 438 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 439 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 440 static void __event_probe__##_name(void *__data);
 441
 442 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 443
 444 /*
 445  * Stage 4 of the trace events.
 446  *
 447  * Create static inline function that calculates event size.
 448  */
 449
 450 /* Reset all macros within TRACEPOINT_EVENT */
 451 #include <probes/lttng-events-reset.h>
 452 #include <probes/lttng-events-write.h>
 453
 454 #undef _ctf_integer_ext
 455 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 456         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 457         __event_len += sizeof(_type);
 458
 459 #undef _ctf_array_encoded
 460 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 461         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 462         __event_len += sizeof(_type) * (_length);
 463
 464 #undef _ctf_array_bitfield
 465 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 466         _ctf_array_encoded(_type, _item, _src, _length, none, _user, _nowrite)
 467
 468 #undef _ctf_sequence_encoded
 469 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 470                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 471         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_length_type)); \
 472         __event_len += sizeof(_length_type);                                   \
 473         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \
 474         {                                                                               \
 475                 size_t __seqlen = (_src_length);                                        \
 476                                                                                         \
 477                 if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 478                         goto error;                                                     \
 479                 barrier();      /* reserve before use. */                               \
 480                 this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = __seqlen; \
 481                 __event_len += sizeof(_type) * __seqlen;                                \
 482         }
 483
 484 #undef _ctf_sequence_bitfield
 485 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 486                         _length_type, _src_length,              \
 487                         _user, _nowrite)                        \
 488         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 489                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 490
 491 /*
 492  * ctf_user_string includes \0. If returns 0, it faulted, so we set size to
 493  * 1 (\0 only).
 494  */
 495 #undef _ctf_string
 496 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 497         if (unlikely(++this_cpu_ptr(&lttng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \
 498                 goto error;                                                    \
 499         barrier();      /* reserve before use. */                              \
 500         if (_user) {                                                           \
 501                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 502                         max_t(size_t, lttng_strlen_user_inatomic(_src), 1);    \
 503         } else {                                                               \
 504                 __event_len += this_cpu_ptr(&lttng_dynamic_len_stack)->stack[this_cpu_ptr(&lttng_dynamic_len_stack)->offset - 1] = \
 505                         strlen((_src) ? (_src) : __LTTNG_NULL_STRING) + 1; \
 506         }
 507
 508 #undef _ctf_enum
 509 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)                  \
 510         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 511
 512 #undef ctf_align
 513 #define ctf_align(_type)                                                \
 514         __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type));
 515
 516 #undef ctf_custom_field
 517 #define ctf_custom_field(_type, _item, _code)                           \
 518         {                                                               \
 519                 _code                                                   \
 520         }
 521
 522 #undef ctf_custom_code
 523 #define ctf_custom_code(...)            __VA_ARGS__
 524
 525 #undef TP_PROTO
 526 #define TP_PROTO(...)   __VA_ARGS__
 527
 528 #undef TP_FIELDS
 529 #define TP_FIELDS(...)  __VA_ARGS__
 530
 531 #undef TP_locvar
 532 #define TP_locvar(...)  __VA_ARGS__
 533
 534 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 535 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 536 static inline ssize_t __event_get_size__##_name(void *__tp_locvar, _proto)     \
 537 {                                                                             \
 538         size_t __event_len = 0;                                               \
 539         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 540         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 541                                                                               \
 542         _fields                                                               \
 543         return __event_len;                                                   \
 544                                                                               \
 545 error:                                                                        \
 546         __attribute__((unused));                                              \
 547         return -1;                                                            \
 548 }
 549
 550 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 551 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 552 static inline ssize_t __event_get_size__##_name(void *__tp_locvar)            \
 553 {                                                                             \
 554         size_t __event_len = 0;                                               \
 555         unsigned int __dynamic_len_idx __attribute__((unused)) = 0;           \
 556         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 557                                                                               \
 558         _fields                                                               \
 559         return __event_len;                                                   \
 560                                                                               \
 561 error:                                                                        \
 562         __attribute__((unused));                                              \
 563         return -1;                                                            \
 564 }
 565
 566 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 567
 568
 569 /*
 570  * Stage 4.1 of tracepoint event generation.
 571  *
 572  * Create static inline function that layout the filter stack data.
 573  * We make both write and nowrite data available to the filter.
 574  */
 575
 576 /* Reset all macros within TRACEPOINT_EVENT */
 577 #include <probes/lttng-events-reset.h>
 578 #include <probes/lttng-events-write.h>
 579 #include <probes/lttng-events-nowrite.h>
 580
 581 #undef _ctf_integer_ext_fetched
 582 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 583         if (lttng_is_signed_type(_type)) {                                     \
 584                 int64_t __ctf_tmp_int64;                                       \
 585                 switch (sizeof(_type)) {                                       \
 586                 case 1:                                                        \
 587                 {                                                              \
 588                         union { _type t; int8_t v; } __tmp = { (_type) (_src) }; \
 589                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 590                         break;                                                 \
 591                 }                                                              \
 592                 case 2:                                                        \
 593                 {                                                              \
 594                         union { _type t; int16_t v; } __tmp = { (_type) (_src) }; \
 595                         if (_byte_order != __BYTE_ORDER)                       \
 596                                 __swab16s(&__tmp.v);                           \
 597                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 598                         break;                                                 \
 599                 }                                                              \
 600                 case 4:                                                        \
 601                 {                                                              \
 602                         union { _type t; int32_t v; } __tmp = { (_type) (_src) }; \
 603                         if (_byte_order != __BYTE_ORDER)                       \
 604                                 __swab32s(&__tmp.v);                           \
 605                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 606                         break;                                                 \
 607                 }                                                              \
 608                 case 8:                                                        \
 609                 {                                                              \
 610                         union { _type t; int64_t v; } __tmp = { (_type) (_src) }; \
 611                         if (_byte_order != __BYTE_ORDER)                       \
 612                                 __swab64s(&__tmp.v);                           \
 613                         __ctf_tmp_int64 = (int64_t) __tmp.v;                   \
 614                         break;                                                 \
 615                 }                                                              \
 616                 default:                                                       \
 617                         BUG_ON(1);                                             \
 618                 };                                                             \
 619                 memcpy(__stack_data, &__ctf_tmp_int64, sizeof(int64_t));       \
 620         } else {                                                               \
 621                 uint64_t __ctf_tmp_uint64;                                     \
 622                 switch (sizeof(_type)) {                                       \
 623                 case 1:                                                        \
 624                 {                                                              \
 625                         union { _type t; uint8_t v; } __tmp = { (_type) (_src) }; \
 626                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 627                         break;                                                 \
 628                 }                                                              \
 629                 case 2:                                                        \
 630                 {                                                              \
 631                         union { _type t; uint16_t v; } __tmp = { (_type) (_src) }; \
 632                         if (_byte_order != __BYTE_ORDER)                       \
 633                                 __swab16s(&__tmp.v);                           \
 634                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 635                         break;                                                 \
 636                 }                                                              \
 637                 case 4:                                                        \
 638                 {                                                              \
 639                         union { _type t; uint32_t v; } __tmp = { (_type) (_src) }; \
 640                         if (_byte_order != __BYTE_ORDER)                       \
 641                                 __swab32s(&__tmp.v);                           \
 642                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 643                         break;                                                 \
 644                 }                                                              \
 645                 case 8:                                                        \
 646                 {                                                              \
 647                         union { _type t; uint64_t v; } __tmp = { (_type) (_src) }; \
 648                         if (_byte_order != __BYTE_ORDER)                       \
 649                                 __swab64s(&__tmp.v);                           \
 650                         __ctf_tmp_uint64 = (uint64_t) __tmp.v;                 \
 651                         break;                                                 \
 652                 }                                                              \
 653                 default:                                                       \
 654                         BUG_ON(1);                                             \
 655                 };                                                             \
 656                 memcpy(__stack_data, &__ctf_tmp_uint64, sizeof(uint64_t));     \
 657         }                                                                      \
 658         __stack_data += sizeof(int64_t);
 659
 660 #undef _ctf_integer_ext_isuser0
 661 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 662         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 663
 664 #undef _ctf_integer_ext_isuser1
 665 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 666 {                                                                                       \
 667         union {                                                                         \
 668                 char __array[sizeof(_user_src)];                                        \
 669                 __typeof__(_user_src) __v;                                              \
 670         } __tmp_fetch;                                                                  \
 671         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 672                                 &(_user_src), sizeof(_user_src)))                       \
 673                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 674         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 675 }
 676
 677 #undef _ctf_integer_ext
 678 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 679         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 680
 681 #undef _ctf_array_encoded
 682 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 683         {                                                                      \
 684                 unsigned long __ctf_tmp_ulong = (unsigned long) (_length);     \
 685                 const void *__ctf_tmp_ptr = (_src);                            \
 686                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 687                 __stack_data += sizeof(unsigned long);                         \
 688                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 689                 __stack_data += sizeof(void *);                                \
 690         }
 691
 692 #undef _ctf_array_bitfield
 693 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 694         _ctf_array_encoded(_type, _item, _src, _length, none, _user, _nowrite)
 695
 696 #undef _ctf_sequence_encoded
 697 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                \
 698                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 699         {                                                                      \
 700                 unsigned long __ctf_tmp_ulong = (unsigned long) (_src_length); \
 701                 const void *__ctf_tmp_ptr = (_src);                            \
 702                 memcpy(__stack_data, &__ctf_tmp_ulong, sizeof(unsigned long)); \
 703                 __stack_data += sizeof(unsigned long);                         \
 704                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 705                 __stack_data += sizeof(void *);                                \
 706         }
 707
 708 #undef _ctf_sequence_bitfield
 709 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 710                         _length_type, _src_length,              \
 711                         _user, _nowrite)                        \
 712         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 713                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 714
 715 #undef _ctf_string
 716 #define _ctf_string(_item, _src, _user, _nowrite)                              \
 717         {                                                                      \
 718                 const void *__ctf_tmp_ptr =                                    \
 719                         ((_src) ? (_src) : __LTTNG_NULL_STRING);               \
 720                 memcpy(__stack_data, &__ctf_tmp_ptr, sizeof(void *));          \
 721                 __stack_data += sizeof(void *);                                \
 722         }
 723
 724 #undef _ctf_enum
 725 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)                  \
 726         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 727
 728 #undef TP_PROTO
 729 #define TP_PROTO(...) __VA_ARGS__
 730
 731 #undef TP_FIELDS
 732 #define TP_FIELDS(...) __VA_ARGS__
 733
 734 #undef TP_locvar
 735 #define TP_locvar(...)  __VA_ARGS__
 736
 737 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 738 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 739 static inline                                                                 \
 740 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 741                 void *__tp_locvar)                                            \
 742 {                                                                             \
 743         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 744                                                                               \
 745         _fields                                                               \
 746 }
 747
 748 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 749 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 750 static inline                                                                 \
 751 void __event_prepare_filter_stack__##_name(char *__stack_data,                \
 752                 void *__tp_locvar, _proto)                                    \
 753 {                                                                             \
 754         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 755                                                                               \
 756         _fields                                                               \
 757 }
 758
 759 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 760
 761 /*
 762  * Stage 5 of the trace events.
 763  *
 764  * Create static inline function that calculates event payload alignment.
 765  */
 766
 767 /* Reset all macros within TRACEPOINT_EVENT */
 768 #include <probes/lttng-events-reset.h>
 769 #include <probes/lttng-events-write.h>
 770
 771 #undef _ctf_integer_ext
 772 #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \
 773         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 774
 775 #undef _ctf_array_encoded
 776 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 777         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 778
 779 #undef _ctf_array_bitfield
 780 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 781         _ctf_array_encoded(_type, _item, _src, _length, none, _user, _nowrite)
 782
 783 #undef _ctf_sequence_encoded
 784 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,                 \
 785                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 786         __event_align = max_t(size_t, __event_align, lttng_alignof(_length_type)); \
 787         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 788
 789 #undef _ctf_sequence_bitfield
 790 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 791                         _length_type, _src_length,              \
 792                         _user, _nowrite)                        \
 793         _ctf_sequence_encoded(_type, _item, _src, _length_type, _src_length, \
 794                 none, __LITTLE_ENDIAN, 10, _user, _nowrite)
 795
 796 #undef _ctf_string
 797 #define _ctf_string(_item, _src, _user, _nowrite)
 798
 799 #undef _ctf_enum
 800 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)   \
 801         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
 802
 803 #undef ctf_align
 804 #define ctf_align(_type)                                                \
 805         __event_align = max_t(size_t, __event_align, lttng_alignof(_type));
 806
 807 #undef TP_PROTO
 808 #define TP_PROTO(...)   __VA_ARGS__
 809
 810 #undef TP_FIELDS
 811 #define TP_FIELDS(...)  __VA_ARGS__
 812
 813 #undef TP_locvar
 814 #define TP_locvar(...)  __VA_ARGS__
 815
 816 #undef ctf_custom_field
 817 #define ctf_custom_field(_type, _item, _code)           _code
 818
 819 #undef ctf_custom_code
 820 #define ctf_custom_code(...)                                            \
 821         {                                                               \
 822                 __VA_ARGS__                                             \
 823         }
 824
 825 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
 826 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
 827 static inline size_t __event_get_align__##_name(void *__tp_locvar, _proto)    \
 828 {                                                                             \
 829         size_t __event_align = 1;                                             \
 830         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 831                                                                               \
 832         _fields                                                               \
 833         return __event_align;                                                 \
 834 }
 835
 836 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
 837 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
 838 static inline size_t __event_get_align__##_name(void *__tp_locvar)            \
 839 {                                                                             \
 840         size_t __event_align = 1;                                             \
 841         struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar;  \
 842                                                                               \
 843         _fields                                                               \
 844         return __event_align;                                                 \
 845 }
 846
 847 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
 848
 849 /*
 850  * Stage 6 of tracepoint event generation.
 851  *
 852  * Create the probe function. This function calls event size calculation
 853  * and writes event data into the buffer.
 854  */
 855
 856 /* Reset all macros within TRACEPOINT_EVENT */
 857 #include <probes/lttng-events-reset.h>
 858 #include <probes/lttng-events-write.h>
 859
 860 #undef _ctf_integer_ext_fetched
 861 #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \
 862         {                                                               \
 863                 _type __tmp = _src;                                     \
 864                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(__tmp));\
 865                 __chan->ops->event_write(&__ctx, &__tmp, sizeof(__tmp));\
 866         }
 867
 868 #undef _ctf_integer_ext_isuser0
 869 #define _ctf_integer_ext_isuser0(_type, _item, _src, _byte_order, _base, _nowrite) \
 870         _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite)
 871
 872 #undef _ctf_integer_ext_isuser1
 873 #define _ctf_integer_ext_isuser1(_type, _item, _user_src, _byte_order, _base, _nowrite) \
 874 {                                                                              \
 875         union {                                                                         \
 876                 char __array[sizeof(_user_src)];                                        \
 877                 __typeof__(_user_src) __v;                                              \
 878         } __tmp_fetch;                                                                  \
 879         if (lib_ring_buffer_copy_from_user_check_nofault(__tmp_fetch.__array,           \
 880                                 &(_user_src), sizeof(_user_src)))                       \
 881                 memset(__tmp_fetch.__array, 0, sizeof(__tmp_fetch.__array));            \
 882         _ctf_integer_ext_fetched(_type, _item, __tmp_fetch.__v, _byte_order, _base, _nowrite) \
 883 }
 884
 885 #undef _ctf_integer_ext
 886 #define _ctf_integer_ext(_type, _item, _user_src, _byte_order, _base, _user, _nowrite) \
 887         _ctf_integer_ext_isuser##_user(_type, _item, _user_src, _byte_order, _base, _nowrite)
 888
 889 #undef _ctf_array_encoded
 890 #define _ctf_array_encoded(_type, _item, _src, _length, _encoding, _user, _nowrite) \
 891         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 892         if (_user) {                                                    \
 893                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 894         } else {                                                        \
 895                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 896         }
 897
 898 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 899 #undef _ctf_array_bitfield
 900 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 901         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 902         if (_user) {                                                    \
 903                 __chan->ops->event_write_from_user(&__ctx, _src, sizeof(_type) * (_length)); \
 904         } else {                                                        \
 905                 __chan->ops->event_write(&__ctx, _src, sizeof(_type) * (_length)); \
 906         }
 907 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 908 /*
 909  * For big endian, we need to byteswap into little endian.
 910  */
 911 #undef _ctf_array_bitfield
 912 #define _ctf_array_bitfield(_type, _item, _src, _length, _user, _nowrite) \
 913         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 914         {                                                               \
 915                 size_t _i;                                              \
 916                                                                         \
 917                 for (_i = 0; _i < (_length); _i++) {                    \
 918                         _type _tmp;                                     \
 919                                                                         \
 920                         if (_user) {                                    \
 921                                 if (get_user(_tmp, (_type *) _src + _i)) \
 922                                         _tmp = 0;                       \
 923                         } else {                                        \
 924                                 _tmp = ((_type *) _src)[_i];            \
 925                         }                                               \
 926                         switch (sizeof(_type)) {                        \
 927                         case 1:                                         \
 928                                 break;                                  \
 929                         case 2:                                         \
 930                                 _tmp = cpu_to_le16(_tmp);               \
 931                                 break;                                  \
 932                         case 4:                                         \
 933                                 _tmp = cpu_to_le32(_tmp);               \
 934                                 break;                                  \
 935                         case 8:                                         \
 936                                 _tmp = cpu_to_le64(_tmp);               \
 937                                 break;                                  \
 938                         default:                                        \
 939                                 BUG_ON(1);                              \
 940                         }                                               \
 941                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
 942                 }                                                       \
 943         }
 944 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 945
 946 #undef _ctf_sequence_encoded
 947 #define _ctf_sequence_encoded(_type, _item, _src, _length_type,         \
 948                         _src_length, _encoding, _byte_order, _base, _user, _nowrite) \
 949         {                                                               \
 950                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx]; \
 951                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 952                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 953         }                                                               \
 954         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 955         if (_user) {                                                    \
 956                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 957                         sizeof(_type) * __get_dynamic_len(dest));       \
 958         } else {                                                        \
 959                 __chan->ops->event_write(&__ctx, _src,                  \
 960                         sizeof(_type) * __get_dynamic_len(dest));       \
 961         }
 962
 963 #if (__BYTE_ORDER == __LITTLE_ENDIAN)
 964 #undef _ctf_sequence_bitfield
 965 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 966                         _length_type, _src_length,              \
 967                         _user, _nowrite)                        \
 968         {                                                               \
 969                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 970                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 971                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 972         }                                                               \
 973         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 974         if (_user) {                                                    \
 975                 __chan->ops->event_write_from_user(&__ctx, _src,        \
 976                         sizeof(_type) * __get_dynamic_len(dest));       \
 977         } else {                                                        \
 978                 __chan->ops->event_write(&__ctx, _src,                  \
 979                         sizeof(_type) * __get_dynamic_len(dest));       \
 980         }
 981 #else /* #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
 982 /*
 983  * For big endian, we need to byteswap into little endian.
 984  */
 985 #undef _ctf_sequence_bitfield
 986 #define _ctf_sequence_bitfield(_type, _item, _src,              \
 987                         _length_type, _src_length,              \
 988                         _user, _nowrite)                        \
 989         {                                                       \
 990                 _length_type __tmpl = this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \
 991                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\
 992                 __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\
 993         }                                                               \
 994         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));        \
 995         {                                                               \
 996                 size_t _i, _length;                                     \
 997                                                                         \
 998                 _length = __get_dynamic_len(dest);                      \
 999                 for (_i = 0; _i < _length; _i++) {                      \
1000                         _type _tmp;                                     \
1001                                                                         \
1002                         if (_user) {                                    \
1003                                 if (get_user(_tmp, (_type *) _src + _i)) \
1004                                         _tmp = 0;                       \
1005                         } else {                                        \
1006                                 _tmp = ((_type *) _src)[_i];            \
1007                         }                                               \
1008                         switch (sizeof(_type)) {                        \
1009                         case 1:                                         \
1010                                 break;                                  \
1011                         case 2:                                         \
1012                                 _tmp = cpu_to_le16(_tmp);               \
1013                                 break;                                  \
1014                         case 4:                                         \
1015                                 _tmp = cpu_to_le32(_tmp);               \
1016                                 break;                                  \
1017                         case 8:                                         \
1018                                 _tmp = cpu_to_le64(_tmp);               \
1019                                 break;                                  \
1020                         default:                                        \
1021                                 BUG_ON(1);                              \
1022                         }                                               \
1023                         __chan->ops->event_write(&__ctx, &_tmp, sizeof(_type)); \
1024                 }                                                       \
1025         }
1026 #endif /* #else #if (__BYTE_ORDER == __LITTLE_ENDIAN) */
1027
1028 #undef _ctf_string
1029 #define _ctf_string(_item, _src, _user, _nowrite)                       \
1030         if (_user) {                                                    \
1031                 lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(*(_src))); \
1032                 __chan->ops->event_strcpy_from_user(&__ctx, _src,       \
1033                         __get_dynamic_len(dest));                       \
1034         } else {                                                        \
1035                 const char *__ctf_tmp_string =                          \
1036                         ((_src) ? (_src) : __LTTNG_NULL_STRING);        \
1037                 lib_ring_buffer_align_ctx(&__ctx,                       \
1038                         lttng_alignof(*__ctf_tmp_string));              \
1039                 __chan->ops->event_strcpy(&__ctx, __ctf_tmp_string,     \
1040                         __get_dynamic_len(dest));                       \
1041         }
1042
1043 #undef _ctf_enum
1044 #define _ctf_enum(_name, _type, _item, _src, _user, _nowrite)           \
1045         _ctf_integer_ext(_type, _item, _src, __BYTE_ORDER, 10, _user, _nowrite)
1046
1047 #undef ctf_align
1048 #define ctf_align(_type)                                                \
1049         lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_type));
1050
1051 #undef ctf_custom_field
1052 #define ctf_custom_field(_type, _item, _code)           _code
1053
1054 #undef ctf_custom_code
1055 #define ctf_custom_code(...)                                            \
1056         {                                                               \
1057                 __VA_ARGS__                                             \
1058         }
1059
1060 /* Beware: this get len actually consumes the len value */
1061 #undef __get_dynamic_len
1062 #define __get_dynamic_len(field)        this_cpu_ptr(&lttng_dynamic_len_stack)->stack[__dynamic_len_idx++]
1063
1064 #undef TP_PROTO
1065 #define TP_PROTO(...)   __VA_ARGS__
1066
1067 #undef TP_ARGS
1068 #define TP_ARGS(...)    __VA_ARGS__
1069
1070 #undef TP_FIELDS
1071 #define TP_FIELDS(...)  __VA_ARGS__
1072
1073 #undef TP_locvar
1074 #define TP_locvar(...)  __VA_ARGS__
1075
1076 #undef TP_code_pre
1077 #define TP_code_pre(...)        __VA_ARGS__
1078
1079 #undef TP_code_post
1080 #define TP_code_post(...)       __VA_ARGS__
1081
1082 /*
1083  * For state dump, check that "session" argument (mandatory) matches the
1084  * session this event belongs to. Ensures that we write state dump data only
1085  * into the started session, not into all sessions.
1086  */
1087 #ifdef TP_SESSION_CHECK
1088 #define _TP_SESSION_CHECK(session, csession)    (session == csession)
1089 #else /* TP_SESSION_CHECK */
1090 #define _TP_SESSION_CHECK(session, csession)    1
1091 #endif /* TP_SESSION_CHECK */
1092
1093 /*
1094  * Using twice size for filter stack data to hold size and pointer for
1095  * each field (worse case). For integers, max size required is 64-bit.
1096  * Same for double-precision floats. Those fit within
1097  * 2*sizeof(unsigned long) for all supported architectures.
1098  * Perform UNION (||) of filter runtime list.
1099  */
1100 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE
1101 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \
1102 static void __event_probe__##_name(void *__data, _proto)                      \
1103 {                                                                             \
1104         struct probe_local_vars { _locvar };                                  \
1105         struct lttng_event *__event = __data;                                 \
1106         struct lttng_probe_ctx __lttng_probe_ctx = {                                  \
1107                 .event = __event,                                             \
1108                 .interruptible = !irqs_disabled(),                            \
1109         };                                                                    \
1110         struct lttng_channel *__chan = __event->chan;                         \
1111         struct lttng_session *__session = __chan->session;                    \
1112         struct lib_ring_buffer_ctx __ctx;                                     \
1113         ssize_t __event_len;                                                  \
1114         size_t __event_align;                                                 \
1115         size_t __orig_dynamic_len_offset, __dynamic_len_idx;                  \
1116         union {                                                               \
1117                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
1118                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1119         } __stackvar;                                                         \
1120         int __ret;                                                            \
1121         struct probe_local_vars __tp_locvar;                                  \
1122         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
1123                         &__tp_locvar;                                         \
1124         struct lttng_pid_tracker *__lpf;                                      \
1125                                                                               \
1126         if (!_TP_SESSION_CHECK(session, __session))                           \
1127                 return;                                                       \
1128         if (unlikely(!ACCESS_ONCE(__session->active)))                        \
1129                 return;                                                       \
1130         if (unlikely(!ACCESS_ONCE(__chan->enabled)))                          \
1131                 return;                                                       \
1132         if (unlikely(!ACCESS_ONCE(__event->enabled)))                         \
1133                 return;                                                       \
1134         __lpf = lttng_rcu_dereference(__session->pid_tracker);                \
1135         if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid)))  \
1136                 return;                                                       \
1137         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
1138         __dynamic_len_idx = __orig_dynamic_len_offset;                        \
1139         _code_pre                                                             \
1140         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
1141                 struct lttng_bytecode_runtime *bc_runtime;                    \
1142                 int __filter_record = __event->has_enablers_without_bytecode; \
1143                                                                               \
1144                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
1145                                 tp_locvar, _args);                                    \
1146                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
1147                         if (unlikely(bc_runtime->filter(bc_runtime, &__lttng_probe_ctx,       \
1148                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) \
1149                                 __filter_record = 1;                          \
1150                 }                                                             \
1151                 if (likely(!__filter_record))                                 \
1152                         goto __post;                                          \
1153         }                                                                     \
1154         __event_len = __event_get_size__##_name(tp_locvar, _args);            \
1155         if (unlikely(__event_len < 0)) {                                      \
1156                 lib_ring_buffer_lost_event_too_big(__chan->chan);             \
1157                 goto __post;                                                  \
1158         }                                                                     \
1159         __event_align = __event_get_align__##_name(tp_locvar, _args);         \
1160         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
1161                                  __event_align, -1);                          \
1162         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1163         if (__ret < 0)                                                        \
1164                 goto __post;                                                  \
1165         _fields                                                               \
1166         __chan->ops->event_commit(&__ctx);                                    \
1167 __post:                                                                       \
1168         _code_post                                                            \
1169         barrier();      /* use before un-reserve. */                          \
1170         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1171         return;                                                               \
1172 }
1173
1174 #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS
1175 #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \
1176 static void __event_probe__##_name(void *__data)                              \
1177 {                                                                             \
1178         struct probe_local_vars { _locvar };                                  \
1179         struct lttng_event *__event = __data;                                 \
1180         struct lttng_probe_ctx __lttng_probe_ctx = {                                  \
1181                 .event = __event,                                             \
1182                 .interruptible = !irqs_disabled(),                            \
1183         };                                                                    \
1184         struct lttng_channel *__chan = __event->chan;                         \
1185         struct lttng_session *__session = __chan->session;                    \
1186         struct lib_ring_buffer_ctx __ctx;                                     \
1187         ssize_t __event_len;                                                  \
1188         size_t __event_align;                                                 \
1189         size_t __orig_dynamic_len_offset, __dynamic_len_idx;                  \
1190         union {                                                               \
1191                 size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)];   \
1192                 char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \
1193         } __stackvar;                                                         \
1194         int __ret;                                                            \
1195         struct probe_local_vars __tp_locvar;                                  \
1196         struct probe_local_vars *tp_locvar __attribute__((unused)) =          \
1197                         &__tp_locvar;                                         \
1198         struct lttng_pid_tracker *__lpf;                                      \
1199                                                                               \
1200         if (!_TP_SESSION_CHECK(session, __session))                           \
1201                 return;                                                       \
1202         if (unlikely(!ACCESS_ONCE(__session->active)))                        \
1203                 return;                                                       \
1204         if (unlikely(!ACCESS_ONCE(__chan->enabled)))                          \
1205                 return;                                                       \
1206         if (unlikely(!ACCESS_ONCE(__event->enabled)))                         \
1207                 return;                                                       \
1208         __lpf = lttng_rcu_dereference(__session->pid_tracker);                \
1209         if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid)))  \
1210                 return;                                                       \
1211         __orig_dynamic_len_offset = this_cpu_ptr(&lttng_dynamic_len_stack)->offset; \
1212         __dynamic_len_idx = __orig_dynamic_len_offset;                        \
1213         _code_pre                                                             \
1214         if (unlikely(!list_empty(&__event->bytecode_runtime_head))) {         \
1215                 struct lttng_bytecode_runtime *bc_runtime;                    \
1216                 int __filter_record = __event->has_enablers_without_bytecode; \
1217                                                                               \
1218                 __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \
1219                                 tp_locvar);                                   \
1220                 lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \
1221                         if (unlikely(bc_runtime->filter(bc_runtime, &__lttng_probe_ctx, \
1222                                         __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) \
1223                                 __filter_record = 1;                          \
1224                 }                                                             \
1225                 if (likely(!__filter_record))                                 \
1226                         goto __post;                                          \
1227         }                                                                     \
1228         __event_len = __event_get_size__##_name(tp_locvar);                   \
1229         if (unlikely(__event_len < 0)) {                                      \
1230                 lib_ring_buffer_lost_event_too_big(__chan->chan);             \
1231                 goto __post;                                                  \
1232         }                                                                     \
1233         __event_align = __event_get_align__##_name(tp_locvar);                \
1234         lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len,  \
1235                                  __event_align, -1);                          \
1236         __ret = __chan->ops->event_reserve(&__ctx, __event->id);              \
1237         if (__ret < 0)                                                        \
1238                 goto __post;                                                  \
1239         _fields                                                               \
1240         __chan->ops->event_commit(&__ctx);                                    \
1241 __post:                                                                       \
1242         _code_post                                                            \
1243         barrier();      /* use before un-reserve. */                          \
1244         this_cpu_ptr(&lttng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \
1245         return;                                                               \
1246 }
1247
1248 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1249
1250 #undef __get_dynamic_len
1251
1252 /*
1253  * Stage 7 of the trace events.
1254  *
1255  * Create event descriptions.
1256  */
1257
1258 /* Named field types must be defined in lttng-types.h */
1259
1260 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1261
1262 #ifndef TP_PROBE_CB
1263 #define TP_PROBE_CB(_template)  &__event_probe__##_template
1264 #endif
1265
1266 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1267 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)      \
1268 static const struct lttng_event_desc __event_desc___##_map = {          \
1269         .fields = __event_fields___##_template,                         \
1270         .name = #_map,                                                  \
1271         .kname = #_name,                                                \
1272         .probe_callback = (void *) TP_PROBE_CB(_template),              \
1273         .nr_fields = ARRAY_SIZE(__event_fields___##_template),          \
1274         .owner = THIS_MODULE,                                           \
1275 };
1276
1277 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1278 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1279         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1280
1281 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1282
1283 /*
1284  * Stage 8 of the trace events.
1285  *
1286  * Create an array of event description pointers.
1287  */
1288
1289 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1290
1291 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS
1292 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \
1293                 &__event_desc___##_map,
1294
1295 #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP
1296 #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(_template, _name, _map, _proto, _args) \
1297         LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map)
1298
1299 #define TP_ID1(_token, _system) _token##_system
1300 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1301
1302 static const struct lttng_event_desc *TP_ID(__event_desc___, TRACE_SYSTEM)[] = {
1303 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
1304 };
1305
1306 #undef TP_ID1
1307 #undef TP_ID
1308
1309 /*
1310  * Stage 9 of the trace events.
1311  *
1312  * Create a toplevel descriptor for the whole probe.
1313  */
1314
1315 #define TP_ID1(_token, _system) _token##_system
1316 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1317
1318 /* non-const because list head will be modified when registered. */
1319 static __used struct lttng_probe_desc TP_ID(__probe_desc___, TRACE_SYSTEM) = {
1320         .provider = __stringify(TRACE_SYSTEM),
1321         .event_desc = TP_ID(__event_desc___, TRACE_SYSTEM),
1322         .nr_events = ARRAY_SIZE(TP_ID(__event_desc___, TRACE_SYSTEM)),
1323         .head = { NULL, NULL },
1324         .lazy_init_head = { NULL, NULL },
1325         .lazy = 0,
1326 };
1327
1328 #undef TP_ID1
1329 #undef TP_ID
1330
1331 /*
1332  * Stage 10 of the trace events.
1333  *
1334  * Register/unregister probes at module load/unload.
1335  */
1336
1337 #include <probes/lttng-events-reset.h>  /* Reset all macros within LTTNG_TRACEPOINT_EVENT */
1338
1339 #define TP_ID1(_token, _system) _token##_system
1340 #define TP_ID(_token, _system)  TP_ID1(_token, _system)
1341 #define module_init_eval1(_token, _system)      module_init(_token##_system)
1342 #define module_init_eval(_token, _system)       module_init_eval1(_token, _system)
1343 #define module_exit_eval1(_token, _system)      module_exit(_token##_system)
1344 #define module_exit_eval(_token, _system)       module_exit_eval1(_token, _system)
1345
1346 #ifndef TP_MODULE_NOINIT
1347 static int TP_ID(__lttng_events_init__, TRACE_SYSTEM)(void)
1348 {
1349         wrapper_vmalloc_sync_all();
1350         return lttng_probe_register(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1351 }
1352
1353 static void TP_ID(__lttng_events_exit__, TRACE_SYSTEM)(void)
1354 {
1355         lttng_probe_unregister(&TP_ID(__probe_desc___, TRACE_SYSTEM));
1356 }
1357
1358 #ifndef TP_MODULE_NOAUTOLOAD
1359 module_init_eval(__lttng_events_init__, TRACE_SYSTEM);
1360 module_exit_eval(__lttng_events_exit__, TRACE_SYSTEM);
1361 #endif
1362
1363 #endif
1364
1365 #undef module_init_eval
1366 #undef module_exit_eval
1367 #undef TP_ID1
1368 #undef TP_ID
1369
1370 #undef TP_PROTO
1371 #undef TP_ARGS