liblttng-ust-libc-wrapper/lttng-ust-malloc.c

   1 /*
   2  * SPDX-License-Identifier: LGPL-2.1-or-later
   3  *
   4  * Copyright (C) 2009 Pierre-Marc Fournier
   5  * Copyright (C) 2011 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  */
   7
   8 /*
   9  * Do _not_ define _LGPL_SOURCE because we don't want to create a
  10  * circular dependency loop between this malloc wrapper, liburcu and
  11  * libc.
  12  */
  13 #include <ust-dlfcn.h>
  14 #include <sys/types.h>
  15 #include <stdio.h>
  16 #include <assert.h>
  17 #include <malloc.h>
  18
  19 #include <urcu/system.h>
  20 #include <urcu/uatomic.h>
  21 #include <urcu/compiler.h>
  22 #include <urcu/tls-compat.h>
  23 #include <urcu/arch.h>
  24
  25 #include <lttng/ust-libc-wrapper.h>
  26
  27 #include <ust-helper.h>
  28 #include "ust-compat.h"
  29
  30 #define TRACEPOINT_DEFINE
  31 #define TRACEPOINT_CREATE_PROBES
  32 #define TP_IP_PARAM ip
  33 #include "ust_libc.h"
  34
  35 #define STATIC_CALLOC_LEN 4096
  36 static char static_calloc_buf[STATIC_CALLOC_LEN];
  37 static unsigned long static_calloc_buf_offset;
  38
  39 struct alloc_functions {
  40         void *(*calloc)(size_t nmemb, size_t size);
  41         void *(*malloc)(size_t size);
  42         void (*free)(void *ptr);
  43         void *(*realloc)(void *ptr, size_t size);
  44         void *(*memalign)(size_t alignment, size_t size);
  45         int (*posix_memalign)(void **memptr, size_t alignment, size_t size);
  46 };
  47
  48 static
  49 struct alloc_functions cur_alloc;
  50
  51 /*
  52  * Make sure our own use of the LTS compat layer will not cause infinite
  53  * recursion by calling calloc.
  54  */
  55
  56 static
  57 void *static_calloc(size_t nmemb, size_t size);
  58
  59 /*
  60  * pthread mutex replacement for URCU tls compat layer.
  61  */
  62 static int ust_malloc_lock;
  63
  64 static
  65 void ust_malloc_spin_lock(pthread_mutex_t *lock)
  66         __attribute__((unused));
  67 static
  68 void ust_malloc_spin_lock(pthread_mutex_t *lock __attribute__((unused)))
  69 {
  70         /*
  71          * The memory barrier within cmpxchg takes care of ordering
  72          * memory accesses with respect to the start of the critical
  73          * section.
  74          */
  75         while (uatomic_cmpxchg(&ust_malloc_lock, 0, 1) != 0)
  76                 caa_cpu_relax();
  77 }
  78
  79 static
  80 void ust_malloc_spin_unlock(pthread_mutex_t *lock)
  81         __attribute__((unused));
  82 static
  83 void ust_malloc_spin_unlock(pthread_mutex_t *lock __attribute__((unused)))
  84 {
  85         /*
  86          * Ensure memory accesses within the critical section do not
  87          * leak outside.
  88          */
  89         cmm_smp_mb();
  90         uatomic_set(&ust_malloc_lock, 0);
  91 }
  92
  93 #define calloc static_calloc
  94 #define pthread_mutex_lock ust_malloc_spin_lock
  95 #define pthread_mutex_unlock ust_malloc_spin_unlock
  96 static DEFINE_URCU_TLS(int, malloc_nesting);
  97 #undef pthread_mutex_unlock
  98 #undef pthread_mutex_lock
  99 #undef calloc
 100
 101 /*
 102  * Static allocator to use when initially executing dlsym(). It keeps a
 103  * size_t value of each object size prior to the object.
 104  */
 105 static
 106 void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
 107 {
 108         size_t prev_offset, new_offset, res_offset, aligned_offset;
 109
 110         if (nmemb * size == 0) {
 111                 return NULL;
 112         }
 113
 114         /*
 115          * Protect static_calloc_buf_offset from concurrent updates
 116          * using a cmpxchg loop rather than a mutex to remove a
 117          * dependency on pthread. This will minimize the risk of bad
 118          * interaction between mutex and malloc instrumentation.
 119          */
 120         res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
 121         do {
 122                 prev_offset = res_offset;
 123                 aligned_offset = LTTNG_UST_ALIGN(prev_offset + sizeof(size_t), alignment);
 124                 new_offset = aligned_offset + nmemb * size;
 125                 if (new_offset > sizeof(static_calloc_buf)) {
 126                         abort();
 127                 }
 128         } while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
 129                         prev_offset, new_offset)) != prev_offset);
 130         *(size_t *) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
 131         return &static_calloc_buf[aligned_offset];
 132 }
 133
 134 static
 135 void *static_calloc(size_t nmemb, size_t size)
 136 {
 137         void *retval;
 138
 139         retval = static_calloc_aligned(nmemb, size, 1);
 140         return retval;
 141 }
 142
 143 static
 144 void *static_malloc(size_t size)
 145 {
 146         void *retval;
 147
 148         retval = static_calloc_aligned(1, size, 1);
 149         return retval;
 150 }
 151
 152 static
 153 void static_free(void *ptr __attribute__((unused)))
 154 {
 155         /* no-op. */
 156 }
 157
 158 static
 159 void *static_realloc(void *ptr, size_t size)
 160 {
 161         size_t *old_size = NULL;
 162         void *retval;
 163
 164         if (size == 0) {
 165                 retval = NULL;
 166                 goto end;
 167         }
 168
 169         if (ptr) {
 170                 old_size = (size_t *) ptr - 1;
 171                 if (size <= *old_size) {
 172                         /* We can re-use the old entry. */
 173                         *old_size = size;
 174                         retval = ptr;
 175                         goto end;
 176                 }
 177         }
 178         /* We need to expand. Don't free previous memory location. */
 179         retval = static_calloc_aligned(1, size, 1);
 180         assert(retval);
 181         if (ptr)
 182                 memcpy(retval, ptr, *old_size);
 183 end:
 184         return retval;
 185 }
 186
 187 static
 188 void *static_memalign(size_t alignment, size_t size)
 189 {
 190         void *retval;
 191
 192         retval = static_calloc_aligned(1, size, alignment);
 193         return retval;
 194 }
 195
 196 static
 197 int static_posix_memalign(void **memptr, size_t alignment, size_t size)
 198 {
 199         void *ptr;
 200
 201         /* Check for power of 2, larger than void *. */
 202         if (alignment & (alignment - 1)
 203                         || alignment < sizeof(void *)
 204                         || alignment == 0) {
 205                 goto end;
 206         }
 207         ptr = static_calloc_aligned(1, size, alignment);
 208         *memptr = ptr;
 209 end:
 210         return 0;
 211 }
 212
 213 static
 214 void setup_static_allocator(void)
 215 {
 216         assert(cur_alloc.calloc == NULL);
 217         cur_alloc.calloc = static_calloc;
 218         assert(cur_alloc.malloc == NULL);
 219         cur_alloc.malloc = static_malloc;
 220         assert(cur_alloc.free == NULL);
 221         cur_alloc.free = static_free;
 222         assert(cur_alloc.realloc == NULL);
 223         cur_alloc.realloc = static_realloc;
 224         assert(cur_alloc.memalign == NULL);
 225         cur_alloc.memalign = static_memalign;
 226         assert(cur_alloc.posix_memalign == NULL);
 227         cur_alloc.posix_memalign = static_posix_memalign;
 228 }
 229
 230 static
 231 void lookup_all_symbols(void)
 232 {
 233         struct alloc_functions af;
 234
 235         /*
 236          * Temporarily redirect allocation functions to
 237          * static_calloc_aligned, and free function to static_free
 238          * (no-op), until the dlsym lookup has completed.
 239          */
 240         setup_static_allocator();
 241
 242         /* Perform the actual lookups */
 243         af.calloc = dlsym(RTLD_NEXT, "calloc");
 244         af.malloc = dlsym(RTLD_NEXT, "malloc");
 245         af.free = dlsym(RTLD_NEXT, "free");
 246         af.realloc = dlsym(RTLD_NEXT, "realloc");
 247         af.memalign = dlsym(RTLD_NEXT, "memalign");
 248         af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");
 249
 250         /* Populate the new allocator functions */
 251         memcpy(&cur_alloc, &af, sizeof(cur_alloc));
 252 }
 253
 254 void *malloc(size_t size)
 255 {
 256         void *retval;
 257
 258         URCU_TLS(malloc_nesting)++;
 259         if (cur_alloc.malloc == NULL) {
 260                 lookup_all_symbols();
 261                 if (cur_alloc.malloc == NULL) {
 262                         fprintf(stderr, "mallocwrap: unable to find malloc\n");
 263                         abort();
 264                 }
 265         }
 266         retval = cur_alloc.malloc(size);
 267         if (URCU_TLS(malloc_nesting) == 1) {
 268                 tracepoint(lttng_ust_libc, malloc,
 269                         size, retval, LTTNG_UST_CALLER_IP());
 270         }
 271         URCU_TLS(malloc_nesting)--;
 272         return retval;
 273 }
 274
 275 void free(void *ptr)
 276 {
 277         URCU_TLS(malloc_nesting)++;
 278         /*
 279          * Check whether the memory was allocated with
 280          * static_calloc_align, in which case there is nothing to free.
 281          */
 282         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 283                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 284                 goto end;
 285         }
 286
 287         if (URCU_TLS(malloc_nesting) == 1) {
 288                 tracepoint(lttng_ust_libc, free,
 289                         ptr, LTTNG_UST_CALLER_IP());
 290         }
 291
 292         if (cur_alloc.free == NULL) {
 293                 lookup_all_symbols();
 294                 if (cur_alloc.free == NULL) {
 295                         fprintf(stderr, "mallocwrap: unable to find free\n");
 296                         abort();
 297                 }
 298         }
 299         cur_alloc.free(ptr);
 300 end:
 301         URCU_TLS(malloc_nesting)--;
 302 }
 303
 304 void *calloc(size_t nmemb, size_t size)
 305 {
 306         void *retval;
 307
 308         URCU_TLS(malloc_nesting)++;
 309         if (cur_alloc.calloc == NULL) {
 310                 lookup_all_symbols();
 311                 if (cur_alloc.calloc == NULL) {
 312                         fprintf(stderr, "callocwrap: unable to find calloc\n");
 313                         abort();
 314                 }
 315         }
 316         retval = cur_alloc.calloc(nmemb, size);
 317         if (URCU_TLS(malloc_nesting) == 1) {
 318                 tracepoint(lttng_ust_libc, calloc,
 319                         nmemb, size, retval, LTTNG_UST_CALLER_IP());
 320         }
 321         URCU_TLS(malloc_nesting)--;
 322         return retval;
 323 }
 324
 325 void *realloc(void *ptr, size_t size)
 326 {
 327         void *retval;
 328
 329         URCU_TLS(malloc_nesting)++;
 330         /*
 331          * Check whether the memory was allocated with
 332          * static_calloc_align, in which case there is nothing
 333          * to free, and we need to copy the old data.
 334          */
 335         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 336                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 337                 size_t *old_size;
 338
 339                 old_size = (size_t *) ptr - 1;
 340                 if (cur_alloc.calloc == NULL) {
 341                         lookup_all_symbols();
 342                         if (cur_alloc.calloc == NULL) {
 343                                 fprintf(stderr, "reallocwrap: unable to find calloc\n");
 344                                 abort();
 345                         }
 346                 }
 347                 retval = cur_alloc.calloc(1, size);
 348                 if (retval) {
 349                         memcpy(retval, ptr, *old_size);
 350                 }
 351                 /*
 352                  * Mimick that a NULL pointer has been received, so
 353                  * memory allocation analysis based on the trace don't
 354                  * get confused by the address from the static
 355                  * allocator.
 356                  */
 357                 ptr = NULL;
 358                 goto end;
 359         }
 360
 361         if (cur_alloc.realloc == NULL) {
 362                 lookup_all_symbols();
 363                 if (cur_alloc.realloc == NULL) {
 364                         fprintf(stderr, "reallocwrap: unable to find realloc\n");
 365                         abort();
 366                 }
 367         }
 368         retval = cur_alloc.realloc(ptr, size);
 369 end:
 370         if (URCU_TLS(malloc_nesting) == 1) {
 371                 tracepoint(lttng_ust_libc, realloc,
 372                         ptr, size, retval, LTTNG_UST_CALLER_IP());
 373         }
 374         URCU_TLS(malloc_nesting)--;
 375         return retval;
 376 }
 377
 378 void *memalign(size_t alignment, size_t size)
 379 {
 380         void *retval;
 381
 382         URCU_TLS(malloc_nesting)++;
 383         if (cur_alloc.memalign == NULL) {
 384                 lookup_all_symbols();
 385                 if (cur_alloc.memalign == NULL) {
 386                         fprintf(stderr, "memalignwrap: unable to find memalign\n");
 387                         abort();
 388                 }
 389         }
 390         retval = cur_alloc.memalign(alignment, size);
 391         if (URCU_TLS(malloc_nesting) == 1) {
 392                 tracepoint(lttng_ust_libc, memalign,
 393                         alignment, size, retval,
 394                         LTTNG_UST_CALLER_IP());
 395         }
 396         URCU_TLS(malloc_nesting)--;
 397         return retval;
 398 }
 399
 400 int posix_memalign(void **memptr, size_t alignment, size_t size)
 401 {
 402         int retval;
 403
 404         URCU_TLS(malloc_nesting)++;
 405         if (cur_alloc.posix_memalign == NULL) {
 406                 lookup_all_symbols();
 407                 if (cur_alloc.posix_memalign == NULL) {
 408                         fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
 409                         abort();
 410                 }
 411         }
 412         retval = cur_alloc.posix_memalign(memptr, alignment, size);
 413         if (URCU_TLS(malloc_nesting) == 1) {
 414                 tracepoint(lttng_ust_libc, posix_memalign,
 415                         *memptr, alignment, size,
 416                         retval, LTTNG_UST_CALLER_IP());
 417         }
 418         URCU_TLS(malloc_nesting)--;
 419         return retval;
 420 }
 421
 422 static
 423 void lttng_ust_fixup_malloc_nesting_tls(void)
 424 {
 425         asm volatile ("" : : "m" (URCU_TLS(malloc_nesting)));
 426 }
 427
 428 void lttng_ust_libc_wrapper_malloc_init(void)
 429 {
 430         /* Initialization already done */
 431         if (cur_alloc.calloc) {
 432                 return;
 433         }
 434         lttng_ust_fixup_malloc_nesting_tls();
 435         /*
 436          * Ensure the allocator is in place before the process becomes
 437          * multithreaded.
 438          */
 439         lookup_all_symbols();
 440 }