[lttng-ust.git] / liblttng-ust-libc-wrapper / lttng-ust-malloc.c

/*
 * Copyright (C) 2009  Pierre-Marc Fournier
 * Copyright (C) 2011  Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
 */

#define _GNU_SOURCE
/*
 * Do _not_ define _LGPL_SOURCE because we don't want to create a
 * circular dependency loop between this malloc wrapper, liburcu and
 * libc.
 */
#include <lttng/ust-dlfcn.h>
#include <sys/types.h>
#include <stdio.h>
#include <assert.h>
#include <urcu/system.h>
#include <urcu/uatomic.h>
#include <urcu/compiler.h>
#include <urcu/tls-compat.h>
#include <urcu/arch.h>
#include <lttng/align.h>
#include <helper.h>

#define TRACEPOINT_DEFINE
#define TRACEPOINT_CREATE_PROBES
#define TP_IP_PARAM ip
#include "ust_libc.h"

#define STATIC_CALLOC_LEN 4096
static char static_calloc_buf[STATIC_CALLOC_LEN];
static unsigned long static_calloc_buf_offset;

struct alloc_functions {
	void *(*calloc)(size_t nmemb, size_t size);
	void *(*malloc)(size_t size);
	void (*free)(void *ptr);
	void *(*realloc)(void *ptr, size_t size);
	void *(*memalign)(size_t alignment, size_t size);
	int (*posix_memalign)(void **memptr, size_t alignment, size_t size);
};

static
struct alloc_functions cur_alloc;

/*
 * Make sure our own use of the LTS compat layer will not cause infinite
 * recursion by calling calloc.
 */

static
void *static_calloc(size_t nmemb, size_t size);

/*
 * pthread mutex replacement for URCU tls compat layer.
 */
static int ust_malloc_lock;

static __attribute__((unused))
void ust_malloc_spin_lock(pthread_mutex_t *lock)
{
	/*
	 * The memory barrier within cmpxchg takes care of ordering
	 * memory accesses with respect to the start of the critical
	 * section.
	 */
	while (uatomic_cmpxchg(&ust_malloc_lock, 0, 1) != 0)
		caa_cpu_relax();
}

static __attribute__((unused))
void ust_malloc_spin_unlock(pthread_mutex_t *lock)
{
	/*
	 * Ensure memory accesses within the critical section do not
	 * leak outside.
	 */
	cmm_smp_mb();
	uatomic_set(&ust_malloc_lock, 0);
}

#define calloc static_calloc
#define pthread_mutex_lock ust_malloc_spin_lock
#define pthread_mutex_unlock ust_malloc_spin_unlock
static DEFINE_URCU_TLS(int, malloc_nesting);
#undef ust_malloc_spin_unlock
#undef ust_malloc_spin_lock
#undef calloc

/*
 * Static allocator to use when initially executing dlsym(). It keeps a
 * size_t value of each object size prior to the object.
 */
static
void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
{
	size_t prev_offset, new_offset, res_offset, aligned_offset;

	if (nmemb * size == 0) {
		return NULL;
	}

	/*
	 * Protect static_calloc_buf_offset from concurrent updates
	 * using a cmpxchg loop rather than a mutex to remove a
	 * dependency on pthread. This will minimize the risk of bad
	 * interaction between mutex and malloc instrumentation.
	 */
	res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
	do {
		prev_offset = res_offset;
		aligned_offset = ALIGN(prev_offset + sizeof(size_t), alignment);
		new_offset = aligned_offset + nmemb * size;
		if (new_offset > sizeof(static_calloc_buf)) {
			abort();
		}
	} while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
			prev_offset, new_offset)) != prev_offset);
	*(size_t *) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
	return &static_calloc_buf[aligned_offset];
}

static
void *static_calloc(size_t nmemb, size_t size)
{
	void *retval;

	retval = static_calloc_aligned(nmemb, size, 1);
	return retval;
}

static
void *static_malloc(size_t size)
{
	void *retval;

	retval = static_calloc_aligned(1, size, 1);
	return retval;
}

static
void static_free(void *ptr)
{
	/* no-op. */
}

static
void *static_realloc(void *ptr, size_t size)
{
	size_t *old_size = NULL;
	void *retval;

	if (size == 0) {
		retval = NULL;
		goto end;
	}

	if (ptr) {
		old_size = (size_t *) ptr - 1;
		if (size <= *old_size) {
			/* We can re-use the old entry. */
			*old_size = size;
			retval = ptr;
			goto end;
		}
	}
	/* We need to expand. Don't free previous memory location. */
	retval = static_calloc_aligned(1, size, 1);
	assert(retval);
	if (ptr)
		memcpy(retval, ptr, *old_size);
end:
	return retval;
}

static
void *static_memalign(size_t alignment, size_t size)
{
	void *retval;

	retval = static_calloc_aligned(1, size, alignment);
	return retval;
}

static
int static_posix_memalign(void **memptr, size_t alignment, size_t size)
{
	void *ptr;

	/* Check for power of 2, larger than void *. */
	if (alignment & (alignment - 1)
			|| alignment < sizeof(void *)
			|| alignment == 0) {
		goto end;
	}
	ptr = static_calloc_aligned(1, size, alignment);
	*memptr = ptr;
end:
	return 0;
}

static
void setup_static_allocator(void)
{
	assert(cur_alloc.calloc == NULL);
	cur_alloc.calloc = static_calloc;
	assert(cur_alloc.malloc == NULL);
	cur_alloc.malloc = static_malloc;
	assert(cur_alloc.free == NULL);
	cur_alloc.free = static_free;
	assert(cur_alloc.realloc == NULL);
	cur_alloc.realloc = static_realloc;
	assert(cur_alloc.memalign == NULL);
	cur_alloc.memalign = static_memalign;
	assert(cur_alloc.posix_memalign == NULL);
	cur_alloc.posix_memalign = static_posix_memalign;
}

static
void lookup_all_symbols(void)
{
	struct alloc_functions af;

	/*
	 * Temporarily redirect allocation functions to
	 * static_calloc_aligned, and free function to static_free
	 * (no-op), until the dlsym lookup has completed.
	 */
	setup_static_allocator();

	/* Perform the actual lookups */
	af.calloc = dlsym(RTLD_NEXT, "calloc");
	af.malloc = dlsym(RTLD_NEXT, "malloc");
	af.free = dlsym(RTLD_NEXT, "free");
	af.realloc = dlsym(RTLD_NEXT, "realloc");
	af.memalign = dlsym(RTLD_NEXT, "memalign");
	af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");

	/* Populate the new allocator functions */
	memcpy(&cur_alloc, &af, sizeof(cur_alloc));
}

void *malloc(size_t size)
{
	void *retval;

	URCU_TLS(malloc_nesting)++;
	if (cur_alloc.malloc == NULL) {
		lookup_all_symbols();
		if (cur_alloc.malloc == NULL) {
			fprintf(stderr, "mallocwrap: unable to find malloc\n");
			abort();
		}
	}
	retval = cur_alloc.malloc(size);
	if (URCU_TLS(malloc_nesting) == 1) {
		tracepoint(lttng_ust_libc, malloc,
			size, retval, LTTNG_UST_CALLER_IP());
	}
	URCU_TLS(malloc_nesting)--;
	return retval;
}

void free(void *ptr)
{
	URCU_TLS(malloc_nesting)++;
	/*
	 * Check whether the memory was allocated with
	 * static_calloc_align, in which case there is nothing to free.
	 */
	if (caa_unlikely((char *)ptr >= static_calloc_buf &&
			(char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
		goto end;
	}

	if (URCU_TLS(malloc_nesting) == 1) {
		tracepoint(lttng_ust_libc, free,
			ptr, LTTNG_UST_CALLER_IP());
	}

	if (cur_alloc.free == NULL) {
		lookup_all_symbols();
		if (cur_alloc.free == NULL) {
			fprintf(stderr, "mallocwrap: unable to find free\n");
			abort();
		}
	}
	cur_alloc.free(ptr);
end:
	URCU_TLS(malloc_nesting)--;
}

void *calloc(size_t nmemb, size_t size)
{
	void *retval;

	URCU_TLS(malloc_nesting)++;
	if (cur_alloc.calloc == NULL) {
		lookup_all_symbols();
		if (cur_alloc.calloc == NULL) {
			fprintf(stderr, "callocwrap: unable to find calloc\n");
			abort();
		}
	}
	retval = cur_alloc.calloc(nmemb, size);
	if (URCU_TLS(malloc_nesting) == 1) {
		tracepoint(lttng_ust_libc, calloc,
			nmemb, size, retval, LTTNG_UST_CALLER_IP());
	}
	URCU_TLS(malloc_nesting)--;
	return retval;
}

void *realloc(void *ptr, size_t size)
{
	void *retval;

	URCU_TLS(malloc_nesting)++;
	/*
	 * Check whether the memory was allocated with
	 * static_calloc_align, in which case there is nothing
	 * to free, and we need to copy the old data.
	 */
	if (caa_unlikely((char *)ptr >= static_calloc_buf &&
			(char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
		size_t *old_size;

		old_size = (size_t *) ptr - 1;
		if (cur_alloc.calloc == NULL) {
			lookup_all_symbols();
			if (cur_alloc.calloc == NULL) {
				fprintf(stderr, "reallocwrap: unable to find calloc\n");
				abort();
			}
		}
		retval = cur_alloc.calloc(1, size);
		if (retval) {
			memcpy(retval, ptr, *old_size);
		}
		/*
		 * Mimick that a NULL pointer has been received, so
		 * memory allocation analysis based on the trace don't
		 * get confused by the address from the static
		 * allocator.
		 */
		ptr = NULL;
		goto end;
	}

	if (cur_alloc.realloc == NULL) {
		lookup_all_symbols();
		if (cur_alloc.realloc == NULL) {
			fprintf(stderr, "reallocwrap: unable to find realloc\n");
			abort();
		}
	}
	retval = cur_alloc.realloc(ptr, size);
end:
	if (URCU_TLS(malloc_nesting) == 1) {
		tracepoint(lttng_ust_libc, realloc,
			ptr, size, retval, LTTNG_UST_CALLER_IP());
	}
	URCU_TLS(malloc_nesting)--;
	return retval;
}

void *memalign(size_t alignment, size_t size)
{
	void *retval;

	URCU_TLS(malloc_nesting)++;
	if (cur_alloc.memalign == NULL) {
		lookup_all_symbols();
		if (cur_alloc.memalign == NULL) {
			fprintf(stderr, "memalignwrap: unable to find memalign\n");
			abort();
		}
	}
	retval = cur_alloc.memalign(alignment, size);
	if (URCU_TLS(malloc_nesting) == 1) {
		tracepoint(lttng_ust_libc, memalign,
			alignment, size, retval,
			LTTNG_UST_CALLER_IP());
	}
	URCU_TLS(malloc_nesting)--;
	return retval;
}

int posix_memalign(void **memptr, size_t alignment, size_t size)
{
	int retval;

	URCU_TLS(malloc_nesting)++;
	if (cur_alloc.posix_memalign == NULL) {
		lookup_all_symbols();
		if (cur_alloc.posix_memalign == NULL) {
			fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
			abort();
		}
	}
	retval = cur_alloc.posix_memalign(memptr, alignment, size);
	if (URCU_TLS(malloc_nesting) == 1) {
		tracepoint(lttng_ust_libc, posix_memalign,
			*memptr, alignment, size,
			retval, LTTNG_UST_CALLER_IP());
	}
	URCU_TLS(malloc_nesting)--;
	return retval;
}

static
void lttng_ust_fixup_malloc_nesting_tls(void)
{
	asm volatile ("" : : "m" (URCU_TLS(malloc_nesting)));
}

__attribute__((constructor))
void lttng_ust_malloc_wrapper_init(void)
{
	/* Initialization already done */
	if (cur_alloc.calloc) {
		return;
	}
	lttng_ust_fixup_malloc_nesting_tls();
	/*
	 * Ensure the allocator is in place before the process becomes
	 * multithreaded.
	 */
	lookup_all_symbols();
}
Commit	Line	Data
	1	/*
	2	* Copyright (C) 2009 Pierre-Marc Fournier
	3	* Copyright (C) 2011 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	4	*
	5	* This library is free software; you can redistribute it and/or
	6	* modify it under the terms of the GNU Lesser General Public
	7	* License as published by the Free Software Foundation; either
	8	* version 2.1 of the License, or (at your option) any later version.
	9	*
	10	* This library is distributed in the hope that it will be useful,
	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	* Lesser General Public License for more details.
	14	*
	15	* You should have received a copy of the GNU Lesser General Public
	16	* License along with this library; if not, write to the Free Software
	17	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	18	*/
	19
	20	#define _GNU_SOURCE
	21	/*
	22	* Do _not_ define _LGPL_SOURCE because we don't want to create a
	23	* circular dependency loop between this malloc wrapper, liburcu and
	24	* libc.
	25	*/
	26	#include <lttng/ust-dlfcn.h>
	27	#include <sys/types.h>
	28	#include <stdio.h>
	29	#include <assert.h>
	30	#include <urcu/system.h>
	31	#include <urcu/uatomic.h>
	32	#include <urcu/compiler.h>
	33	#include <urcu/tls-compat.h>
	34	#include <urcu/arch.h>
	35	#include <lttng/align.h>
	36	#include <helper.h>
	37
	38	#define TRACEPOINT_DEFINE
	39	#define TRACEPOINT_CREATE_PROBES
	40	#define TP_IP_PARAM ip
	41	#include "ust_libc.h"
	42
	43	#define STATIC_CALLOC_LEN 4096
	44	static char static_calloc_buf[STATIC_CALLOC_LEN];
	45	static unsigned long static_calloc_buf_offset;
	46
	47	struct alloc_functions {
	48	void (calloc)(size_t nmemb, size_t size);
	49	void (malloc)(size_t size);
	50	void (free)(void ptr);
	51	void (realloc)(void *ptr, size_t size);
	52	void (memalign)(size_t alignment, size_t size);
	53	int (posix_memalign)(void *memptr, size_t alignment, size_t size);
	54	};
	55
	56	static
	57	struct alloc_functions cur_alloc;
	58
	59	/*
	60	* Make sure our own use of the LTS compat layer will not cause infinite
	61	* recursion by calling calloc.
	62	*/
	63
	64	static
	65	void *static_calloc(size_t nmemb, size_t size);
	66
	67	/*
	68	* pthread mutex replacement for URCU tls compat layer.
	69	*/
	70	static int ust_malloc_lock;
	71
	72	static __attribute__((unused))
	73	void ust_malloc_spin_lock(pthread_mutex_t *lock)
	74	{
	75	/*
	76	* The memory barrier within cmpxchg takes care of ordering
	77	* memory accesses with respect to the start of the critical
	78	* section.
	79	*/
	80	while (uatomic_cmpxchg(&ust_malloc_lock, 0, 1) != 0)
	81	caa_cpu_relax();
	82	}
	83
	84	static __attribute__((unused))
	85	void ust_malloc_spin_unlock(pthread_mutex_t *lock)
	86	{
	87	/*
	88	* Ensure memory accesses within the critical section do not
	89	* leak outside.
	90	*/
	91	cmm_smp_mb();
	92	uatomic_set(&ust_malloc_lock, 0);
	93	}
	94
	95	#define calloc static_calloc
	96	#define pthread_mutex_lock ust_malloc_spin_lock
	97	#define pthread_mutex_unlock ust_malloc_spin_unlock
	98	static DEFINE_URCU_TLS(int, malloc_nesting);
	99	#undef ust_malloc_spin_unlock
	100	#undef ust_malloc_spin_lock
	101	#undef calloc
	102
	103	/*
	104	* Static allocator to use when initially executing dlsym(). It keeps a
	105	* size_t value of each object size prior to the object.
	106	*/
	107	static
	108	void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
	109	{
	110	size_t prev_offset, new_offset, res_offset, aligned_offset;
	111
	112	if (nmemb * size == 0) {
	113	return NULL;
	114	}
	115
	116	/*
	117	* Protect static_calloc_buf_offset from concurrent updates
	118	* using a cmpxchg loop rather than a mutex to remove a
	119	* dependency on pthread. This will minimize the risk of bad
	120	* interaction between mutex and malloc instrumentation.
	121	*/
	122	res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
	123	do {
	124	prev_offset = res_offset;
	125	aligned_offset = ALIGN(prev_offset + sizeof(size_t), alignment);
	126	new_offset = aligned_offset + nmemb * size;
	127	if (new_offset > sizeof(static_calloc_buf)) {
	128	abort();
	129	}
	130	} while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
	131	prev_offset, new_offset)) != prev_offset);
	132	(size_t ) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
	133	return &static_calloc_buf[aligned_offset];
	134	}
	135
	136	static
	137	void *static_calloc(size_t nmemb, size_t size)
	138	{
	139	void *retval;
	140
	141	retval = static_calloc_aligned(nmemb, size, 1);
	142	return retval;
	143	}
	144
	145	static
	146	void *static_malloc(size_t size)
	147	{
	148	void *retval;
	149
	150	retval = static_calloc_aligned(1, size, 1);
	151	return retval;
	152	}
	153
	154	static
	155	void static_free(void *ptr)
	156	{
	157	/* no-op. */
	158	}
	159
	160	static
	161	void static_realloc(void ptr, size_t size)
	162	{
	163	size_t *old_size = NULL;
	164	void *retval;
	165
	166	if (size == 0) {
	167	retval = NULL;
	168	goto end;
	169	}
	170
	171	if (ptr) {
	172	old_size = (size_t *) ptr - 1;
	173	if (size <= *old_size) {
	174	/* We can re-use the old entry. */
	175	*old_size = size;
	176	retval = ptr;
	177	goto end;
	178	}
	179	}
	180	/* We need to expand. Don't free previous memory location. */
	181	retval = static_calloc_aligned(1, size, 1);
	182	assert(retval);
	183	if (ptr)
	184	memcpy(retval, ptr, *old_size);
	185	end:
	186	return retval;
	187	}
	188
	189	static
	190	void *static_memalign(size_t alignment, size_t size)
	191	{
	192	void *retval;
	193
	194	retval = static_calloc_aligned(1, size, alignment);
	195	return retval;
	196	}
	197
	198	static
	199	int static_posix_memalign(void **memptr, size_t alignment, size_t size)
	200	{
	201	void *ptr;
	202
	203	/* Check for power of 2, larger than void . /
	204	if (alignment & (alignment - 1)
	205	\|\| alignment < sizeof(void *)
	206	\|\| alignment == 0) {
	207	goto end;
	208	}
	209	ptr = static_calloc_aligned(1, size, alignment);
	210	*memptr = ptr;
	211	end:
	212	return 0;
	213	}
	214
	215	static
	216	void setup_static_allocator(void)
	217	{
	218	assert(cur_alloc.calloc == NULL);
	219	cur_alloc.calloc = static_calloc;
	220	assert(cur_alloc.malloc == NULL);
	221	cur_alloc.malloc = static_malloc;
	222	assert(cur_alloc.free == NULL);
	223	cur_alloc.free = static_free;
	224	assert(cur_alloc.realloc == NULL);
	225	cur_alloc.realloc = static_realloc;
	226	assert(cur_alloc.memalign == NULL);
	227	cur_alloc.memalign = static_memalign;
	228	assert(cur_alloc.posix_memalign == NULL);
	229	cur_alloc.posix_memalign = static_posix_memalign;
	230	}
	231
	232	static
	233	void lookup_all_symbols(void)
	234	{
	235	struct alloc_functions af;
	236
	237	/*
	238	* Temporarily redirect allocation functions to
	239	* static_calloc_aligned, and free function to static_free
	240	* (no-op), until the dlsym lookup has completed.
	241	*/
	242	setup_static_allocator();
	243
	244	/* Perform the actual lookups */
	245	af.calloc = dlsym(RTLD_NEXT, "calloc");
	246	af.malloc = dlsym(RTLD_NEXT, "malloc");
	247	af.free = dlsym(RTLD_NEXT, "free");
	248	af.realloc = dlsym(RTLD_NEXT, "realloc");
	249	af.memalign = dlsym(RTLD_NEXT, "memalign");
	250	af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");
	251
	252	/* Populate the new allocator functions */
	253	memcpy(&cur_alloc, &af, sizeof(cur_alloc));
	254	}
	255
	256	void *malloc(size_t size)
	257	{
	258	void *retval;
	259
	260	URCU_TLS(malloc_nesting)++;
	261	if (cur_alloc.malloc == NULL) {
	262	lookup_all_symbols();
	263	if (cur_alloc.malloc == NULL) {
	264	fprintf(stderr, "mallocwrap: unable to find malloc\n");
	265	abort();
	266	}
	267	}
	268	retval = cur_alloc.malloc(size);
	269	if (URCU_TLS(malloc_nesting) == 1) {
	270	tracepoint(lttng_ust_libc, malloc,
	271	size, retval, LTTNG_UST_CALLER_IP());
	272	}
	273	URCU_TLS(malloc_nesting)--;
	274	return retval;
	275	}
	276
	277	void free(void *ptr)
	278	{
	279	URCU_TLS(malloc_nesting)++;
	280	/*
	281	* Check whether the memory was allocated with
	282	* static_calloc_align, in which case there is nothing to free.
	283	*/
	284	if (caa_unlikely((char *)ptr >= static_calloc_buf &&
	285	(char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
	286	goto end;
	287	}
	288
	289	if (URCU_TLS(malloc_nesting) == 1) {
	290	tracepoint(lttng_ust_libc, free,
	291	ptr, LTTNG_UST_CALLER_IP());
	292	}
	293
	294	if (cur_alloc.free == NULL) {
	295	lookup_all_symbols();
	296	if (cur_alloc.free == NULL) {
	297	fprintf(stderr, "mallocwrap: unable to find free\n");
	298	abort();
	299	}
	300	}
	301	cur_alloc.free(ptr);
	302	end:
	303	URCU_TLS(malloc_nesting)--;
	304	}
	305
	306	void *calloc(size_t nmemb, size_t size)
	307	{
	308	void *retval;
	309
	310	URCU_TLS(malloc_nesting)++;
	311	if (cur_alloc.calloc == NULL) {
	312	lookup_all_symbols();
	313	if (cur_alloc.calloc == NULL) {
	314	fprintf(stderr, "callocwrap: unable to find calloc\n");
	315	abort();
	316	}
	317	}
	318	retval = cur_alloc.calloc(nmemb, size);
	319	if (URCU_TLS(malloc_nesting) == 1) {
	320	tracepoint(lttng_ust_libc, calloc,
	321	nmemb, size, retval, LTTNG_UST_CALLER_IP());
	322	}
	323	URCU_TLS(malloc_nesting)--;
	324	return retval;
	325	}
	326
	327	void realloc(void ptr, size_t size)
	328	{
	329	void *retval;
	330
	331	URCU_TLS(malloc_nesting)++;
	332	/*
	333	* Check whether the memory was allocated with
	334	* static_calloc_align, in which case there is nothing
	335	* to free, and we need to copy the old data.
	336	*/
	337	if (caa_unlikely((char *)ptr >= static_calloc_buf &&
	338	(char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
	339	size_t *old_size;
	340
	341	old_size = (size_t *) ptr - 1;
	342	if (cur_alloc.calloc == NULL) {
	343	lookup_all_symbols();
	344	if (cur_alloc.calloc == NULL) {
	345	fprintf(stderr, "reallocwrap: unable to find calloc\n");
	346	abort();
	347	}
	348	}
	349	retval = cur_alloc.calloc(1, size);
	350	if (retval) {
	351	memcpy(retval, ptr, *old_size);
	352	}
	353	/*
	354	* Mimick that a NULL pointer has been received, so
	355	* memory allocation analysis based on the trace don't
	356	* get confused by the address from the static
	357	* allocator.
	358	*/
	359	ptr = NULL;
	360	goto end;
	361	}
	362
	363	if (cur_alloc.realloc == NULL) {
	364	lookup_all_symbols();
	365	if (cur_alloc.realloc == NULL) {
	366	fprintf(stderr, "reallocwrap: unable to find realloc\n");
	367	abort();
	368	}
	369	}
	370	retval = cur_alloc.realloc(ptr, size);
	371	end:
	372	if (URCU_TLS(malloc_nesting) == 1) {
	373	tracepoint(lttng_ust_libc, realloc,
	374	ptr, size, retval, LTTNG_UST_CALLER_IP());
	375	}
	376	URCU_TLS(malloc_nesting)--;
	377	return retval;
	378	}
	379
	380	void *memalign(size_t alignment, size_t size)
	381	{
	382	void *retval;
	383
	384	URCU_TLS(malloc_nesting)++;
	385	if (cur_alloc.memalign == NULL) {
	386	lookup_all_symbols();
	387	if (cur_alloc.memalign == NULL) {
	388	fprintf(stderr, "memalignwrap: unable to find memalign\n");
	389	abort();
	390	}
	391	}
	392	retval = cur_alloc.memalign(alignment, size);
	393	if (URCU_TLS(malloc_nesting) == 1) {
	394	tracepoint(lttng_ust_libc, memalign,
	395	alignment, size, retval,
	396	LTTNG_UST_CALLER_IP());
	397	}
	398	URCU_TLS(malloc_nesting)--;
	399	return retval;
	400	}
	401
	402	int posix_memalign(void **memptr, size_t alignment, size_t size)
	403	{
	404	int retval;
	405
	406	URCU_TLS(malloc_nesting)++;
	407	if (cur_alloc.posix_memalign == NULL) {
	408	lookup_all_symbols();
	409	if (cur_alloc.posix_memalign == NULL) {
	410	fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
	411	abort();
	412	}
	413	}
	414	retval = cur_alloc.posix_memalign(memptr, alignment, size);
	415	if (URCU_TLS(malloc_nesting) == 1) {
	416	tracepoint(lttng_ust_libc, posix_memalign,
	417	*memptr, alignment, size,
	418	retval, LTTNG_UST_CALLER_IP());
	419	}
	420	URCU_TLS(malloc_nesting)--;
	421	return retval;
	422	}
	423
	424	static
	425	void lttng_ust_fixup_malloc_nesting_tls(void)
	426	{
	427	asm volatile ("" : : "m" (URCU_TLS(malloc_nesting)));
	428	}
	429
	430	__attribute__((constructor))
	431	void lttng_ust_malloc_wrapper_init(void)
	432	{
	433	/* Initialization already done */
	434	if (cur_alloc.calloc) {
	435	return;
	436	}
	437	lttng_ust_fixup_malloc_nesting_tls();
	438	/*
	439	* Ensure the allocator is in place before the process becomes
	440	* multithreaded.
	441	*/
	442	lookup_all_symbols();
	443	}