[lttng-ust.git] / liblttng-ust / lttng-context-perf-counters.c

/*
 * lttng-context-perf-counters.c
 *
 * LTTng UST performance monitoring counters (perf-counters) integration.
 *
 * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; only
 * version 2.1 of the License.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <sys/types.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdbool.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <lttng/ust-events.h>
#include <lttng/ust-tracer.h>
#include <lttng/ringbuffer-config.h>
#include <urcu/system.h>
#include <urcu/arch.h>
#include <urcu/rculist.h>
#include <helper.h>
#include <urcu/ref.h>
#include <usterr-signal-safe.h>
#include <signal.h>
#include "perf_event.h"
#include "lttng-tracer-core.h"

/*
 * We use a global perf counter key and iterate on per-thread RCU lists
 * of fields in the fast path, even though this is not strictly speaking
 * what would provide the best fast-path complexity, to ensure teardown
 * of sessions vs thread exit is handled racelessly.
 *
 * Updates and traversals of thread_list are protected by UST lock.
 * Updates to rcu_field_list are protected by UST lock.
 */

struct lttng_perf_counter_thread_field {
	struct lttng_perf_counter_field *field;	/* Back reference */
	struct perf_event_mmap_page *pc;
	struct cds_list_head thread_field_node;	/* Per-field list of thread fields (node) */
	struct cds_list_head rcu_field_node;	/* RCU per-thread list of fields (node) */
	int fd;					/* Perf FD */
};

struct lttng_perf_counter_thread {
	struct cds_list_head rcu_field_list;	/* RCU per-thread list of fields */
};

struct lttng_perf_counter_field {
	struct perf_event_attr attr;
	struct cds_list_head thread_field_list;	/* Per-field list of thread fields */
};

static pthread_key_t perf_counter_key;

static
size_t perf_counter_get_size(struct lttng_ctx_field *field, size_t offset)
{
	size_t size = 0;

	size += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
	size += sizeof(uint64_t);
	return size;
}

static
uint64_t read_perf_counter_syscall(
		struct lttng_perf_counter_thread_field *thread_field)
{
	uint64_t count;

	if (caa_unlikely(thread_field->fd < 0))
		return 0;

	if (caa_unlikely(read(thread_field->fd, &count, sizeof(count))
				< sizeof(count)))
		return 0;

	return count;
}

#if defined(__x86_64__) || defined(__i386__)

static
uint64_t rdpmc(unsigned int counter)
{
	unsigned int low, high;

	asm volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (counter));

	return low | ((uint64_t) high) << 32;
}

static
bool has_rdpmc(struct perf_event_mmap_page *pc)
{
	if (caa_unlikely(!pc->cap_bit0_is_deprecated))
		return false;
	/* Since Linux kernel 3.12. */
	return pc->cap_user_rdpmc;
}

static
uint64_t arch_read_perf_counter(
		struct lttng_perf_counter_thread_field *thread_field)
{
	uint32_t seq, idx;
	uint64_t count;
	struct perf_event_mmap_page *pc = thread_field->pc;

	if (caa_unlikely(!pc))
		return 0;

	do {
		seq = CMM_LOAD_SHARED(pc->lock);
		cmm_barrier();

		idx = pc->index;
		if (caa_likely(has_rdpmc(pc) && idx)) {
			int64_t pmcval;

			pmcval = rdpmc(idx - 1);
			/* Sign-extend the pmc register result. */
			pmcval <<= 64 - pc->pmc_width;
			pmcval >>= 64 - pc->pmc_width;
			count = pc->offset + pmcval;
		} else {
			/* Fall-back on system call if rdpmc cannot be used. */
			return read_perf_counter_syscall(thread_field);
		}
		cmm_barrier();
	} while (CMM_LOAD_SHARED(pc->lock) != seq);

	return count;
}

static
int arch_perf_keep_fd(struct lttng_perf_counter_thread_field *thread_field)
{
	struct perf_event_mmap_page *pc = thread_field->pc;

	if (!pc)
		return 0;
	return !has_rdpmc(pc);
}

#endif

static
int sys_perf_event_open(struct perf_event_attr *attr,
		pid_t pid, int cpu, int group_fd,
		unsigned long flags)
{
	return syscall(SYS_perf_event_open, attr, pid, cpu,
			group_fd, flags);
}

static
int open_perf_fd(struct perf_event_attr *attr)
{
	int fd;

	fd = sys_perf_event_open(attr, 0, -1, -1, 0);
	if (fd < 0)
		return -1;

	return fd;
}

static
void close_perf_fd(int fd)
{
	int ret;

	if (fd < 0)
		return;

	ret = close(fd);
	if (ret) {
		perror("Error closing LTTng-UST perf memory mapping FD");
	}
}

static void setup_perf(struct lttng_perf_counter_thread_field *thread_field)
{
	void *perf_addr;

	perf_addr = mmap(NULL, sizeof(struct perf_event_mmap_page),
			PROT_READ, MAP_SHARED, thread_field->fd, 0);
	if (perf_addr == MAP_FAILED)
		perf_addr = NULL;
	thread_field->pc = perf_addr;

	if (!arch_perf_keep_fd(thread_field)) {
		close_perf_fd(thread_field->fd);
		thread_field->fd = -1;
	}
}

static
void unmap_perf_page(struct perf_event_mmap_page *pc)
{
	int ret;

	if (!pc)
		return;
	ret = munmap(pc, sizeof(struct perf_event_mmap_page));
	if (ret < 0) {
		PERROR("Error in munmap");
		abort();
	}
}

static
struct lttng_perf_counter_thread *alloc_perf_counter_thread(void)
{
	struct lttng_perf_counter_thread *perf_thread;
	sigset_t newmask, oldmask;
	int ret;

	ret = sigfillset(&newmask);
	if (ret)
		abort();
	ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
	if (ret)
		abort();
	/* Check again with signals disabled */
	perf_thread = pthread_getspecific(perf_counter_key);
	if (perf_thread)
		goto skip;
	perf_thread = zmalloc(sizeof(*perf_thread));
	if (!perf_thread)
		abort();
	CDS_INIT_LIST_HEAD(&perf_thread->rcu_field_list);
	ret = pthread_setspecific(perf_counter_key, perf_thread);
	if (ret)
		abort();
skip:
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	if (ret)
		abort();
	return perf_thread;
}

static
struct lttng_perf_counter_thread_field *
	add_thread_field(struct lttng_perf_counter_field *perf_field,
		struct lttng_perf_counter_thread *perf_thread)
{
	struct lttng_perf_counter_thread_field *thread_field;
	sigset_t newmask, oldmask;
	int ret;

	ret = sigfillset(&newmask);
	if (ret)
		abort();
	ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
	if (ret)
		abort();
	/* Check again with signals disabled */
	cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
			rcu_field_node) {
		if (thread_field->field == perf_field)
			goto skip;
	}
	thread_field = zmalloc(sizeof(*thread_field));
	if (!thread_field)
		abort();
	thread_field->field = perf_field;
	thread_field->fd = open_perf_fd(&perf_field->attr);
	if (thread_field->fd >= 0)
		setup_perf(thread_field);
	/*
	 * Note: thread_field->pc can be NULL if setup_perf() fails.
	 * Also, thread_field->fd can be -1 if open_perf_fd() fails.
	 */
	ust_lock_nocheck();
	cds_list_add_rcu(&thread_field->rcu_field_node,
			&perf_thread->rcu_field_list);
	cds_list_add(&thread_field->thread_field_node,
			&perf_field->thread_field_list);
	ust_unlock();
skip:
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	if (ret)
		abort();
	return thread_field;
}

static
struct lttng_perf_counter_thread_field *
		get_thread_field(struct lttng_perf_counter_field *field)
{
	struct lttng_perf_counter_thread *perf_thread;
	struct lttng_perf_counter_thread_field *thread_field;

	perf_thread = pthread_getspecific(perf_counter_key);
	if (!perf_thread)
		perf_thread = alloc_perf_counter_thread();
	cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
			rcu_field_node) {
		if (thread_field->field == field)
			return thread_field;
	}
	/* perf_counter_thread_field not found, need to add one */
	return add_thread_field(field, perf_thread);
}

static
uint64_t wrapper_perf_counter_read(struct lttng_ctx_field *field)
{
	struct lttng_perf_counter_field *perf_field;
	struct lttng_perf_counter_thread_field *perf_thread_field;

	perf_field = field->u.perf_counter;
	perf_thread_field = get_thread_field(perf_field);
	return arch_read_perf_counter(perf_thread_field);
}

static
void perf_counter_record(struct lttng_ctx_field *field,
		 struct lttng_ust_lib_ring_buffer_ctx *ctx,
		 struct lttng_channel *chan)
{
	uint64_t value;

	value = wrapper_perf_counter_read(field);
	lib_ring_buffer_align_ctx(ctx, lttng_alignof(value));
	chan->ops->event_write(ctx, &value, sizeof(value));
}

static
void perf_counter_get_value(struct lttng_ctx_field *field,
		struct lttng_ctx_value *value)
{
	uint64_t v;

	v = wrapper_perf_counter_read(field);
	value->u.s64 = v;
}

/* Called with UST lock held */
static
void lttng_destroy_perf_thread_field(
		struct lttng_perf_counter_thread_field *thread_field)
{
	close_perf_fd(thread_field->fd);
	unmap_perf_page(thread_field->pc);
	cds_list_del_rcu(&thread_field->rcu_field_node);
	cds_list_del(&thread_field->thread_field_node);
	free(thread_field);
}

static
void lttng_destroy_perf_thread_key(void *_key)
{
	struct lttng_perf_counter_thread *perf_thread = _key;
	struct lttng_perf_counter_thread_field *pos, *p;

	ust_lock_nocheck();
	cds_list_for_each_entry_safe(pos, p, &perf_thread->rcu_field_list,
			rcu_field_node)
		lttng_destroy_perf_thread_field(pos);
	ust_unlock();
	free(perf_thread);
}

/* Called with UST lock held */
static
void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field)
{
	struct lttng_perf_counter_field *perf_field;
	struct lttng_perf_counter_thread_field *pos, *p;

	free((char *) field->event_field.name);
	perf_field = field->u.perf_counter;
	/*
	 * This put is performed when no threads can concurrently
	 * perform a "get" concurrently, thanks to urcu-bp grace
	 * period.
	 */
	cds_list_for_each_entry_safe(pos, p, &perf_field->thread_field_list,
			thread_field_node)
		lttng_destroy_perf_thread_field(pos);
	free(perf_field);
}

/* Called with UST lock held */
int lttng_add_perf_counter_to_ctx(uint32_t type,
				uint64_t config,
				const char *name,
				struct lttng_ctx **ctx)
{
	struct lttng_ctx_field *field;
	struct lttng_perf_counter_field *perf_field;
	char *name_alloc;
	int ret;

	name_alloc = strdup(name);
	if (!name_alloc) {
		ret = -ENOMEM;
		goto name_alloc_error;
	}
	perf_field = zmalloc(sizeof(*perf_field));
	if (!perf_field) {
		ret = -ENOMEM;
		goto perf_field_alloc_error;
	}
	field = lttng_append_context(ctx);
	if (!field) {
		ret = -ENOMEM;
		goto append_context_error;
	}
	if (lttng_find_context(*ctx, name_alloc)) {
		ret = -EEXIST;
		goto find_error;
	}

	field->destroy = lttng_destroy_perf_counter_field;

	field->event_field.name = name_alloc;
	field->event_field.type.atype = atype_integer;
	field->event_field.type.u.basic.integer.size =
			sizeof(uint64_t) * CHAR_BIT;
	field->event_field.type.u.basic.integer.alignment =
			lttng_alignof(uint64_t) * CHAR_BIT;
	field->event_field.type.u.basic.integer.signedness =
			lttng_is_signed_type(uint64_t);
	field->event_field.type.u.basic.integer.reverse_byte_order = 0;
	field->event_field.type.u.basic.integer.base = 10;
	field->event_field.type.u.basic.integer.encoding = lttng_encode_none;
	field->get_size = perf_counter_get_size;
	field->record = perf_counter_record;
	field->get_value = perf_counter_get_value;

	perf_field->attr.type = type;
	perf_field->attr.config = config;
	perf_field->attr.exclude_kernel = 1;
	CDS_INIT_LIST_HEAD(&perf_field->thread_field_list);
	field->u.perf_counter = perf_field;

	/* Ensure that this perf counter can be used in this process. */
	ret = open_perf_fd(&perf_field->attr);
	if (ret < 0) {
		ret = -ENODEV;
		goto setup_error;
	}
	close_perf_fd(ret);

	/*
	 * Contexts can only be added before tracing is started, so we
	 * don't have to synchronize against concurrent threads using
	 * the field here.
	 */

	lttng_context_update(*ctx);
	return 0;

setup_error:
find_error:
	lttng_remove_context_field(ctx, field);
append_context_error:
	free(perf_field);
perf_field_alloc_error:
	free(name_alloc);
name_alloc_error:
	return ret;
}

int lttng_perf_counter_init(void)
{
	int ret;

	ret = pthread_key_create(&perf_counter_key,
			lttng_destroy_perf_thread_key);
	if (ret)
		ret = -ret;
	return ret;
}

void lttng_perf_counter_exit(void)
{
	int ret;

	ret = pthread_key_delete(perf_counter_key);
	if (ret) {
		errno = ret;
		PERROR("Error in pthread_key_delete");
	}
}
Commit	Line	Data
d58d1454 MD	1	/*
	2	* lttng-context-perf-counters.c
	3	*
	4	* LTTng UST performance monitoring counters (perf-counters) integration.
	5	*
	6	* Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	7	*
	8	* This library is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU Lesser General Public
	10	* License as published by the Free Software Foundation; only
	11	* version 2.1 of the License.
	12	*
	13	* This library is distributed in the hope that it will be useful,
	14	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	* Lesser General Public License for more details.
	17	*
	18	* You should have received a copy of the GNU Lesser General Public
	19	* License along with this library; if not, write to the Free Software
	20	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	21	*/
	22
	23	#include <sys/types.h>
	24	#include <unistd.h>
	25	#include <string.h>
	26	#include <stdlib.h>
	27	#include <stdio.h>
e336bbe5	28	#include <stdbool.h>
d58d1454 MD	29	#include <sys/mman.h>
d58d1454 MD	30	#include <sys/syscall.h>
d58d1454 MD	31	#include <lttng/ust-events.h>
	32	#include <lttng/ust-tracer.h>
	33	#include <lttng/ringbuffer-config.h>
	34	#include <urcu/system.h>
	35	#include <urcu/arch.h>
	36	#include <urcu/rculist.h>
	37	#include <helper.h>
	38	#include <urcu/ref.h>
	39	#include <usterr-signal-safe.h>
	40	#include <signal.h>
e336bbe5	41	#include "perf_event.h"
d58d1454 MD	42	#include "lttng-tracer-core.h"
	43
	44	/*
	45	* We use a global perf counter key and iterate on per-thread RCU lists
	46	* of fields in the fast path, even though this is not strictly speaking
	47	* what would provide the best fast-path complexity, to ensure teardown
	48	* of sessions vs thread exit is handled racelessly.
	49	*
	50	* Updates and traversals of thread_list are protected by UST lock.
	51	* Updates to rcu_field_list are protected by UST lock.
	52	*/
	53
	54	struct lttng_perf_counter_thread_field {
	55	struct lttng_perf_counter_field field; / Back reference */
	56	struct perf_event_mmap_page *pc;
	57	struct cds_list_head thread_field_node; /* Per-field list of thread fields (node) */
	58	struct cds_list_head rcu_field_node; /* RCU per-thread list of fields (node) */
e336bbe5	59	int fd; /* Perf FD */
d58d1454 MD	60	};
	61
	62	struct lttng_perf_counter_thread {
	63	struct cds_list_head rcu_field_list; /* RCU per-thread list of fields */
	64	};
	65
	66	struct lttng_perf_counter_field {
	67	struct perf_event_attr attr;
	68	struct cds_list_head thread_field_list; /* Per-field list of thread fields */
	69	};
	70
	71	static pthread_key_t perf_counter_key;
	72
	73	static
53569322	74	size_t perf_counter_get_size(struct lttng_ctx_field *field, size_t offset)
d58d1454 MD	75	{
	76	size_t size = 0;
	77
	78	size += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
	79	size += sizeof(uint64_t);
	80	return size;
	81	}
	82
e336bbe5 MD	83	static
	84	uint64_t read_perf_counter_syscall(
	85	struct lttng_perf_counter_thread_field *thread_field)
	86	{
	87	uint64_t count;
	88
	89	if (caa_unlikely(thread_field->fd < 0))
	90	return 0;
	91
	92	if (caa_unlikely(read(thread_field->fd, &count, sizeof(count))
	93	< sizeof(count)))
	94	return 0;
	95
	96	return count;
	97	}
	98
d58d1454 MD	99	#if defined(__x86_64__) \|\| defined(__i386__)
	100
	101	static
	102	uint64_t rdpmc(unsigned int counter)
	103	{
	104	unsigned int low, high;
	105
	106	asm volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (counter));
	107
	108	return low \| ((uint64_t) high) << 32;
	109	}
	110
e336bbe5 MD	111	static
	112	bool has_rdpmc(struct perf_event_mmap_page *pc)
	113	{
	114	if (caa_unlikely(!pc->cap_bit0_is_deprecated))
	115	return false;
	116	/* Since Linux kernel 3.12. */
	117	return pc->cap_user_rdpmc;
	118	}
d58d1454 MD	119
d58d1454 MD	120	static
e336bbe5 MD	121	uint64_t arch_read_perf_counter(
e336bbe5 MD	122	struct lttng_perf_counter_thread_field *thread_field)
d58d1454 MD	123	{
	124	uint32_t seq, idx;
	125	uint64_t count;
e336bbe5	126	struct perf_event_mmap_page *pc = thread_field->pc;
d58d1454 MD	127
	128	if (caa_unlikely(!pc))
	129	return 0;
	130
	131	do {
	132	seq = CMM_LOAD_SHARED(pc->lock);
	133	cmm_barrier();
	134
	135	idx = pc->index;
e336bbe5	136	if (caa_likely(has_rdpmc(pc) && idx)) {
3046b99e MD	137	int64_t pmcval;
	138
	139	pmcval = rdpmc(idx - 1);
	140	/* Sign-extend the pmc register result. */
	141	pmcval <<= 64 - pc->pmc_width;
	142	pmcval >>= 64 - pc->pmc_width;
	143	count = pc->offset + pmcval;
	144	} else {
e336bbe5 MD	145	/* Fall-back on system call if rdpmc cannot be used. */
e336bbe5 MD	146	return read_perf_counter_syscall(thread_field);
3046b99e	147	}
d58d1454 MD	148	cmm_barrier();
	149	} while (CMM_LOAD_SHARED(pc->lock) != seq);
	150
	151	return count;
	152	}
	153
e336bbe5 MD	154	static
	155	int arch_perf_keep_fd(struct lttng_perf_counter_thread_field *thread_field)
	156	{
	157	struct perf_event_mmap_page *pc = thread_field->pc;
	158
	159	if (!pc)
	160	return 0;
	161	return !has_rdpmc(pc);
	162	}
	163
	164	#endif
	165
d58d1454 MD	166	static
	167	int sys_perf_event_open(struct perf_event_attr *attr,
	168	pid_t pid, int cpu, int group_fd,
	169	unsigned long flags)
	170	{
	171	return syscall(SYS_perf_event_open, attr, pid, cpu,
	172	group_fd, flags);
	173	}
	174
	175	static
e336bbe5	176	int open_perf_fd(struct perf_event_attr *attr)
d58d1454	177	{
e336bbe5	178	int fd;
d58d1454 MD	179
	180	fd = sys_perf_event_open(attr, 0, -1, -1, 0);
	181	if (fd < 0)
e336bbe5 MD	182	return -1;
	183
	184	return fd;
	185	}
	186
	187	static
	188	void close_perf_fd(int fd)
	189	{
	190	int ret;
	191
	192	if (fd < 0)
	193	return;
d58d1454	194
6c2125af MD	195	ret = close(fd);
	196	if (ret) {
	197	perror("Error closing LTTng-UST perf memory mapping FD");
	198	}
e336bbe5 MD	199	}
	200
	201	static void setup_perf(struct lttng_perf_counter_thread_field *thread_field)
	202	{
	203	void *perf_addr;
	204
	205	perf_addr = mmap(NULL, sizeof(struct perf_event_mmap_page),
	206	PROT_READ, MAP_SHARED, thread_field->fd, 0);
	207	if (perf_addr == MAP_FAILED)
	208	perf_addr = NULL;
	209	thread_field->pc = perf_addr;
	210
	211	if (!arch_perf_keep_fd(thread_field)) {
	212	close_perf_fd(thread_field->fd);
	213	thread_field->fd = -1;
	214	}
d58d1454 MD	215	}
	216
	217	static
	218	void unmap_perf_page(struct perf_event_mmap_page *pc)
	219	{
	220	int ret;
	221
	222	if (!pc)
	223	return;
	224	ret = munmap(pc, sizeof(struct perf_event_mmap_page));
	225	if (ret < 0) {
	226	PERROR("Error in munmap");
	227	abort();
	228	}
	229	}
	230
	231	static
	232	struct lttng_perf_counter_thread *alloc_perf_counter_thread(void)
	233	{
	234	struct lttng_perf_counter_thread *perf_thread;
	235	sigset_t newmask, oldmask;
	236	int ret;
	237
	238	ret = sigfillset(&newmask);
	239	if (ret)
	240	abort();
	241	ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
	242	if (ret)
	243	abort();
	244	/* Check again with signals disabled */
	245	perf_thread = pthread_getspecific(perf_counter_key);
	246	if (perf_thread)
	247	goto skip;
	248	perf_thread = zmalloc(sizeof(*perf_thread));
	249	if (!perf_thread)
	250	abort();
	251	CDS_INIT_LIST_HEAD(&perf_thread->rcu_field_list);
	252	ret = pthread_setspecific(perf_counter_key, perf_thread);
	253	if (ret)
	254	abort();
	255	skip:
	256	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	257	if (ret)
	258	abort();
	259	return perf_thread;
	260	}
	261
	262	static
	263	struct lttng_perf_counter_thread_field *
	264	add_thread_field(struct lttng_perf_counter_field *perf_field,
	265	struct lttng_perf_counter_thread *perf_thread)
	266	{
	267	struct lttng_perf_counter_thread_field *thread_field;
	268	sigset_t newmask, oldmask;
	269	int ret;
	270
	271	ret = sigfillset(&newmask);
	272	if (ret)
	273	abort();
	274	ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
	275	if (ret)
	276	abort();
	277	/* Check again with signals disabled */
	278	cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
279	rcu_field_node) {
280	if (thread_field->field == perf_field)
281	goto skip;
282	}
283	thread_field = zmalloc(sizeof(*thread_field));
284	if (!thread_field)
285	abort();
286	thread_field->field = perf_field;
e336bbe5 MD	287	thread_field->fd = open_perf_fd(&perf_field->attr);
	288	if (thread_field->fd >= 0)
	289	setup_perf(thread_field);
	290	/*
	291	* Note: thread_field->pc can be NULL if setup_perf() fails.
	292	* Also, thread_field->fd can be -1 if open_perf_fd() fails.
	293	*/
d58d1454 MD	294	ust_lock_nocheck();
	295	cds_list_add_rcu(&thread_field->rcu_field_node,
	296	&perf_thread->rcu_field_list);
	297	cds_list_add(&thread_field->thread_field_node,
	298	&perf_field->thread_field_list);
	299	ust_unlock();
	300	skip:
	301	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	302	if (ret)
	303	abort();
	304	return thread_field;
	305	}
	306
	307	static
	308	struct lttng_perf_counter_thread_field *
	309	get_thread_field(struct lttng_perf_counter_field *field)
	310	{
	311	struct lttng_perf_counter_thread *perf_thread;
	312	struct lttng_perf_counter_thread_field *thread_field;
	313
	314	perf_thread = pthread_getspecific(perf_counter_key);
	315	if (!perf_thread)
	316	perf_thread = alloc_perf_counter_thread();
	317	cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
	318	rcu_field_node) {
	319	if (thread_field->field == field)
	320	return thread_field;
	321	}
	322	/* perf_counter_thread_field not found, need to add one */
	323	return add_thread_field(field, perf_thread);
	324	}
	325
	326	static
	327	uint64_t wrapper_perf_counter_read(struct lttng_ctx_field *field)
	328	{
	329	struct lttng_perf_counter_field *perf_field;
	330	struct lttng_perf_counter_thread_field *perf_thread_field;
	331
	332	perf_field = field->u.perf_counter;
	333	perf_thread_field = get_thread_field(perf_field);
e336bbe5	334	return arch_read_perf_counter(perf_thread_field);
d58d1454 MD	335	}
	336
	337	static
	338	void perf_counter_record(struct lttng_ctx_field *field,
	339	struct lttng_ust_lib_ring_buffer_ctx *ctx,
	340	struct lttng_channel *chan)
	341	{
	342	uint64_t value;
	343
	344	value = wrapper_perf_counter_read(field);
	345	lib_ring_buffer_align_ctx(ctx, lttng_alignof(value));
	346	chan->ops->event_write(ctx, &value, sizeof(value));
	347	}
	348
	349	static
	350	void perf_counter_get_value(struct lttng_ctx_field *field,
53569322	351	struct lttng_ctx_value *value)
d58d1454 MD	352	{
	353	uint64_t v;
	354
	355	v = wrapper_perf_counter_read(field);
53569322	356	value->u.s64 = v;
d58d1454 MD	357	}
	358
	359	/* Called with UST lock held */
	360	static
	361	void lttng_destroy_perf_thread_field(
	362	struct lttng_perf_counter_thread_field *thread_field)
	363	{
e336bbe5	364	close_perf_fd(thread_field->fd);
d58d1454 MD	365	unmap_perf_page(thread_field->pc);
	366	cds_list_del_rcu(&thread_field->rcu_field_node);
	367	cds_list_del(&thread_field->thread_field_node);
	368	free(thread_field);
	369	}
	370
	371	static
	372	void lttng_destroy_perf_thread_key(void *_key)
	373	{
	374	struct lttng_perf_counter_thread *perf_thread = _key;
	375	struct lttng_perf_counter_thread_field pos, p;
	376
	377	ust_lock_nocheck();
	378	cds_list_for_each_entry_safe(pos, p, &perf_thread->rcu_field_list,
	379	rcu_field_node)
	380	lttng_destroy_perf_thread_field(pos);
	381	ust_unlock();
	382	free(perf_thread);
	383	}
	384
	385	/* Called with UST lock held */
	386	static
	387	void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field)
	388	{
	389	struct lttng_perf_counter_field *perf_field;
	390	struct lttng_perf_counter_thread_field pos, p;
	391
	392	free((char *) field->event_field.name);
	393	perf_field = field->u.perf_counter;
	394	/*
	395	* This put is performed when no threads can concurrently
	396	* perform a "get" concurrently, thanks to urcu-bp grace
	397	* period.
	398	*/
	399	cds_list_for_each_entry_safe(pos, p, &perf_field->thread_field_list,
	400	thread_field_node)
	401	lttng_destroy_perf_thread_field(pos);
	402	free(perf_field);
	403	}
	404
	405	/* Called with UST lock held */
	406	int lttng_add_perf_counter_to_ctx(uint32_t type,
	407	uint64_t config,
	408	const char *name,
	409	struct lttng_ctx **ctx)
	410	{
	411	struct lttng_ctx_field *field;
	412	struct lttng_perf_counter_field *perf_field;
d58d1454 MD	413	char *name_alloc;
	414	int ret;
	415
	416	name_alloc = strdup(name);
	417	if (!name_alloc) {
	418	ret = -ENOMEM;
	419	goto name_alloc_error;
	420	}
	421	perf_field = zmalloc(sizeof(*perf_field));
	422	if (!perf_field) {
	423	ret = -ENOMEM;
	424	goto perf_field_alloc_error;
	425	}
	426	field = lttng_append_context(ctx);
	427	if (!field) {
	428	ret = -ENOMEM;
	429	goto append_context_error;
	430	}
	431	if (lttng_find_context(*ctx, name_alloc)) {
	432	ret = -EEXIST;
	433	goto find_error;
	434	}
	435
	436	field->destroy = lttng_destroy_perf_counter_field;
	437
	438	field->event_field.name = name_alloc;
	439	field->event_field.type.atype = atype_integer;
	440	field->event_field.type.u.basic.integer.size =
	441	sizeof(uint64_t) * CHAR_BIT;
	442	field->event_field.type.u.basic.integer.alignment =
	443	lttng_alignof(uint64_t) * CHAR_BIT;
	444	field->event_field.type.u.basic.integer.signedness =
	445	lttng_is_signed_type(uint64_t);
	446	field->event_field.type.u.basic.integer.reverse_byte_order = 0;
	447	field->event_field.type.u.basic.integer.base = 10;
	448	field->event_field.type.u.basic.integer.encoding = lttng_encode_none;
	449	field->get_size = perf_counter_get_size;
	450	field->record = perf_counter_record;
	451	field->get_value = perf_counter_get_value;
	452
	453	perf_field->attr.type = type;
	454	perf_field->attr.config = config;
	455	perf_field->attr.exclude_kernel = 1;
	456	CDS_INIT_LIST_HEAD(&perf_field->thread_field_list);
	457	field->u.perf_counter = perf_field;
	458
	459	/* Ensure that this perf counter can be used in this process. */
e336bbe5 MD	460	ret = open_perf_fd(&perf_field->attr);
e336bbe5 MD	461	if (ret < 0) {
d58d1454 MD	462	ret = -ENODEV;
	463	goto setup_error;
	464	}
e336bbe5	465	close_perf_fd(ret);
d58d1454 MD	466
	467	/*
	468	* Contexts can only be added before tracing is started, so we
	469	* don't have to synchronize against concurrent threads using
	470	* the field here.
	471	*/
	472
b2cc986a	473	lttng_context_update(*ctx);
d58d1454 MD	474	return 0;
	475
	476	setup_error:
	477	find_error:
	478	lttng_remove_context_field(ctx, field);
	479	append_context_error:
	480	free(perf_field);
	481	perf_field_alloc_error:
	482	free(name_alloc);
	483	name_alloc_error:
	484	return ret;
	485	}
	486
	487	int lttng_perf_counter_init(void)
	488	{
	489	int ret;
	490
	491	ret = pthread_key_create(&perf_counter_key,
	492	lttng_destroy_perf_thread_key);
	493	if (ret)
	494	ret = -ret;
	495	return ret;
	496	}
	497
	498	void lttng_perf_counter_exit(void)
	499	{
	500	int ret;
	501
	502	ret = pthread_key_delete(perf_counter_key);
	503	if (ret) {
	504	errno = ret;
	505	PERROR("Error in pthread_key_delete");
	506	}
	507	}