[lttng-modules.git] / lttng-context-perf-counters.c

/*
 * lttng-context-perf-counters.c
 *
 * LTTng performance monitoring counters (perf-counters) integration module.
 *
 * Copyright (C) 2009-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; only
 * version 2.1 of the License.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/perf_event.h>
#include <linux/list.h>
#include <linux/string.h>
#include "lttng-events.h"
#include "wrapper/ringbuffer/frontend_types.h"
#include "wrapper/vmalloc.h"
#include "wrapper/perf.h"
#include "lttng-tracer.h"

static
size_t perf_counter_get_size(size_t offset)
{
	size_t size = 0;

	size += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
	size += sizeof(uint64_t);
	return size;
}

static
void perf_counter_record(struct lttng_ctx_field *field,
			 struct lib_ring_buffer_ctx *ctx,
			 struct lttng_channel *chan)
{
	struct perf_event *event;
	uint64_t value;

	event = field->u.perf_counter->e[ctx->cpu];
	if (likely(event)) {
		if (unlikely(event->state == PERF_EVENT_STATE_ERROR)) {
			value = 0;
		} else {
			event->pmu->read(event);
			value = local64_read(&event->count);
		}
	} else {
		/*
		 * Perf chooses not to be clever and not to support enabling a
		 * perf counter before the cpu is brought up. Therefore, we need
		 * to support having events coming (e.g. scheduler events)
		 * before the counter is setup. Write an arbitrary 0 in this
		 * case.
		 */
		value = 0;
	}
	lib_ring_buffer_align_ctx(ctx, lttng_alignof(value));
	chan->ops->event_write(ctx, &value, sizeof(value));
}

#if defined(CONFIG_PERF_EVENTS) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,99))
static
void overflow_callback(struct perf_event *event,
		       struct perf_sample_data *data,
		       struct pt_regs *regs)
{
}
#else
static
void overflow_callback(struct perf_event *event, int nmi,
		       struct perf_sample_data *data,
		       struct pt_regs *regs)
{
}
#endif

static
void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field)
{
	struct perf_event **events = field->u.perf_counter->e;
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu)
		perf_event_release_kernel(events[cpu]);
	put_online_cpus();
#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu_notifier(&field->u.perf_counter->nb);
#endif
	kfree(field->event_field.name);
	kfree(field->u.perf_counter->attr);
	kfree(events);
	kfree(field->u.perf_counter);
}

#ifdef CONFIG_HOTPLUG_CPU

/**
 *	lttng_perf_counter_hp_callback - CPU hotplug callback
 *	@nb: notifier block
 *	@action: hotplug action to take
 *	@hcpu: CPU number
 *
 *	Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
 *
 * We can setup perf counters when the cpu is online (up prepare seems to be too
 * soon).
 */
static
int __cpuinit lttng_perf_counter_cpu_hp_callback(struct notifier_block *nb,
						 unsigned long action,
						 void *hcpu)
{
	unsigned int cpu = (unsigned long) hcpu;
	struct lttng_perf_counter_field *perf_field =
		container_of(nb, struct lttng_perf_counter_field, nb);
	struct perf_event **events = perf_field->e;
	struct perf_event_attr *attr = perf_field->attr;
	struct perf_event *pevent;

	if (!perf_field->hp_enable)
		return NOTIFY_OK;

	switch (action) {
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		pevent = wrapper_perf_event_create_kernel_counter(attr,
				cpu, NULL, overflow_callback);
		if (!pevent || IS_ERR(pevent))
			return NOTIFY_BAD;
		if (pevent->state == PERF_EVENT_STATE_ERROR) {
			perf_event_release_kernel(pevent);
			return NOTIFY_BAD;
		}
		barrier();	/* Create perf counter before setting event */
		events[cpu] = pevent;
		break;
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
		pevent = events[cpu];
		events[cpu] = NULL;
		barrier();	/* NULLify event before perf counter teardown */
		perf_event_release_kernel(pevent);
		break;
	}
	return NOTIFY_OK;
}

#endif

int lttng_add_perf_counter_to_ctx(uint32_t type,
				  uint64_t config,
				  const char *name,
				  struct lttng_ctx **ctx)
{
	struct lttng_ctx_field *field;
	struct lttng_perf_counter_field *perf_field;
	struct perf_event **events;
	struct perf_event_attr *attr;
	int ret;
	int cpu;
	char *name_alloc;

	events = kzalloc(num_possible_cpus() * sizeof(*events), GFP_KERNEL);
	if (!events)
		return -ENOMEM;

	attr = kzalloc(sizeof(struct perf_event_attr), GFP_KERNEL);
	if (!attr) {
		ret = -ENOMEM;
		goto error_attr;
	}

	attr->type = type;
	attr->config = config;
	attr->size = sizeof(struct perf_event_attr);
	attr->pinned = 1;
	attr->disabled = 0;

	perf_field = kzalloc(sizeof(struct lttng_perf_counter_field), GFP_KERNEL);
	if (!perf_field) {
		ret = -ENOMEM;
		goto error_alloc_perf_field;
	}
	perf_field->e = events;
	perf_field->attr = attr;

	name_alloc = kstrdup(name, GFP_KERNEL);
	if (!name_alloc) {
		ret = -ENOMEM;
		goto name_alloc_error;
	}

	field = lttng_append_context(ctx);
	if (!field) {
		ret = -ENOMEM;
		goto append_context_error;
	}
	if (lttng_find_context(*ctx, name_alloc)) {
		ret = -EEXIST;
		goto find_error;
	}

#ifdef CONFIG_HOTPLUG_CPU
	perf_field->nb.notifier_call =
		lttng_perf_counter_cpu_hp_callback;
	perf_field->nb.priority = 0;
	register_cpu_notifier(&perf_field->nb);
#endif

	get_online_cpus();
	for_each_online_cpu(cpu) {
		events[cpu] = wrapper_perf_event_create_kernel_counter(attr,
					cpu, NULL, overflow_callback);
		if (!events[cpu] || IS_ERR(events[cpu])) {
			ret = -EINVAL;
			goto counter_error;
		}
		if (events[cpu]->state == PERF_EVENT_STATE_ERROR) {
			ret = -EBUSY;
			goto counter_busy;
		}
	}
	put_online_cpus();

	field->destroy = lttng_destroy_perf_counter_field;

	field->event_field.name = name_alloc;
	field->event_field.type.atype = atype_integer;
	field->event_field.type.u.basic.integer.size = sizeof(uint64_t) * CHAR_BIT;
	field->event_field.type.u.basic.integer.alignment = lttng_alignof(uint64_t) * CHAR_BIT;
	field->event_field.type.u.basic.integer.signedness = lttng_is_signed_type(uint64_t);
	field->event_field.type.u.basic.integer.reverse_byte_order = 0;
	field->event_field.type.u.basic.integer.base = 10;
	field->event_field.type.u.basic.integer.encoding = lttng_encode_none;
	field->get_size = perf_counter_get_size;
	field->record = perf_counter_record;
	field->u.perf_counter = perf_field;
	perf_field->hp_enable = 1;

	wrapper_vmalloc_sync_all();
	return 0;

counter_busy:
counter_error:
	for_each_online_cpu(cpu) {
		if (events[cpu] && !IS_ERR(events[cpu]))
			perf_event_release_kernel(events[cpu]);
	}
	put_online_cpus();
#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu_notifier(&perf_field->nb);
#endif
find_error:
	lttng_remove_context_field(ctx, field);
append_context_error:
	kfree(name_alloc);
name_alloc_error:
	kfree(perf_field);
error_alloc_perf_field:
	kfree(attr);
error_attr:
	kfree(events);
	return ret;
}

MODULE_LICENSE("GPL and additional rights");
MODULE_AUTHOR("Mathieu Desnoyers");
MODULE_DESCRIPTION("Linux Trace Toolkit Perf Support");
Commit	Line	Data
833ad6a0	1	/*
886d51a3	2	* lttng-context-perf-counters.c
833ad6a0 MD	3	*
	4	* LTTng performance monitoring counters (perf-counters) integration module.
	5	*
886d51a3 MD	6	* Copyright (C) 2009-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	7	*
	8	* This library is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU Lesser General Public
	10	* License as published by the Free Software Foundation; only
	11	* version 2.1 of the License.
	12	*
	13	* This library is distributed in the hope that it will be useful,
	14	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	* Lesser General Public License for more details.
	17	*
	18	* You should have received a copy of the GNU Lesser General Public
	19	* License along with this library; if not, write to the Free Software
	20	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
833ad6a0 MD	21	*/
	22
	23	#include <linux/module.h>
	24	#include <linux/slab.h>
	25	#include <linux/perf_event.h>
	26	#include <linux/list.h>
c24a0d71	27	#include <linux/string.h>
a90917c3	28	#include "lttng-events.h"
c24a0d71 MD	29	#include "wrapper/ringbuffer/frontend_types.h"
c24a0d71 MD	30	#include "wrapper/vmalloc.h"
90f5546c	31	#include "wrapper/perf.h"
a90917c3	32	#include "lttng-tracer.h"
833ad6a0	33
f1676205 MD	34	static
	35	size_t perf_counter_get_size(size_t offset)
	36	{
	37	size_t size = 0;
	38
a90917c3	39	size += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
f1676205 MD	40	size += sizeof(uint64_t);
	41	return size;
	42	}
	43
833ad6a0 MD	44	static
	45	void perf_counter_record(struct lttng_ctx_field *field,
	46	struct lib_ring_buffer_ctx *ctx,
a90917c3	47	struct lttng_channel *chan)
833ad6a0 MD	48	{
	49	struct perf_event *event;
	50	uint64_t value;
	51
2001023e	52	event = field->u.perf_counter->e[ctx->cpu];
0478c519	53	if (likely(event)) {
7b745a96 MD	54	if (unlikely(event->state == PERF_EVENT_STATE_ERROR)) {
	55	value = 0;
	56	} else {
	57	event->pmu->read(event);
	58	value = local64_read(&event->count);
	59	}
f91fd73b MD	60	} else {
	61	/*
	62	* Perf chooses not to be clever and not to support enabling a
	63	* perf counter before the cpu is brought up. Therefore, we need
	64	* to support having events coming (e.g. scheduler events)
	65	* before the counter is setup. Write an arbitrary 0 in this
	66	* case.
	67	*/
	68	value = 0;
	69	}
a90917c3	70	lib_ring_buffer_align_ctx(ctx, lttng_alignof(value));
833ad6a0 MD	71	chan->ops->event_write(ctx, &value, sizeof(value));
	72	}
	73
90f5546c MD	74	#if defined(CONFIG_PERF_EVENTS) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,99))
	75	static
	76	void overflow_callback(struct perf_event *event,
	77	struct perf_sample_data *data,
	78	struct pt_regs *regs)
	79	{
	80	}
	81	#else
833ad6a0 MD	82	static
	83	void overflow_callback(struct perf_event *event, int nmi,
	84	struct perf_sample_data *data,
	85	struct pt_regs *regs)
	86	{
	87	}
90f5546c	88	#endif
833ad6a0	89
2dccf128 MD	90	static
	91	void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field)
	92	{
2001023e	93	struct perf_event **events = field->u.perf_counter->e;
2dccf128 MD	94	int cpu;
2dccf128 MD	95
8289661d	96	get_online_cpus();
2dccf128 MD	97	for_each_online_cpu(cpu)
2dccf128 MD	98	perf_event_release_kernel(events[cpu]);
8289661d MD	99	put_online_cpus();
8289661d MD	100	#ifdef CONFIG_HOTPLUG_CPU
2001023e	101	unregister_cpu_notifier(&field->u.perf_counter->nb);
8289661d	102	#endif
c24a0d71	103	kfree(field->event_field.name);
2001023e	104	kfree(field->u.perf_counter->attr);
2dccf128	105	kfree(events);
2001023e	106	kfree(field->u.perf_counter);
2dccf128 MD	107	}
2dccf128 MD	108
8289661d MD	109	#ifdef CONFIG_HOTPLUG_CPU
	110
	111	/**
	112	* lttng_perf_counter_hp_callback - CPU hotplug callback
	113	* @nb: notifier block
	114	* @action: hotplug action to take
	115	* @hcpu: CPU number
	116	*
	117	* Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
	118	*
	119	* We can setup perf counters when the cpu is online (up prepare seems to be too
	120	* soon).
	121	*/
	122	static
	123	int __cpuinit lttng_perf_counter_cpu_hp_callback(struct notifier_block *nb,
	124	unsigned long action,
	125	void *hcpu)
	126	{
	127	unsigned int cpu = (unsigned long) hcpu;
2001023e MD	128	struct lttng_perf_counter_field *perf_field =
	129	container_of(nb, struct lttng_perf_counter_field, nb);
	130	struct perf_event **events = perf_field->e;
	131	struct perf_event_attr *attr = perf_field->attr;
f91fd73b	132	struct perf_event *pevent;
8289661d	133
2001023e	134	if (!perf_field->hp_enable)
8289661d MD	135	return NOTIFY_OK;
	136
	137	switch (action) {
	138	case CPU_ONLINE:
	139	case CPU_ONLINE_FROZEN:
90f5546c	140	pevent = wrapper_perf_event_create_kernel_counter(attr,
8289661d	141	cpu, NULL, overflow_callback);
0478c519	142	if (!pevent \|\| IS_ERR(pevent))
8289661d	143	return NOTIFY_BAD;
7b745a96 MD	144	if (pevent->state == PERF_EVENT_STATE_ERROR) {
	145	perf_event_release_kernel(pevent);
	146	return NOTIFY_BAD;
	147	}
f91fd73b MD	148	barrier(); /* Create perf counter before setting event */
f91fd73b MD	149	events[cpu] = pevent;
8289661d MD	150	break;
	151	case CPU_UP_CANCELED:
	152	case CPU_UP_CANCELED_FROZEN:
	153	case CPU_DEAD:
	154	case CPU_DEAD_FROZEN:
f91fd73b MD	155	pevent = events[cpu];
	156	events[cpu] = NULL;
	157	barrier(); /* NULLify event before perf counter teardown */
	158	perf_event_release_kernel(pevent);
8289661d MD	159	break;
	160	}
	161	return NOTIFY_OK;
	162	}
	163
	164	#endif
	165
833ad6a0 MD	166	int lttng_add_perf_counter_to_ctx(uint32_t type,
833ad6a0 MD	167	uint64_t config,
c24a0d71	168	const char *name,
2dccf128	169	struct lttng_ctx **ctx)
833ad6a0 MD	170	{
833ad6a0 MD	171	struct lttng_ctx_field *field;
2001023e	172	struct lttng_perf_counter_field *perf_field;
833ad6a0 MD	173	struct perf_event **events;
	174	struct perf_event_attr *attr;
	175	int ret;
	176	int cpu;
c24a0d71	177	char *name_alloc;
833ad6a0 MD	178
	179	events = kzalloc(num_possible_cpus() * sizeof(*events), GFP_KERNEL);
	180	if (!events)
	181	return -ENOMEM;
	182
2001023e	183	attr = kzalloc(sizeof(struct perf_event_attr), GFP_KERNEL);
833ad6a0 MD	184	if (!attr) {
	185	ret = -ENOMEM;
	186	goto error_attr;
	187	}
	188
	189	attr->type = type;
	190	attr->config = config;
	191	attr->size = sizeof(struct perf_event_attr);
	192	attr->pinned = 1;
	193	attr->disabled = 0;
	194
2001023e MD	195	perf_field = kzalloc(sizeof(struct lttng_perf_counter_field), GFP_KERNEL);
	196	if (!perf_field) {
	197	ret = -ENOMEM;
	198	goto error_alloc_perf_field;
	199	}
	200	perf_field->e = events;
	201	perf_field->attr = attr;
	202
c24a0d71	203	name_alloc = kstrdup(name, GFP_KERNEL);
bef96e48 MD	204	if (!name_alloc) {
bef96e48 MD	205	ret = -ENOMEM;
c24a0d71	206	goto name_alloc_error;
bef96e48	207	}
c24a0d71	208
2dccf128 MD	209	field = lttng_append_context(ctx);
	210	if (!field) {
	211	ret = -ENOMEM;
8289661d	212	goto append_context_error;
833ad6a0	213	}
2001023e	214	if (lttng_find_context(*ctx, name_alloc)) {
44252f0f MD	215	ret = -EEXIST;
	216	goto find_error;
	217	}
8289661d MD	218
8289661d MD	219	#ifdef CONFIG_HOTPLUG_CPU
2001023e	220	perf_field->nb.notifier_call =
8289661d	221	lttng_perf_counter_cpu_hp_callback;
2001023e MD	222	perf_field->nb.priority = 0;
2001023e MD	223	register_cpu_notifier(&perf_field->nb);
8289661d MD	224	#endif
	225
	226	get_online_cpus();
	227	for_each_online_cpu(cpu) {
90f5546c	228	events[cpu] = wrapper_perf_event_create_kernel_counter(attr,
8289661d	229	cpu, NULL, overflow_callback);
0478c519	230	if (!events[cpu] \|\| IS_ERR(events[cpu])) {
8289661d MD	231	ret = -EINVAL;
	232	goto counter_error;
	233	}
7b745a96 MD	234	if (events[cpu]->state == PERF_EVENT_STATE_ERROR) {
	235	ret = -EBUSY;
	236	goto counter_busy;
	237	}
8289661d MD	238	}
	239	put_online_cpus();
	240
2dccf128	241	field->destroy = lttng_destroy_perf_counter_field;
833ad6a0	242
c24a0d71	243	field->event_field.name = name_alloc;
8070f5c0	244	field->event_field.type.atype = atype_integer;
9d7d747f	245	field->event_field.type.u.basic.integer.size = sizeof(uint64_t) * CHAR_BIT;
a90917c3	246	field->event_field.type.u.basic.integer.alignment = lttng_alignof(uint64_t) * CHAR_BIT;
06254b0f	247	field->event_field.type.u.basic.integer.signedness = lttng_is_signed_type(uint64_t);
8070f5c0 MD	248	field->event_field.type.u.basic.integer.reverse_byte_order = 0;
	249	field->event_field.type.u.basic.integer.base = 10;
	250	field->event_field.type.u.basic.integer.encoding = lttng_encode_none;
f1676205 MD	251	field->get_size = perf_counter_get_size;
f1676205 MD	252	field->record = perf_counter_record;
2001023e MD	253	field->u.perf_counter = perf_field;
2001023e MD	254	perf_field->hp_enable = 1;
833ad6a0 MD	255
	256	wrapper_vmalloc_sync_all();
	257	return 0;
	258
7b745a96	259	counter_busy:
8289661d	260	counter_error:
833ad6a0	261	for_each_online_cpu(cpu) {
0478c519	262	if (events[cpu] && !IS_ERR(events[cpu]))
833ad6a0 MD	263	perf_event_release_kernel(events[cpu]);
833ad6a0 MD	264	}
8289661d MD	265	put_online_cpus();
8289661d MD	266	#ifdef CONFIG_HOTPLUG_CPU
2001023e	267	unregister_cpu_notifier(&perf_field->nb);
8289661d	268	#endif
44252f0f	269	find_error:
8289661d MD	270	lttng_remove_context_field(ctx, field);
	271	append_context_error:
	272	kfree(name_alloc);
	273	name_alloc_error:
2001023e MD	274	kfree(perf_field);
2001023e MD	275	error_alloc_perf_field:
833ad6a0 MD	276	kfree(attr);
	277	error_attr:
	278	kfree(events);
	279	return ret;
	280	}
	281
833ad6a0 MD	282	MODULE_LICENSE("GPL and additional rights");
	283	MODULE_AUTHOR("Mathieu Desnoyers");
	284	MODULE_DESCRIPTION("Linux Trace Toolkit Perf Support");