[lttng-modules.git] / src / lttng-context-callstack-legacy-impl.h

/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
 *
 * lttng-context-callstack-legacy-impl.h
 *
 * LTTng callstack event context, legacy implementation. Targets
 * kernels and architectures not yet using the stacktrace common
 * infrastructure introduced in the upstream Linux kernel by commit
 * 214d8ca6ee "stacktrace: Provide common infrastructure" (merged in
 * Linux 5.2, then gradually introduced within architectures).
 *
 * Copyright (C) 2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 * Copyright (C) 2014 Francis Giraldeau <francis.giraldeau@gmail.com>
 */

#define MAX_ENTRIES 128

enum lttng_cs_ctx_modes {
	CALLSTACK_KERNEL = 0,
	CALLSTACK_USER = 1,
	NR_CALLSTACK_MODES,
};

struct lttng_cs_dispatch {
	struct stack_trace stack_trace;
	unsigned long entries[MAX_ENTRIES];
};

struct lttng_cs {
	struct lttng_cs_dispatch dispatch[RING_BUFFER_MAX_NESTING];
};

struct field_data {
	struct lttng_cs __percpu *cs_percpu;
	enum lttng_cs_ctx_modes mode;
};

struct lttng_cs_type {
	const char *name;
	const char *length_name;
	const char *save_func_name;
	void (*save_func)(struct stack_trace *trace);
};

static struct lttng_cs_type cs_types[] = {
	{
		.name		= "callstack_kernel",
		.length_name	= "_callstack_kernel_length",
		.save_func_name	= "save_stack_trace",
		.save_func	= NULL,
	},
	{
		.name		= "callstack_user",
		.length_name	= "_callstack_user_length",
		.save_func_name	= "save_stack_trace_user",
		.save_func	= NULL,
	},
};

static
int init_type(enum lttng_cs_ctx_modes mode)
{
	unsigned long func;

	if (cs_types[mode].save_func)
		return 0;
	func = kallsyms_lookup_funcptr(cs_types[mode].save_func_name);
	if (!func) {
		printk(KERN_WARNING "LTTng: symbol lookup failed: %s\n",
				cs_types[mode].save_func_name);
		return -EINVAL;
	}
	cs_types[mode].save_func = (void *) func;
	return 0;
}

static
void lttng_cs_set_init(struct lttng_cs __percpu *cs_set)
{
	int cpu, i;

	for_each_possible_cpu(cpu) {
		struct lttng_cs *cs;

		cs = per_cpu_ptr(cs_set, cpu);
		for (i = 0; i < RING_BUFFER_MAX_NESTING; i++) {
			struct lttng_cs_dispatch *dispatch;

			dispatch = &cs->dispatch[i];
			dispatch->stack_trace.entries = dispatch->entries;
			dispatch->stack_trace.max_entries = MAX_ENTRIES;
		}
	}
}

/* Keep track of nesting inside userspace callstack context code */
DEFINE_PER_CPU(int, callstack_user_nesting);

/*
 * Note: these callbacks expect to be invoked with preemption disabled across
 * get_size and record due to its use of a per-cpu stack.
 */
static
struct stack_trace *stack_trace_context(struct field_data *fdata, int cpu)
{
	int buffer_nesting, cs_user_nesting;
	struct lttng_cs *cs;

	/*
	 * Do not gather the userspace callstack context when the event was
	 * triggered by the userspace callstack context saving mechanism.
	 */
	cs_user_nesting = per_cpu(callstack_user_nesting, cpu);

	if (fdata->mode == CALLSTACK_USER && cs_user_nesting >= 1)
		return NULL;

	/*
	 * get_cpu() is not required, preemption is already
	 * disabled while event is written.
	 *
	 * max nesting is checked in lib_ring_buffer_get_cpu().
	 * Check it again as a safety net.
	 */
	cs = per_cpu_ptr(fdata->cs_percpu, cpu);
	buffer_nesting = per_cpu(lib_ring_buffer_nesting, cpu) - 1;
	if (buffer_nesting >= RING_BUFFER_MAX_NESTING)
		return NULL;

	return &cs->dispatch[buffer_nesting].stack_trace;
}

static
size_t lttng_callstack_length_get_size(void *priv, struct lttng_kernel_probe_ctx *probe_ctx, size_t offset)
{
	size_t orig_offset = offset;

	offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned int));
	offset += sizeof(unsigned int);
	return offset - orig_offset;
}

/*
 * In order to reserve the correct size, the callstack is computed. The
 * resulting callstack is saved to be accessed in the record step.
 */
static
size_t lttng_callstack_sequence_get_size(void *priv, struct lttng_kernel_probe_ctx *probe_ctx, size_t offset)
{
	struct stack_trace *trace;
	struct field_data *fdata = (struct field_data *) priv;
	size_t orig_offset = offset;
	int cpu = smp_processor_id();
	struct irq_ibt_state irq_ibt_state;

	/* do not write data if no space is available */
	trace = stack_trace_context(fdata, cpu);
	if (unlikely(!trace)) {
		offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned long));
		return offset - orig_offset;
	}

	/* reset stack trace, no need to clear memory */
	trace->nr_entries = 0;

	if (fdata->mode == CALLSTACK_USER)
		++per_cpu(callstack_user_nesting, cpu);

	/* do the real work and reserve space */
	irq_ibt_state = wrapper_irq_ibt_save();
	cs_types[fdata->mode].save_func(trace);
	wrapper_irq_ibt_restore(irq_ibt_state);

	if (fdata->mode == CALLSTACK_USER)
		per_cpu(callstack_user_nesting, cpu)--;

	/*
	 * Remove final ULONG_MAX delimiter. If we cannot find it, add
	 * our own marker to show that the stack is incomplete. This is
	 * more compact for a trace.
	 */
	if (trace->nr_entries > 0
			&& trace->entries[trace->nr_entries - 1] == ULONG_MAX) {
		trace->nr_entries--;
	}
	offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned long));
	offset += sizeof(unsigned long) * trace->nr_entries;
	/* Add our own ULONG_MAX delimiter to show incomplete stack. */
	if (trace->nr_entries == trace->max_entries)
		offset += sizeof(unsigned long);
	return offset - orig_offset;
}

static
void lttng_callstack_length_record(void *priv, struct lttng_kernel_probe_ctx *probe_ctx,
			struct lttng_kernel_ring_buffer_ctx *ctx,
			struct lttng_kernel_channel_buffer *chan)
{
	int cpu = ctx->priv.reserve_cpu;
	struct field_data *fdata = (struct field_data *) priv;
	struct stack_trace *trace = stack_trace_context(fdata, cpu);
	unsigned int nr_seq_entries;

	if (unlikely(!trace)) {
		nr_seq_entries = 0;
	} else {
		nr_seq_entries = trace->nr_entries;
		if (trace->nr_entries == trace->max_entries)
			nr_seq_entries++;
	}
	chan->ops->event_write(ctx, &nr_seq_entries, sizeof(unsigned int), lttng_alignof(unsigned int));
}
static
void lttng_callstack_sequence_record(void *priv, struct lttng_kernel_probe_ctx *probe_ctx,
			struct lttng_kernel_ring_buffer_ctx *ctx,
			struct lttng_kernel_channel_buffer *chan)
{
	int cpu = ctx->priv.reserve_cpu;
	struct field_data *fdata = (struct field_data *) priv;
	struct stack_trace *trace = stack_trace_context(fdata, cpu);
	unsigned int nr_seq_entries;

	if (unlikely(!trace)) {
		lib_ring_buffer_align_ctx(ctx, lttng_alignof(unsigned long));
		return;
	}
	nr_seq_entries = trace->nr_entries;
	if (trace->nr_entries == trace->max_entries)
		nr_seq_entries++;
	chan->ops->event_write(ctx, trace->entries,
			sizeof(unsigned long) * trace->nr_entries, lttng_alignof(unsigned long));
	/* Add our own ULONG_MAX delimiter to show incomplete stack. */
	if (trace->nr_entries == trace->max_entries) {
		unsigned long delim = ULONG_MAX;

		chan->ops->event_write(ctx, &delim, sizeof(unsigned long), 1);
	}
}
Commit	Line	Data
b7cdc182	1	/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
b6ee48d2 MD	2	*
	3	* lttng-context-callstack-legacy-impl.h
	4	*
	5	* LTTng callstack event context, legacy implementation. Targets
	6	* kernels and architectures not yet using the stacktrace common
	7	* infrastructure introduced in the upstream Linux kernel by commit
	8	* 214d8ca6ee "stacktrace: Provide common infrastructure" (merged in
	9	* Linux 5.2, then gradually introduced within architectures).
	10	*
	11	* Copyright (C) 2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	12	* Copyright (C) 2014 Francis Giraldeau <francis.giraldeau@gmail.com>
	13	*/
	14
	15	#define MAX_ENTRIES 128
	16
	17	enum lttng_cs_ctx_modes {
	18	CALLSTACK_KERNEL = 0,
	19	CALLSTACK_USER = 1,
	20	NR_CALLSTACK_MODES,
	21	};
	22
	23	struct lttng_cs_dispatch {
	24	struct stack_trace stack_trace;
	25	unsigned long entries[MAX_ENTRIES];
	26	};
	27
	28	struct lttng_cs {
	29	struct lttng_cs_dispatch dispatch[RING_BUFFER_MAX_NESTING];
	30	};
	31
	32	struct field_data {
	33	struct lttng_cs __percpu *cs_percpu;
	34	enum lttng_cs_ctx_modes mode;
	35	};
	36
	37	struct lttng_cs_type {
	38	const char *name;
ceabb767	39	const char *length_name;
b6ee48d2 MD	40	const char *save_func_name;
	41	void (save_func)(struct stack_trace trace);
	42	};
	43
	44	static struct lttng_cs_type cs_types[] = {
	45	{
	46	.name = "callstack_kernel",
ceabb767	47	.length_name = "_callstack_kernel_length",
b6ee48d2 MD	48	.save_func_name = "save_stack_trace",
	49	.save_func = NULL,
	50	},
	51	{
	52	.name = "callstack_user",
ceabb767	53	.length_name = "_callstack_user_length",
b6ee48d2 MD	54	.save_func_name = "save_stack_trace_user",
	55	.save_func = NULL,
	56	},
	57	};
	58
	59	static
	60	int init_type(enum lttng_cs_ctx_modes mode)
	61	{
	62	unsigned long func;
	63
	64	if (cs_types[mode].save_func)
	65	return 0;
	66	func = kallsyms_lookup_funcptr(cs_types[mode].save_func_name);
	67	if (!func) {
	68	printk(KERN_WARNING "LTTng: symbol lookup failed: %s\n",
	69	cs_types[mode].save_func_name);
	70	return -EINVAL;
	71	}
	72	cs_types[mode].save_func = (void *) func;
	73	return 0;
	74	}
	75
b5a89a3f MD	76	static
	77	void lttng_cs_set_init(struct lttng_cs __percpu *cs_set)
	78	{
	79	int cpu, i;
	80
	81	for_each_possible_cpu(cpu) {
	82	struct lttng_cs *cs;
	83
	84	cs = per_cpu_ptr(cs_set, cpu);
	85	for (i = 0; i < RING_BUFFER_MAX_NESTING; i++) {
	86	struct lttng_cs_dispatch *dispatch;
	87
	88	dispatch = &cs->dispatch[i];
	89	dispatch->stack_trace.entries = dispatch->entries;
	90	dispatch->stack_trace.max_entries = MAX_ENTRIES;
	91	}
	92	}
	93	}
	94
b6ee48d2 MD	95	/* Keep track of nesting inside userspace callstack context code */
	96	DEFINE_PER_CPU(int, callstack_user_nesting);
	97
2dc781e0 MD	98	/*
	99	* Note: these callbacks expect to be invoked with preemption disabled across
	100	* get_size and record due to its use of a per-cpu stack.
	101	*/
b6ee48d2	102	static
2dc781e0	103	struct stack_trace stack_trace_context(struct field_data fdata, int cpu)
b6ee48d2 MD	104	{
	105	int buffer_nesting, cs_user_nesting;
	106	struct lttng_cs *cs;
b6ee48d2 MD	107
	108	/*
	109	* Do not gather the userspace callstack context when the event was
	110	* triggered by the userspace callstack context saving mechanism.
	111	*/
2dc781e0	112	cs_user_nesting = per_cpu(callstack_user_nesting, cpu);
b6ee48d2 MD	113
	114	if (fdata->mode == CALLSTACK_USER && cs_user_nesting >= 1)
	115	return NULL;
	116
	117	/*
	118	* get_cpu() is not required, preemption is already
	119	* disabled while event is written.
	120	*
	121	* max nesting is checked in lib_ring_buffer_get_cpu().
	122	* Check it again as a safety net.
	123	*/
2dc781e0 MD	124	cs = per_cpu_ptr(fdata->cs_percpu, cpu);
2dc781e0 MD	125	buffer_nesting = per_cpu(lib_ring_buffer_nesting, cpu) - 1;
b6ee48d2 MD	126	if (buffer_nesting >= RING_BUFFER_MAX_NESTING)
	127	return NULL;
	128
	129	return &cs->dispatch[buffer_nesting].stack_trace;
	130	}
	131
ceabb767	132	static
a92e844e	133	size_t lttng_callstack_length_get_size(void priv, struct lttng_kernel_probe_ctx probe_ctx, size_t offset)
ceabb767 MD	134	{
	135	size_t orig_offset = offset;
	136
	137	offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned int));
	138	offset += sizeof(unsigned int);
	139	return offset - orig_offset;
	140	}
	141
b6ee48d2 MD	142	/*
	143	* In order to reserve the correct size, the callstack is computed. The
	144	* resulting callstack is saved to be accessed in the record step.
	145	*/
	146	static
a92e844e	147	size_t lttng_callstack_sequence_get_size(void priv, struct lttng_kernel_probe_ctx probe_ctx, size_t offset)
b6ee48d2 MD	148	{
b6ee48d2 MD	149	struct stack_trace *trace;
2dc781e0	150	struct field_data fdata = (struct field_data ) priv;
b6ee48d2	151	size_t orig_offset = offset;
2dc781e0	152	int cpu = smp_processor_id();
92e2c5fe	153	struct irq_ibt_state irq_ibt_state;
b6ee48d2 MD	154
b6ee48d2 MD	155	/* do not write data if no space is available */
2dc781e0	156	trace = stack_trace_context(fdata, cpu);
b6ee48d2	157	if (unlikely(!trace)) {
b6ee48d2 MD	158	offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned long));
	159	return offset - orig_offset;
	160	}
	161
	162	/* reset stack trace, no need to clear memory */
	163	trace->nr_entries = 0;
	164
	165	if (fdata->mode == CALLSTACK_USER)
2dc781e0	166	++per_cpu(callstack_user_nesting, cpu);
b6ee48d2 MD	167
b6ee48d2 MD	168	/* do the real work and reserve space */
92e2c5fe	169	irq_ibt_state = wrapper_irq_ibt_save();
b6ee48d2	170	cs_types[fdata->mode].save_func(trace);
92e2c5fe	171	wrapper_irq_ibt_restore(irq_ibt_state);
b6ee48d2 MD	172
b6ee48d2 MD	173	if (fdata->mode == CALLSTACK_USER)
2dc781e0	174	per_cpu(callstack_user_nesting, cpu)--;
b6ee48d2 MD	175
	176	/*
	177	* Remove final ULONG_MAX delimiter. If we cannot find it, add
	178	* our own marker to show that the stack is incomplete. This is
	179	* more compact for a trace.
	180	*/
	181	if (trace->nr_entries > 0
	182	&& trace->entries[trace->nr_entries - 1] == ULONG_MAX) {
	183	trace->nr_entries--;
	184	}
b6ee48d2 MD	185	offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned long));
	186	offset += sizeof(unsigned long) * trace->nr_entries;
	187	/* Add our own ULONG_MAX delimiter to show incomplete stack. */
	188	if (trace->nr_entries == trace->max_entries)
	189	offset += sizeof(unsigned long);
	190	return offset - orig_offset;
	191	}
	192
	193	static
a92e844e	194	void lttng_callstack_length_record(void priv, struct lttng_kernel_probe_ctx probe_ctx,
8a57ec02	195	struct lttng_kernel_ring_buffer_ctx *ctx,
f7d06400	196	struct lttng_kernel_channel_buffer *chan)
b6ee48d2	197	{
2dc781e0 MD	198	int cpu = ctx->priv.reserve_cpu;
	199	struct field_data fdata = (struct field_data ) priv;
	200	struct stack_trace *trace = stack_trace_context(fdata, cpu);
b6ee48d2 MD	201	unsigned int nr_seq_entries;
	202
	203	if (unlikely(!trace)) {
	204	nr_seq_entries = 0;
ceabb767 MD	205	} else {
	206	nr_seq_entries = trace->nr_entries;
	207	if (trace->nr_entries == trace->max_entries)
	208	nr_seq_entries++;
	209	}
f5ffbd77	210	chan->ops->event_write(ctx, &nr_seq_entries, sizeof(unsigned int), lttng_alignof(unsigned int));
ceabb767 MD	211	}
ceabb767 MD	212	static
a92e844e	213	void lttng_callstack_sequence_record(void priv, struct lttng_kernel_probe_ctx probe_ctx,
8a57ec02	214	struct lttng_kernel_ring_buffer_ctx *ctx,
f7d06400	215	struct lttng_kernel_channel_buffer *chan)
ceabb767	216	{
2dc781e0 MD	217	int cpu = ctx->priv.reserve_cpu;
	218	struct field_data fdata = (struct field_data ) priv;
	219	struct stack_trace *trace = stack_trace_context(fdata, cpu);
ceabb767 MD	220	unsigned int nr_seq_entries;
ceabb767 MD	221
ceabb767	222	if (unlikely(!trace)) {
f5ffbd77	223	lib_ring_buffer_align_ctx(ctx, lttng_alignof(unsigned long));
b6ee48d2 MD	224	return;
b6ee48d2 MD	225	}
b6ee48d2 MD	226	nr_seq_entries = trace->nr_entries;
	227	if (trace->nr_entries == trace->max_entries)
	228	nr_seq_entries++;
b6ee48d2	229	chan->ops->event_write(ctx, trace->entries,
f5ffbd77	230	sizeof(unsigned long) * trace->nr_entries, lttng_alignof(unsigned long));
b6ee48d2 MD	231	/* Add our own ULONG_MAX delimiter to show incomplete stack. */
	232	if (trace->nr_entries == trace->max_entries) {
	233	unsigned long delim = ULONG_MAX;
	234
f5ffbd77	235	chan->ops->event_write(ctx, &delim, sizeof(unsigned long), 1);
b6ee48d2 MD	236	}
b6ee48d2 MD	237	}