[lttng-modules.git] / src / lttng-context-callstack-stackwalk-impl.h

/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
 *
 * lttng-context-callstack-stackwalk-impl.h
 *
 * LTTng callstack event context, stackwalk implementation. Targets
 * kernels and architectures using the stacktrace common infrastructure
 * introduced in the upstream Linux kernel by commit 214d8ca6ee
 * "stacktrace: Provide common infrastructure" (merged in Linux 5.2,
 * then gradually introduced within architectures).
 *
 * Copyright (C) 2014-2019 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 * Copyright (C) 2014 Francis Giraldeau <francis.giraldeau@gmail.com>
 */

#define MAX_ENTRIES 128

enum lttng_cs_ctx_modes {
	CALLSTACK_KERNEL = 0,
	CALLSTACK_USER = 1,
	NR_CALLSTACK_MODES,
};

struct lttng_stack_trace {
	unsigned long entries[MAX_ENTRIES];
	unsigned int nr_entries;
};

struct lttng_cs {
	struct lttng_stack_trace stack_trace[RING_BUFFER_MAX_NESTING];
};

struct field_data {
	struct lttng_cs __percpu *cs_percpu;
	enum lttng_cs_ctx_modes mode;
};

static
unsigned int (*save_func_kernel)(unsigned long *store, unsigned int size,
				unsigned int skipnr);
static
unsigned int (*save_func_user)(unsigned long *store, unsigned int size);

static
int init_type_callstack_kernel(void)
{
	unsigned long func;
	const char *func_name = "stack_trace_save";

	if (save_func_kernel)
		return 0;
	func = kallsyms_lookup_funcptr(func_name);
	if (!func) {
		printk(KERN_WARNING "LTTng: symbol lookup failed: %s\n",
				func_name);
		return -EINVAL;
	}
	save_func_kernel = (void *) func;
	return 0;
}

static
int init_type_callstack_user(void)
{
	unsigned long func;
	const char *func_name = "stack_trace_save_user";

	if (save_func_user)
		return 0;
	func = kallsyms_lookup_funcptr(func_name);
	if (!func) {
		printk(KERN_WARNING "LTTng: symbol lookup failed: %s\n",
				func_name);
		return -EINVAL;
	}
	save_func_user = (void *) func;
	return 0;
}

static
int init_type(enum lttng_cs_ctx_modes mode)
{
	switch (mode) {
	case CALLSTACK_KERNEL:
		return init_type_callstack_kernel();
	case CALLSTACK_USER:
		return init_type_callstack_user();
	default:
		return -EINVAL;
	}
}

static
void lttng_cs_set_init(struct lttng_cs __percpu *cs_set)
{
}

/* Keep track of nesting inside userspace callstack context code */
DEFINE_PER_CPU(int, callstack_user_nesting);

/*
 * Note: these callbacks expect to be invoked with preemption disabled across
 * get_size and record due to its use of a per-cpu stack.
 */
static
struct lttng_stack_trace *stack_trace_context(struct field_data *fdata, int cpu)
{
	int buffer_nesting, cs_user_nesting;
	struct lttng_cs *cs;

	/*
	 * Do not gather the userspace callstack context when the event was
	 * triggered by the userspace callstack context saving mechanism.
	 */
	cs_user_nesting = per_cpu(callstack_user_nesting, cpu);

	if (fdata->mode == CALLSTACK_USER && cs_user_nesting >= 1)
		return NULL;

	/*
	 * get_cpu() is not required, preemption is already
	 * disabled while event is written.
	 *
	 * max nesting is checked in lib_ring_buffer_get_cpu().
	 * Check it again as a safety net.
	 */
	cs = per_cpu_ptr(fdata->cs_percpu, cpu);
	buffer_nesting = per_cpu(lib_ring_buffer_nesting, cpu) - 1;
	if (buffer_nesting >= RING_BUFFER_MAX_NESTING)
		return NULL;

	return &cs->stack_trace[buffer_nesting];
}

static
size_t lttng_callstack_length_get_size(void *priv, struct lttng_kernel_probe_ctx *probe_ctx, size_t offset)
{
	size_t orig_offset = offset;

	offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned int));
	offset += sizeof(unsigned int);
	return offset - orig_offset;
}

/*
 * In order to reserve the correct size, the callstack is computed. The
 * resulting callstack is saved to be accessed in the record step.
 */
static
size_t lttng_callstack_sequence_get_size(void *priv, struct lttng_kernel_probe_ctx *probe_ctx, size_t offset)
{
	struct lttng_stack_trace *trace;
	struct field_data *fdata = (struct field_data *) priv;
	size_t orig_offset = offset;
	int cpu = smp_processor_id();
	struct irq_ibt_state irq_ibt_state;

	/* do not write data if no space is available */
	trace = stack_trace_context(fdata, cpu);
	if (unlikely(!trace)) {
		offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned long));
		return offset - orig_offset;
	}

	/* reset stack trace, no need to clear memory */
	trace->nr_entries = 0;

	switch (fdata->mode) {
	case CALLSTACK_KERNEL:
		/* do the real work and reserve space */
		irq_ibt_state = wrapper_irq_ibt_save();
		trace->nr_entries = save_func_kernel(trace->entries,
						MAX_ENTRIES, 0);
		wrapper_irq_ibt_restore(irq_ibt_state);
		break;
	case CALLSTACK_USER:
		++per_cpu(callstack_user_nesting, cpu);
		/* do the real work and reserve space */
		irq_ibt_state = wrapper_irq_ibt_save();
		trace->nr_entries = save_func_user(trace->entries,
						MAX_ENTRIES);
		wrapper_irq_ibt_restore(irq_ibt_state);
		per_cpu(callstack_user_nesting, cpu)--;
		break;
	default:
		WARN_ON_ONCE(1);
	}

	/*
	 * If the array is filled, add our own marker to show that the
	 * stack is incomplete.
	 */
	offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned long));
	offset += sizeof(unsigned long) * trace->nr_entries;
	/* Add our own ULONG_MAX delimiter to show incomplete stack. */
	if (trace->nr_entries == MAX_ENTRIES)
		offset += sizeof(unsigned long);
	return offset - orig_offset;
}

static
void lttng_callstack_length_record(void *priv, struct lttng_kernel_probe_ctx *probe_ctx,
			struct lttng_kernel_ring_buffer_ctx *ctx,
			struct lttng_kernel_channel_buffer *chan)
{
	int cpu = ctx->priv.reserve_cpu;
	struct field_data *fdata = (struct field_data *) priv;
	struct lttng_stack_trace *trace = stack_trace_context(fdata, cpu);
	unsigned int nr_seq_entries;

	if (unlikely(!trace)) {
		nr_seq_entries = 0;
	} else {
		nr_seq_entries = trace->nr_entries;
		if (trace->nr_entries == MAX_ENTRIES)
			nr_seq_entries++;
	}
	chan->ops->event_write(ctx, &nr_seq_entries, sizeof(unsigned int), lttng_alignof(unsigned int));
}

static
void lttng_callstack_sequence_record(void *priv, struct lttng_kernel_probe_ctx *probe_ctx,
			struct lttng_kernel_ring_buffer_ctx *ctx,
			struct lttng_kernel_channel_buffer *chan)
{
	int cpu = ctx->priv.reserve_cpu;
	struct field_data *fdata = (struct field_data *) priv;
	struct lttng_stack_trace *trace = stack_trace_context(fdata, cpu);
	unsigned int nr_seq_entries;

	if (unlikely(!trace)) {
		/* We need to align even if there are 0 elements. */
		lib_ring_buffer_align_ctx(ctx, lttng_alignof(unsigned long));
		return;
	}
	nr_seq_entries = trace->nr_entries;
	if (trace->nr_entries == MAX_ENTRIES)
		nr_seq_entries++;
	chan->ops->event_write(ctx, trace->entries,
			sizeof(unsigned long) * trace->nr_entries, lttng_alignof(unsigned long));
	/* Add our own ULONG_MAX delimiter to show incomplete stack. */
	if (trace->nr_entries == MAX_ENTRIES) {
		unsigned long delim = ULONG_MAX;

		chan->ops->event_write(ctx, &delim, sizeof(unsigned long), 1);
	}
}
Commit	Line	Data
b7cdc182	1	/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
b29c6286 MD	2	*
	3	* lttng-context-callstack-stackwalk-impl.h
	4	*
	5	* LTTng callstack event context, stackwalk implementation. Targets
	6	* kernels and architectures using the stacktrace common infrastructure
	7	* introduced in the upstream Linux kernel by commit 214d8ca6ee
	8	* "stacktrace: Provide common infrastructure" (merged in Linux 5.2,
	9	* then gradually introduced within architectures).
	10	*
	11	* Copyright (C) 2014-2019 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	12	* Copyright (C) 2014 Francis Giraldeau <francis.giraldeau@gmail.com>
	13	*/
	14
	15	#define MAX_ENTRIES 128
	16
	17	enum lttng_cs_ctx_modes {
	18	CALLSTACK_KERNEL = 0,
	19	CALLSTACK_USER = 1,
	20	NR_CALLSTACK_MODES,
	21	};
	22
	23	struct lttng_stack_trace {
	24	unsigned long entries[MAX_ENTRIES];
	25	unsigned int nr_entries;
	26	};
	27
	28	struct lttng_cs {
	29	struct lttng_stack_trace stack_trace[RING_BUFFER_MAX_NESTING];
	30	};
	31
	32	struct field_data {
	33	struct lttng_cs __percpu *cs_percpu;
	34	enum lttng_cs_ctx_modes mode;
	35	};
	36
	37	static
	38	unsigned int (save_func_kernel)(unsigned long store, unsigned int size,
	39	unsigned int skipnr);
	40	static
	41	unsigned int (save_func_user)(unsigned long store, unsigned int size);
	42
b29c6286 MD	43	static
	44	int init_type_callstack_kernel(void)
	45	{
	46	unsigned long func;
	47	const char *func_name = "stack_trace_save";
	48
	49	if (save_func_kernel)
	50	return 0;
	51	func = kallsyms_lookup_funcptr(func_name);
	52	if (!func) {
	53	printk(KERN_WARNING "LTTng: symbol lookup failed: %s\n",
	54	func_name);
	55	return -EINVAL;
	56	}
	57	save_func_kernel = (void *) func;
	58	return 0;
	59	}
	60
	61	static
	62	int init_type_callstack_user(void)
	63	{
	64	unsigned long func;
	65	const char *func_name = "stack_trace_save_user";
	66
	67	if (save_func_user)
	68	return 0;
	69	func = kallsyms_lookup_funcptr(func_name);
	70	if (!func) {
	71	printk(KERN_WARNING "LTTng: symbol lookup failed: %s\n",
	72	func_name);
	73	return -EINVAL;
	74	}
	75	save_func_user = (void *) func;
	76	return 0;
	77	}
	78
	79	static
	80	int init_type(enum lttng_cs_ctx_modes mode)
	81	{
	82	switch (mode) {
	83	case CALLSTACK_KERNEL:
	84	return init_type_callstack_kernel();
	85	case CALLSTACK_USER:
	86	return init_type_callstack_user();
	87	default:
	88	return -EINVAL;
	89	}
	90	}
	91
	92	static
	93	void lttng_cs_set_init(struct lttng_cs __percpu *cs_set)
	94	{
	95	}
	96
	97	/* Keep track of nesting inside userspace callstack context code */
	98	DEFINE_PER_CPU(int, callstack_user_nesting);
	99
2dc781e0 MD	100	/*
	101	* Note: these callbacks expect to be invoked with preemption disabled across
	102	* get_size and record due to its use of a per-cpu stack.
	103	*/
b29c6286	104	static
2dc781e0	105	struct lttng_stack_trace stack_trace_context(struct field_data fdata, int cpu)
b29c6286 MD	106	{
	107	int buffer_nesting, cs_user_nesting;
	108	struct lttng_cs *cs;
b29c6286 MD	109
	110	/*
	111	* Do not gather the userspace callstack context when the event was
	112	* triggered by the userspace callstack context saving mechanism.
	113	*/
2dc781e0	114	cs_user_nesting = per_cpu(callstack_user_nesting, cpu);
b29c6286 MD	115
	116	if (fdata->mode == CALLSTACK_USER && cs_user_nesting >= 1)
	117	return NULL;
	118
	119	/*
	120	* get_cpu() is not required, preemption is already
	121	* disabled while event is written.
	122	*
	123	* max nesting is checked in lib_ring_buffer_get_cpu().
	124	* Check it again as a safety net.
	125	*/
2dc781e0 MD	126	cs = per_cpu_ptr(fdata->cs_percpu, cpu);
2dc781e0 MD	127	buffer_nesting = per_cpu(lib_ring_buffer_nesting, cpu) - 1;
b29c6286 MD	128	if (buffer_nesting >= RING_BUFFER_MAX_NESTING)
	129	return NULL;
	130
	131	return &cs->stack_trace[buffer_nesting];
	132	}
	133
ceabb767	134	static
a92e844e	135	size_t lttng_callstack_length_get_size(void priv, struct lttng_kernel_probe_ctx probe_ctx, size_t offset)
ceabb767 MD	136	{
	137	size_t orig_offset = offset;
	138
	139	offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned int));
	140	offset += sizeof(unsigned int);
	141	return offset - orig_offset;
	142	}
	143
b29c6286 MD	144	/*
	145	* In order to reserve the correct size, the callstack is computed. The
	146	* resulting callstack is saved to be accessed in the record step.
	147	*/
	148	static
a92e844e	149	size_t lttng_callstack_sequence_get_size(void priv, struct lttng_kernel_probe_ctx probe_ctx, size_t offset)
b29c6286 MD	150	{
b29c6286 MD	151	struct lttng_stack_trace *trace;
2dc781e0	152	struct field_data fdata = (struct field_data ) priv;
b29c6286	153	size_t orig_offset = offset;
2dc781e0	154	int cpu = smp_processor_id();
92e2c5fe	155	struct irq_ibt_state irq_ibt_state;
b29c6286 MD	156
b29c6286 MD	157	/* do not write data if no space is available */
2dc781e0	158	trace = stack_trace_context(fdata, cpu);
b29c6286	159	if (unlikely(!trace)) {
b29c6286 MD	160	offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned long));
	161	return offset - orig_offset;
	162	}
	163
	164	/* reset stack trace, no need to clear memory */
	165	trace->nr_entries = 0;
	166
	167	switch (fdata->mode) {
	168	case CALLSTACK_KERNEL:
	169	/* do the real work and reserve space */
92e2c5fe	170	irq_ibt_state = wrapper_irq_ibt_save();
b29c6286 MD	171	trace->nr_entries = save_func_kernel(trace->entries,
b29c6286 MD	172	MAX_ENTRIES, 0);
92e2c5fe	173	wrapper_irq_ibt_restore(irq_ibt_state);
b29c6286 MD	174	break;
b29c6286 MD	175	case CALLSTACK_USER:
2dc781e0	176	++per_cpu(callstack_user_nesting, cpu);
b29c6286	177	/* do the real work and reserve space */
92e2c5fe	178	irq_ibt_state = wrapper_irq_ibt_save();
b29c6286 MD	179	trace->nr_entries = save_func_user(trace->entries,
b29c6286 MD	180	MAX_ENTRIES);
92e2c5fe	181	wrapper_irq_ibt_restore(irq_ibt_state);
2dc781e0	182	per_cpu(callstack_user_nesting, cpu)--;
b29c6286 MD	183	break;
	184	default:
	185	WARN_ON_ONCE(1);
	186	}
	187
	188	/*
	189	* If the array is filled, add our own marker to show that the
	190	* stack is incomplete.
	191	*/
b29c6286 MD	192	offset += lib_ring_buffer_align(offset, lttng_alignof(unsigned long));
	193	offset += sizeof(unsigned long) * trace->nr_entries;
	194	/* Add our own ULONG_MAX delimiter to show incomplete stack. */
	195	if (trace->nr_entries == MAX_ENTRIES)
	196	offset += sizeof(unsigned long);
	197	return offset - orig_offset;
	198	}
	199
	200	static
a92e844e	201	void lttng_callstack_length_record(void priv, struct lttng_kernel_probe_ctx probe_ctx,
8a57ec02	202	struct lttng_kernel_ring_buffer_ctx *ctx,
f7d06400	203	struct lttng_kernel_channel_buffer *chan)
b29c6286	204	{
2dc781e0 MD	205	int cpu = ctx->priv.reserve_cpu;
	206	struct field_data fdata = (struct field_data ) priv;
	207	struct lttng_stack_trace *trace = stack_trace_context(fdata, cpu);
b29c6286 MD	208	unsigned int nr_seq_entries;
	209
	210	if (unlikely(!trace)) {
	211	nr_seq_entries = 0;
ceabb767 MD	212	} else {
	213	nr_seq_entries = trace->nr_entries;
	214	if (trace->nr_entries == MAX_ENTRIES)
	215	nr_seq_entries++;
	216	}
f5ffbd77	217	chan->ops->event_write(ctx, &nr_seq_entries, sizeof(unsigned int), lttng_alignof(unsigned int));
ceabb767 MD	218	}
	219
	220	static
a92e844e	221	void lttng_callstack_sequence_record(void priv, struct lttng_kernel_probe_ctx probe_ctx,
8a57ec02	222	struct lttng_kernel_ring_buffer_ctx *ctx,
f7d06400	223	struct lttng_kernel_channel_buffer *chan)
ceabb767	224	{
2dc781e0 MD	225	int cpu = ctx->priv.reserve_cpu;
	226	struct field_data fdata = (struct field_data ) priv;
	227	struct lttng_stack_trace *trace = stack_trace_context(fdata, cpu);
ceabb767 MD	228	unsigned int nr_seq_entries;
ceabb767 MD	229
ceabb767	230	if (unlikely(!trace)) {
f5ffbd77 MD	231	/* We need to align even if there are 0 elements. */
f5ffbd77 MD	232	lib_ring_buffer_align_ctx(ctx, lttng_alignof(unsigned long));
b29c6286 MD	233	return;
b29c6286 MD	234	}
b29c6286 MD	235	nr_seq_entries = trace->nr_entries;
	236	if (trace->nr_entries == MAX_ENTRIES)
	237	nr_seq_entries++;
b29c6286	238	chan->ops->event_write(ctx, trace->entries,
f5ffbd77	239	sizeof(unsigned long) * trace->nr_entries, lttng_alignof(unsigned long));
b29c6286 MD	240	/* Add our own ULONG_MAX delimiter to show incomplete stack. */
	241	if (trace->nr_entries == MAX_ENTRIES) {
	242	unsigned long delim = ULONG_MAX;
	243
f5ffbd77	244	chan->ops->event_write(ctx, &delim, sizeof(unsigned long), 1);
b29c6286 MD	245	}
b29c6286 MD	246	}