[lttng-modules.git] / wrapper / vmalloc.h

/* SPDX-License-Identifier: (GPL-2.0 or LGPL-2.1)
 *
 * wrapper/vmalloc.h
 *
 * wrapper around vmalloc_sync_all. Using KALLSYMS to get its address when
 * available, else we need to have a kernel that exports this function to GPL
 * modules.
 *
 * Copyright (C) 2011-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 */

#ifndef _LTTNG_WRAPPER_VMALLOC_H
#define _LTTNG_WRAPPER_VMALLOC_H

#include <lttng-kernel-version.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>

#ifdef CONFIG_KALLSYMS

#include <linux/kallsyms.h>
#include <wrapper/kallsyms.h>
#include <lttng-kernel-version.h>

#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,8,0))

/*
 * wrapper_vmalloc_sync_mappings was removed in v5.8, the vmalloc mappings
 * are now synchronized when they are created or torn down.
 */
static inline
void wrapper_vmalloc_sync_mappings(void)
{}

#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0) \
  || LTTNG_KERNEL_RANGE(5,5,12, 5,6,0)            \
  || LTTNG_KERNEL_RANGE(5,4,28, 5,5,0)            \
  || LTTNG_KERNEL_RANGE(5,2,37, 5,3,0)            \
  || LTTNG_KERNEL_RANGE(4,19,113, 4,20,0)         \
  || LTTNG_KERNEL_RANGE(4,14,175, 4,15,0)         \
  || LTTNG_KERNEL_RANGE(4,9,218, 4,10,0)          \
  || LTTNG_KERNEL_RANGE(4,4,218, 4,5,0)		  \
  || LTTNG_UBUNTU_KERNEL_RANGE(4,15,18,97, 4,16,0,0) \
  || LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,48, 5,1,0,0)   \
  || LTTNG_UBUNTU_KERNEL_RANGE(5,3,18,52, 5,4,0,0)   \
  || LTTNG_RHEL_KERNEL_RANGE(4,18,0,240,0,0, 4,19,0,0,0,0))

static inline
void wrapper_vmalloc_sync_mappings(void)
{
	void (*vmalloc_sync_mappings_sym)(void);

	vmalloc_sync_mappings_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_mappings");
	if (vmalloc_sync_mappings_sym) {
		vmalloc_sync_mappings_sym();
	} else {
#ifdef CONFIG_X86
		/*
		 * Only x86 needs vmalloc_sync_mappings to make sure LTTng does not
		 * trigger recursive page faults.
		 */
		printk_once(KERN_WARNING "LTTng: vmalloc_sync_mappings symbol lookup failed.\n");
		printk_once(KERN_WARNING "Page fault handler and NMI tracing might trigger faults.\n");
#endif
	}
}

/*
 * Canary function to check for 'vmalloc_sync_mappings()' at compile time.
 *
 * From 'include/linux/vmalloc.h':
 *
 *   void vmalloc_sync_mappings(void);
 */
static inline
void __canary__vmalloc_sync_mappings(void)
{
	vmalloc_sync_mappings();
}

#else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */

/*
 * Map vmalloc_sync_mappings to vmalloc_sync_all() on kernels before 5.6.
 */
static inline
void wrapper_vmalloc_sync_mappings(void)
{
	void (*vmalloc_sync_all_sym)(void);

	vmalloc_sync_all_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_all");
	if (vmalloc_sync_all_sym) {
		vmalloc_sync_all_sym();
	} else {
#ifdef CONFIG_X86
		/*
		 * Only x86 needs vmalloc_sync_all to make sure LTTng does not
		 * trigger recursive page faults.
		 */
		printk_once(KERN_WARNING "LTTng: vmalloc_sync_all symbol lookup failed.\n");
		printk_once(KERN_WARNING "Page fault handler and NMI tracing might trigger faults.\n");
#endif
	}
}

/*
 * Canary function to check for 'vmalloc_sync_all()' at compile time.
 *
 * From 'include/linux/vmalloc.h':
 *
 *   void vmalloc_sync_all(void);
 */
static inline
void __canary__vmalloc_sync_all(void)
{
	vmalloc_sync_all();
}

#endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */

#else /* CONFIG_KALLSYMS */

#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,8,0))

/*
 * wrapper_vmalloc_sync_mappings was removed in v5.8, the vmalloc mappings
 * are now synchronized when they are created or torn down.
 */
static inline
void wrapper_vmalloc_sync_mappings(void)
{}

#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0) \
  || LTTNG_KERNEL_RANGE(5,5,12, 5,6,0)            \
  || LTTNG_KERNEL_RANGE(5,4,28, 5,5,0)            \
  || LTTNG_KERNEL_RANGE(5,2,37, 5,3,0)            \
  || LTTNG_KERNEL_RANGE(4,19,113, 4,20,0)         \
  || LTTNG_KERNEL_RANGE(4,14,175, 4,15,0)         \
  || LTTNG_KERNEL_RANGE(4,9,218, 4,10,0)          \
  || LTTNG_KERNEL_RANGE(4,4,218, 4,5,0))	  \
  || LTTNG_UBUNTU_KERNEL_RANGE(4,15,18,97, 4,18,0,0) \
  || LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,48, 5,1,0,0)   \
  || LTTNG_UBUNTU_KERNEL_RANGE(5,3,18,52, 5,4,0,0)

static inline
void wrapper_vmalloc_sync_mappings(void)
{
	return vmalloc_sync_mappings();
}

#else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */

static inline
void wrapper_vmalloc_sync_mappings(void)
{
	return vmalloc_sync_all();
}

#endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */

#endif

#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,12,0))
static inline
void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node)
{
	void *ret;

	ret = kvmalloc_node(size, flags, node);
	if (is_vmalloc_addr(ret)) {
		/*
		 * Make sure we don't trigger recursive page faults in the
		 * tracing fast path.
		 */
		wrapper_vmalloc_sync_mappings();
	}
	return ret;
}

static inline
void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node)
{
	return lttng_kvmalloc_node(size, flags | __GFP_ZERO, node);
}

static inline
void *lttng_kvmalloc(unsigned long size, gfp_t flags)
{
	return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE);
}

static inline
void *lttng_kvzalloc(unsigned long size, gfp_t flags)
{
	return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE);
}

static inline
void lttng_kvfree(const void *addr)
{
	kvfree(addr);
}

#else

#include <linux/slab.h>

static inline
void print_vmalloc_node_range_warning(void)
{
	printk_once(KERN_WARNING "LTTng: __vmalloc_node_range symbol lookup failed.\n");
	printk_once(KERN_WARNING "Tracer performance will be degraded on NUMA systems.\n");
	printk_once(KERN_WARNING "Please rebuild your kernel with CONFIG_KALLSYMS enabled.\n");
}

#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,0,0))

/*
 * kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
 */
static inline
void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			const void *caller)
{
#ifdef CONFIG_KALLSYMS
	/*
	 * If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
	 */
	void *(*lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			const void *caller);

	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	if (lttng__vmalloc_node_range)
		return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
				vm_flags, node, caller);
#endif
	if (node != NUMA_NO_NODE)
		print_vmalloc_node_range_warning();
	return __vmalloc(size, gfp_mask, prot);
}

/*
 * Canary function to check for '__vmalloc_node_range()' at compile time.
 *
 * From 'include/linux/vmalloc.h':
 *
 *   extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
 *                           unsigned long start, unsigned long end, gfp_t gfp_mask,
 *                           pgprot_t prot, unsigned long vm_flags, int node,
 *                           const void *caller);
 */
static inline
void *__canary____lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			const void *caller)
{
	return __vmalloc_node_range(size, align, start, end, gfp_mask, prot,
			vm_flags, node, caller);
}

#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,6,0))

/*
 * kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
 */
static inline
void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			const void *caller)
{
#ifdef CONFIG_KALLSYMS
	/*
	 * If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
	 */
	void *(*lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, int node, const void *caller);

	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	if (lttng__vmalloc_node_range)
		return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
				node, caller);
#endif
	if (node != NUMA_NO_NODE)
		print_vmalloc_node_range_warning();
	return __vmalloc(size, gfp_mask, prot);
}

/*
 * Canary function to check for '__vmalloc_node_range()' at compile time.
 *
 * From 'include/linux/vmalloc.h':
 *
 *   extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
 *                           unsigned long start, unsigned long end, gfp_t gfp_mask,
 *                           pgprot_t prot, unsigned long vm_flags, int node,
 *                           const void *caller);
 */
static inline
void *__canary____lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, int node, const void *caller)
{
	return __vmalloc_node_range(size, align, start, end, gfp_mask, prot,
			node, caller);
}

#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(2,6,38))

/*
 * kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
 */
static inline
void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			void *caller)
{
#ifdef CONFIG_KALLSYMS
	/*
	 * If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
	 */
	void *(*lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, int node, void *caller);

	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	if (lttng__vmalloc_node_range)
		return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
				node, caller);
#endif
	if (node != NUMA_NO_NODE)
		print_vmalloc_node_range_warning();
	return __vmalloc(size, gfp_mask, prot);
}

/*
 * Canary function to check for '__vmalloc_node_range()' at compile time.
 *
 * From 'include/linux/vmalloc.h':
 *
 *   extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
 *                           unsigned long start, unsigned long end, gfp_t gfp_mask,
 *                           pgprot_t prot, unsigned long vm_flags, int node,
 *                           void *caller);
 */
static inline
void *__canary____lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, int node, void *caller)
{
	return __vmalloc_node_range(size, align, start, end, gfp_mask, prot,
			node, caller);
}

#else /* (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(2,6,38)) */

/*
 * Basic fallback for kernel prior to 2.6.38 without __vmalloc_node_range()
 */
static inline
void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
			unsigned long start, unsigned long end, gfp_t gfp_mask,
			pgprot_t prot, unsigned long vm_flags, int node,
			void *caller)
{
	return __vmalloc(size, gfp_mask, prot);
}

#endif

/**
 * lttng_kvmalloc_node - attempt to allocate physically contiguous memory, but upon
 * failure, fall back to non-contiguous (vmalloc) allocation.
 * @size: size of the request.
 * @flags: gfp mask for the allocation - must be compatible with GFP_KERNEL.
 *
 * Uses kmalloc to get the memory but if the allocation fails then falls back
 * to the vmalloc allocator. Use lttng_kvfree to free the memory.
 *
 * Reclaim modifiers - __GFP_NORETRY, __GFP_REPEAT and __GFP_NOFAIL are not supported
 */
static inline
void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node)
{
	void *ret;

	/*
	 * vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables)
	 * so the given set of flags has to be compatible.
	 */
	WARN_ON_ONCE((flags & GFP_KERNEL) != GFP_KERNEL);

	/*
	 * If the allocation fits in a single page, do not fallback.
	 */
	if (size <= PAGE_SIZE) {
		return kmalloc_node(size, flags, node);
	}

	/*
	 * Make sure that larger requests are not too disruptive - no OOM
	 * killer and no allocation failure warnings as we have a fallback
	 */
	ret = kmalloc_node(size, flags | __GFP_NOWARN | __GFP_NORETRY, node);
	if (!ret) {
		ret = __lttng_vmalloc_node_range(size, 1,
				VMALLOC_START, VMALLOC_END,
				flags | __GFP_HIGHMEM, PAGE_KERNEL, 0,
				node, __builtin_return_address(0));
		/*
		 * Make sure we don't trigger recursive page faults in the
		 * tracing fast path.
		 */
		wrapper_vmalloc_sync_mappings();
	}
	return ret;
}

static inline
void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node)
{
	return lttng_kvmalloc_node(size, flags | __GFP_ZERO, node);
}

static inline
void *lttng_kvmalloc(unsigned long size, gfp_t flags)
{
	return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE);
}

static inline
void *lttng_kvzalloc(unsigned long size, gfp_t flags)
{
	return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE);
}

static inline
void lttng_kvfree(const void *addr)
{
	if (is_vmalloc_addr(addr)) {
		vfree(addr);
	} else {
		kfree(addr);
	}
}
#endif

#endif /* _LTTNG_WRAPPER_VMALLOC_H */
Commit	Line	Data
9f36eaed MJ	1	/* SPDX-License-Identifier: (GPL-2.0 or LGPL-2.1)
9f36eaed MJ	2	*
886d51a3	3	* wrapper/vmalloc.h
6d2a620c MD	4	*
	5	* wrapper around vmalloc_sync_all. Using KALLSYMS to get its address when
	6	* available, else we need to have a kernel that exports this function to GPL
	7	* modules.
	8	*
886d51a3	9	* Copyright (C) 2011-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6d2a620c MD	10	*/
6d2a620c MD	11
9f36eaed MJ	12	#ifndef _LTTNG_WRAPPER_VMALLOC_H
	13	#define _LTTNG_WRAPPER_VMALLOC_H
	14
b78104db	15	#include <lttng-kernel-version.h>
48f5e0b5	16	#include <linux/vmalloc.h>
01ab5113	17	#include <linux/mm.h>
48f5e0b5	18
6d2a620c MD	19	#ifdef CONFIG_KALLSYMS
	20
	21	#include <linux/kallsyms.h>
5a2f5e92	22	#include <wrapper/kallsyms.h>
5939591f	23	#include <lttng-kernel-version.h>
6d2a620c	24
b78104db	25	#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,8,0))
1bf0964a MJ	26
	27	/*
	28	* wrapper_vmalloc_sync_mappings was removed in v5.8, the vmalloc mappings
	29	* are now synchronized when they are created or torn down.
	30	*/
	31	static inline
	32	void wrapper_vmalloc_sync_mappings(void)
	33	{}
	34
b78104db	35	#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0) \
3dd99395	36	\|\| LTTNG_KERNEL_RANGE(5,5,12, 5,6,0) \
5939591f OP	37	\|\| LTTNG_KERNEL_RANGE(5,4,28, 5,5,0) \
	38	\|\| LTTNG_KERNEL_RANGE(5,2,37, 5,3,0) \
	39	\|\| LTTNG_KERNEL_RANGE(4,19,113, 4,20,0) \
	40	\|\| LTTNG_KERNEL_RANGE(4,14,175, 4,15,0) \
	41	\|\| LTTNG_KERNEL_RANGE(4,9,218, 4,10,0) \
46e6a937	42	\|\| LTTNG_KERNEL_RANGE(4,4,218, 4,5,0) \
4ad90183 SB	43	\|\| LTTNG_UBUNTU_KERNEL_RANGE(4,15,18,97, 4,16,0,0) \
4ad90183 SB	44	\|\| LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,48, 5,1,0,0) \
46e6a937 MJ	45	\|\| LTTNG_UBUNTU_KERNEL_RANGE(5,3,18,52, 5,4,0,0) \
46e6a937 MJ	46	\|\| LTTNG_RHEL_KERNEL_RANGE(4,18,0,240,0,0, 4,19,0,0,0,0))
da0fcb14 MD	47
	48	static inline
	49	void wrapper_vmalloc_sync_mappings(void)
	50	{
	51	void (*vmalloc_sync_mappings_sym)(void);
	52
	53	vmalloc_sync_mappings_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_mappings");
	54	if (vmalloc_sync_mappings_sym) {
	55	vmalloc_sync_mappings_sym();
	56	} else {
	57	#ifdef CONFIG_X86
	58	/*
	59	* Only x86 needs vmalloc_sync_mappings to make sure LTTng does not
	60	* trigger recursive page faults.
	61	*/
	62	printk_once(KERN_WARNING "LTTng: vmalloc_sync_mappings symbol lookup failed.\n");
	63	printk_once(KERN_WARNING "Page fault handler and NMI tracing might trigger faults.\n");
	64	#endif
	65	}
	66	}
	67
9e52289b MJ	68	/*
	69	* Canary function to check for 'vmalloc_sync_mappings()' at compile time.
	70	*
	71	* From 'include/linux/vmalloc.h':
	72	*
	73	* void vmalloc_sync_mappings(void);
	74	*/
	75	static inline
	76	void __canary__vmalloc_sync_mappings(void)
	77	{
	78	vmalloc_sync_mappings();
	79	}
	80
b78104db	81	#else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */
da0fcb14 MD	82
da0fcb14 MD	83	/*
9e52289b	84	* Map vmalloc_sync_mappings to vmalloc_sync_all() on kernels before 5.6.
da0fcb14	85	*/
6d2a620c	86	static inline
da0fcb14	87	void wrapper_vmalloc_sync_mappings(void)
6d2a620c MD	88	{
	89	void (*vmalloc_sync_all_sym)(void);
	90
c539a324	91	vmalloc_sync_all_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_all");
6d2a620c MD	92	if (vmalloc_sync_all_sym) {
	93	vmalloc_sync_all_sym();
	94	} else {
	95	#ifdef CONFIG_X86
	96	/*
	97	* Only x86 needs vmalloc_sync_all to make sure LTTng does not
	98	* trigger recursive page faults.
	99	*/
e36de50d MD	100	printk_once(KERN_WARNING "LTTng: vmalloc_sync_all symbol lookup failed.\n");
e36de50d MD	101	printk_once(KERN_WARNING "Page fault handler and NMI tracing might trigger faults.\n");
6d2a620c MD	102	#endif
	103	}
	104	}
da0fcb14	105
9e52289b MJ	106	/*
	107	* Canary function to check for 'vmalloc_sync_all()' at compile time.
	108	*
	109	* From 'include/linux/vmalloc.h':
	110	*
	111	* void vmalloc_sync_all(void);
	112	*/
	113	static inline
	114	void __canary__vmalloc_sync_all(void)
	115	{
	116	vmalloc_sync_all();
	117	}
	118
b78104db	119	#endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */
da0fcb14	120
02f0c95c	121	#else /* CONFIG_KALLSYMS */
6d2a620c	122
b78104db	123	#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,8,0))
02f0c95c MJ	124
	125	/*
	126	* wrapper_vmalloc_sync_mappings was removed in v5.8, the vmalloc mappings
	127	* are now synchronized when they are created or torn down.
	128	*/
	129	static inline
	130	void wrapper_vmalloc_sync_mappings(void)
	131	{}
	132
b78104db	133	#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0) \
3dd99395	134	\|\| LTTNG_KERNEL_RANGE(5,5,12, 5,6,0) \
5939591f OP	135	\|\| LTTNG_KERNEL_RANGE(5,4,28, 5,5,0) \
	136	\|\| LTTNG_KERNEL_RANGE(5,2,37, 5,3,0) \
	137	\|\| LTTNG_KERNEL_RANGE(4,19,113, 4,20,0) \
	138	\|\| LTTNG_KERNEL_RANGE(4,14,175, 4,15,0) \
	139	\|\| LTTNG_KERNEL_RANGE(4,9,218, 4,10,0) \
4ad90183 SB	140	\|\| LTTNG_KERNEL_RANGE(4,4,218, 4,5,0)) \
	141	\|\| LTTNG_UBUNTU_KERNEL_RANGE(4,15,18,97, 4,18,0,0) \
	142	\|\| LTTNG_UBUNTU_KERNEL_RANGE(5,0,21,48, 5,1,0,0) \
	143	\|\| LTTNG_UBUNTU_KERNEL_RANGE(5,3,18,52, 5,4,0,0)
da0fcb14	144
6d2a620c	145	static inline
da0fcb14 MD	146	void wrapper_vmalloc_sync_mappings(void)
	147	{
	148	return vmalloc_sync_mappings();
	149	}
	150
b78104db	151	#else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */
da0fcb14 MD	152
	153	static inline
	154	void wrapper_vmalloc_sync_mappings(void)
6d2a620c MD	155	{
	156	return vmalloc_sync_all();
	157	}
da0fcb14	158
b78104db	159	#endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0)) */
da0fcb14	160
6d2a620c	161	#endif
b13f3ebe	162
b78104db	163	#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,12,0))
48f5e0b5 MJ	164	static inline
	165	void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node)
	166	{
	167	void *ret;
	168
	169	ret = kvmalloc_node(size, flags, node);
	170	if (is_vmalloc_addr(ret)) {
	171	/*
	172	* Make sure we don't trigger recursive page faults in the
	173	* tracing fast path.
	174	*/
da0fcb14	175	wrapper_vmalloc_sync_mappings();
48f5e0b5 MJ	176	}
	177	return ret;
	178	}
	179
	180	static inline
	181	void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node)
	182	{
	183	return lttng_kvmalloc_node(size, flags \| __GFP_ZERO, node);
	184	}
	185
	186	static inline
	187	void *lttng_kvmalloc(unsigned long size, gfp_t flags)
	188	{
	189	return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE);
	190	}
	191
	192	static inline
	193	void *lttng_kvzalloc(unsigned long size, gfp_t flags)
	194	{
	195	return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE);
	196	}
	197
	198	static inline
	199	void lttng_kvfree(const void *addr)
	200	{
	201	kvfree(addr);
	202	}
	203
	204	#else
	205
	206	#include <linux/slab.h>
48f5e0b5	207
20eb87c9 MD	208	static inline
	209	void print_vmalloc_node_range_warning(void)
	210	{
	211	printk_once(KERN_WARNING "LTTng: __vmalloc_node_range symbol lookup failed.\n");
	212	printk_once(KERN_WARNING "Tracer performance will be degraded on NUMA systems.\n");
	213	printk_once(KERN_WARNING "Please rebuild your kernel with CONFIG_KALLSYMS enabled.\n");
	214	}
	215
b78104db	216	#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,0,0))
24b1dd94	217
48f5e0b5 MJ	218	/*
	219	* kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
	220	*/
	221	static inline
20eb87c9 MD	222	void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	223	unsigned long start, unsigned long end, gfp_t gfp_mask,
	224	pgprot_t prot, unsigned long vm_flags, int node,
	225	const void *caller)
48f5e0b5	226	{
48f5e0b5 MJ	227	#ifdef CONFIG_KALLSYMS
48f5e0b5 MJ	228	/*
20eb87c9	229	* If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
48f5e0b5	230	*/
20eb87c9 MD	231	void (lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
	232	unsigned long start, unsigned long end, gfp_t gfp_mask,
	233	pgprot_t prot, unsigned long vm_flags, int node,
	234	const void *caller);
	235
	236	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	237	if (lttng__vmalloc_node_range)
	238	return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	239	vm_flags, node, caller);
48f5e0b5	240	#endif
20eb87c9 MD	241	if (node != NUMA_NO_NODE)
	242	print_vmalloc_node_range_warning();
	243	return __vmalloc(size, gfp_mask, prot);
48f5e0b5	244	}
9e52289b MJ	245
	246	/*
	247	* Canary function to check for '__vmalloc_node_range()' at compile time.
	248	*
	249	* From 'include/linux/vmalloc.h':
	250	*
	251	* extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
	252	* unsigned long start, unsigned long end, gfp_t gfp_mask,
	253	* pgprot_t prot, unsigned long vm_flags, int node,
	254	* const void *caller);
	255	*/
	256	static inline
	257	void *__canary____lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	258	unsigned long start, unsigned long end, gfp_t gfp_mask,
	259	pgprot_t prot, unsigned long vm_flags, int node,
	260	const void *caller)
	261	{
	262	return __vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	263	vm_flags, node, caller);
	264	}
48f5e0b5	265
b78104db	266	#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,6,0))
24b1dd94 MJ	267
	268	/*
	269	* kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
	270	*/
	271	static inline
	272	void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	273	unsigned long start, unsigned long end, gfp_t gfp_mask,
	274	pgprot_t prot, unsigned long vm_flags, int node,
	275	const void *caller)
	276	{
	277	#ifdef CONFIG_KALLSYMS
	278	/*
	279	* If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
	280	*/
	281	void (lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
	282	unsigned long start, unsigned long end, gfp_t gfp_mask,
	283	pgprot_t prot, int node, const void *caller);
	284
	285	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	286	if (lttng__vmalloc_node_range)
	287	return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	288	node, caller);
	289	#endif
	290	if (node != NUMA_NO_NODE)
	291	print_vmalloc_node_range_warning();
	292	return __vmalloc(size, gfp_mask, prot);
	293	}
	294
	295	/*
	296	* Canary function to check for '__vmalloc_node_range()' at compile time.
	297	*
	298	* From 'include/linux/vmalloc.h':
	299	*
	300	* extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
	301	* unsigned long start, unsigned long end, gfp_t gfp_mask,
	302	* pgprot_t prot, unsigned long vm_flags, int node,
	303	* const void *caller);
	304	*/
	305	static inline
	306	void *__canary____lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	307	unsigned long start, unsigned long end, gfp_t gfp_mask,
	308	pgprot_t prot, int node, const void *caller)
	309	{
	310	return __vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	311	node, caller);
	312	}
	313
b78104db	314	#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(2,6,38))
e1c79262 MJ	315
	316	/*
	317	* kallsyms wrapper of __vmalloc_node with a fallback to kmalloc_node.
	318	*/
	319	static inline
	320	void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	321	unsigned long start, unsigned long end, gfp_t gfp_mask,
	322	pgprot_t prot, unsigned long vm_flags, int node,
	323	void *caller)
	324	{
	325	#ifdef CONFIG_KALLSYMS
	326	/*
	327	* If we have KALLSYMS, get * __vmalloc_node_range which is not exported.
	328	*/
	329	void (lttng__vmalloc_node_range)(unsigned long size, unsigned long align,
	330	unsigned long start, unsigned long end, gfp_t gfp_mask,
	331	pgprot_t prot, int node, void *caller);
	332
	333	lttng__vmalloc_node_range = (void *) kallsyms_lookup_funcptr("__vmalloc_node_range");
	334	if (lttng__vmalloc_node_range)
	335	return lttng__vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	336	node, caller);
	337	#endif
	338	if (node != NUMA_NO_NODE)
	339	print_vmalloc_node_range_warning();
	340	return __vmalloc(size, gfp_mask, prot);
	341	}
	342
	343	/*
	344	* Canary function to check for '__vmalloc_node_range()' at compile time.
	345	*
	346	* From 'include/linux/vmalloc.h':
	347	*
	348	* extern void *__vmalloc_node_range(unsigned long size, unsigned long align,
	349	* unsigned long start, unsigned long end, gfp_t gfp_mask,
	350	* pgprot_t prot, unsigned long vm_flags, int node,
	351	* void *caller);
	352	*/
	353	static inline
	354	void *__canary____lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	355	unsigned long start, unsigned long end, gfp_t gfp_mask,
	356	pgprot_t prot, int node, void *caller)
	357	{
	358	return __vmalloc_node_range(size, align, start, end, gfp_mask, prot,
	359	node, caller);
	360	}
	361
b78104db	362	#else /* (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(2,6,38)) */
4b1927af MJ	363
	364	/*
	365	* Basic fallback for kernel prior to 2.6.38 without __vmalloc_node_range()
	366	*/
	367	static inline
	368	void *__lttng_vmalloc_node_range(unsigned long size, unsigned long align,
	369	unsigned long start, unsigned long end, gfp_t gfp_mask,
	370	pgprot_t prot, unsigned long vm_flags, int node,
	371	void *caller)
	372	{
	373	return __vmalloc(size, gfp_mask, prot);
	374	}
	375
24b1dd94 MJ	376	#endif
24b1dd94 MJ	377
48f5e0b5 MJ	378	/**
	379	* lttng_kvmalloc_node - attempt to allocate physically contiguous memory, but upon
	380	* failure, fall back to non-contiguous (vmalloc) allocation.
	381	* @size: size of the request.
	382	* @flags: gfp mask for the allocation - must be compatible with GFP_KERNEL.
	383	*
	384	* Uses kmalloc to get the memory but if the allocation fails then falls back
	385	* to the vmalloc allocator. Use lttng_kvfree to free the memory.
	386	*
	387	* Reclaim modifiers - __GFP_NORETRY, __GFP_REPEAT and __GFP_NOFAIL are not supported
	388	*/
	389	static inline
	390	void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node)
	391	{
	392	void *ret;
	393
	394	/*
	395	* vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables)
	396	* so the given set of flags has to be compatible.
	397	*/
	398	WARN_ON_ONCE((flags & GFP_KERNEL) != GFP_KERNEL);
	399
	400	/*
	401	* If the allocation fits in a single page, do not fallback.
	402	*/
	403	if (size <= PAGE_SIZE) {
	404	return kmalloc_node(size, flags, node);
	405	}
	406
	407	/*
	408	* Make sure that larger requests are not too disruptive - no OOM
	409	* killer and no allocation failure warnings as we have a fallback
	410	*/
	411	ret = kmalloc_node(size, flags \| __GFP_NOWARN \| __GFP_NORETRY, node);
	412	if (!ret) {
20eb87c9 MD	413	ret = __lttng_vmalloc_node_range(size, 1,
	414	VMALLOC_START, VMALLOC_END,
	415	flags \| __GFP_HIGHMEM, PAGE_KERNEL, 0,
	416	node, __builtin_return_address(0));
48f5e0b5 MJ	417	/*
	418	* Make sure we don't trigger recursive page faults in the
	419	* tracing fast path.
	420	*/
c1ba644f	421	wrapper_vmalloc_sync_mappings();
48f5e0b5 MJ	422	}
	423	return ret;
	424	}
	425
	426	static inline
	427	void *lttng_kvzalloc_node(unsigned long size, gfp_t flags, int node)
	428	{
	429	return lttng_kvmalloc_node(size, flags \| __GFP_ZERO, node);
	430	}
	431
	432	static inline
	433	void *lttng_kvmalloc(unsigned long size, gfp_t flags)
	434	{
	435	return lttng_kvmalloc_node(size, flags, NUMA_NO_NODE);
	436	}
	437
	438	static inline
	439	void *lttng_kvzalloc(unsigned long size, gfp_t flags)
	440	{
	441	return lttng_kvzalloc_node(size, flags, NUMA_NO_NODE);
	442	}
	443
	444	static inline
	445	void lttng_kvfree(const void *addr)
	446	{
	447	if (is_vmalloc_addr(addr)) {
	448	vfree(addr);
	449	} else {
	450	kfree(addr);
	451	}
	452	}
	453	#endif
	454
a90917c3	455	#endif /* _LTTNG_WRAPPER_VMALLOC_H */