From 263b6c88138c3354d63dba3c70a965de94becd22 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Tue, 5 May 2020 13:38:31 -0400 Subject: [PATCH] Update for kernel 5.7: use vmalloc_sync_mappings on kernels >= 5.7 Signed-off-by: Mathieu Desnoyers --- lib/ringbuffer/ring_buffer_backend.c | 4 +-- lttng-abi.c | 4 +-- lttng-context-callstack.c | 2 +- lttng-context-cgroup-ns.c | 2 +- lttng-context-cpu-id.c | 2 +- lttng-context-egid.c | 2 +- lttng-context-euid.c | 2 +- lttng-context-gid.c | 2 +- lttng-context-hostname.c | 2 +- lttng-context-interruptible.c | 2 +- lttng-context-ipc-ns.c | 2 +- lttng-context-migratable.c | 2 +- lttng-context-mnt-ns.c | 2 +- lttng-context-need-reschedule.c | 2 +- lttng-context-net-ns.c | 2 +- lttng-context-nice.c | 2 +- lttng-context-perf-counters.c | 2 +- lttng-context-pid-ns.c | 2 +- lttng-context-pid.c | 2 +- lttng-context-ppid.c | 2 +- lttng-context-preemptible.c | 2 +- lttng-context-prio.c | 2 +- lttng-context-procname.c | 2 +- lttng-context-sgid.c | 2 +- lttng-context-suid.c | 2 +- lttng-context-tid.c | 2 +- lttng-context-uid.c | 2 +- lttng-context-user-ns.c | 2 +- lttng-context-uts-ns.c | 2 +- lttng-context-vegid.c | 2 +- lttng-context-veuid.c | 2 +- lttng-context-vgid.c | 2 +- lttng-context-vpid.c | 2 +- lttng-context-vppid.c | 2 +- lttng-context-vsgid.c | 2 +- lttng-context-vsuid.c | 2 +- lttng-context-vtid.c | 2 +- lttng-context-vuid.c | 2 +- lttng-context.c | 2 +- lttng-events.c | 10 +++--- lttng-ring-buffer-client.h | 4 +-- lttng-ring-buffer-metadata-client.h | 4 +-- lttng-syscalls.c | 2 +- probes/lttng-kprobes.c | 2 +- probes/lttng-kretprobes.c | 2 +- probes/lttng-tracepoint-event-impl.h | 4 +-- probes/lttng-uprobes.c | 2 +- probes/lttng.c | 2 +- tests/probes/lttng-test.c | 2 +- wrapper/vmalloc.h | 49 ++++++++++++++++++++++++++-- 50 files changed, 104 insertions(+), 61 deletions(-) diff --git a/lib/ringbuffer/ring_buffer_backend.c b/lib/ringbuffer/ring_buffer_backend.c index 468b0e9b..2a47948e 100644 --- a/lib/ringbuffer/ring_buffer_backend.c +++ b/lib/ringbuffer/ring_buffer_backend.c @@ -17,7 +17,7 @@ #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include #include @@ -156,7 +156,7 @@ int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config *config * If kmalloc ever uses vmalloc underneath, make sure the buffer pages * will not fault. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); wrapper_clear_current_oom_origin(); vfree(pages); return 0; diff --git a/lttng-abi.c b/lttng-abi.c index 590945c4..4051264e 100644 --- a/lttng-abi.c +++ b/lttng-abi.c @@ -30,7 +30,7 @@ #include #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include #include @@ -1938,7 +1938,7 @@ int __init lttng_abi_init(void) { int ret = 0; - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); lttng_clock_ref(); ret = lttng_tp_mempool_init(); diff --git a/lttng-context-callstack.c b/lttng-context-callstack.c index 317efdf4..1e791f9c 100644 --- a/lttng-context-callstack.c +++ b/lttng-context-callstack.c @@ -152,7 +152,7 @@ int __lttng_add_callstack_generic(struct lttng_ctx **ctx, sequence_field->priv = fdata; sequence_field->destroy = lttng_callstack_sequence_destroy; - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; error_create: diff --git a/lttng-context-cgroup-ns.c b/lttng-context-cgroup-ns.c index a7f4e80f..ced8c124 100644 --- a/lttng-context-cgroup-ns.c +++ b/lttng-context-cgroup-ns.c @@ -97,7 +97,7 @@ int lttng_add_cgroup_ns_to_ctx(struct lttng_ctx **ctx) field->record = cgroup_ns_record; field->get_value = cgroup_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_cgroup_ns_to_ctx); diff --git a/lttng-context-cpu-id.c b/lttng-context-cpu-id.c index d3d55b88..20c186a4 100644 --- a/lttng-context-cpu-id.c +++ b/lttng-context-cpu-id.c @@ -68,7 +68,7 @@ int lttng_add_cpu_id_to_ctx(struct lttng_ctx **ctx) field->record = cpu_id_record; field->get_value = cpu_id_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_cpu_id_to_ctx); diff --git a/lttng-context-egid.c b/lttng-context-egid.c index fcd01461..6ec39224 100644 --- a/lttng-context-egid.c +++ b/lttng-context-egid.c @@ -70,7 +70,7 @@ int lttng_add_egid_to_ctx(struct lttng_ctx **ctx) field->record = egid_record; field->get_value = egid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_egid_to_ctx); diff --git a/lttng-context-euid.c b/lttng-context-euid.c index b4b72184..2895cef9 100644 --- a/lttng-context-euid.c +++ b/lttng-context-euid.c @@ -70,7 +70,7 @@ int lttng_add_euid_to_ctx(struct lttng_ctx **ctx) field->record = euid_record; field->get_value = euid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_euid_to_ctx); diff --git a/lttng-context-gid.c b/lttng-context-gid.c index 586934ed..ebdbc737 100644 --- a/lttng-context-gid.c +++ b/lttng-context-gid.c @@ -70,7 +70,7 @@ int lttng_add_gid_to_ctx(struct lttng_ctx **ctx) field->record = gid_record; field->get_value = gid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_gid_to_ctx); diff --git a/lttng-context-hostname.c b/lttng-context-hostname.c index 0bbb1c78..a16da350 100644 --- a/lttng-context-hostname.c +++ b/lttng-context-hostname.c @@ -100,7 +100,7 @@ int lttng_add_hostname_to_ctx(struct lttng_ctx **ctx) field->record = hostname_record; field->get_value = hostname_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_hostname_to_ctx); diff --git a/lttng-context-interruptible.c b/lttng-context-interruptible.c index c07cba18..c9faaa7f 100644 --- a/lttng-context-interruptible.c +++ b/lttng-context-interruptible.c @@ -75,7 +75,7 @@ int lttng_add_interruptible_to_ctx(struct lttng_ctx **ctx) field->record = interruptible_record; field->get_value = interruptible_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_interruptible_to_ctx); diff --git a/lttng-context-ipc-ns.c b/lttng-context-ipc-ns.c index cccd9766..7171928c 100644 --- a/lttng-context-ipc-ns.c +++ b/lttng-context-ipc-ns.c @@ -96,7 +96,7 @@ int lttng_add_ipc_ns_to_ctx(struct lttng_ctx **ctx) field->record = ipc_ns_record; field->get_value = ipc_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_ipc_ns_to_ctx); diff --git a/lttng-context-migratable.c b/lttng-context-migratable.c index 1a0dc4a1..08a55f4b 100644 --- a/lttng-context-migratable.c +++ b/lttng-context-migratable.c @@ -68,7 +68,7 @@ int lttng_add_migratable_to_ctx(struct lttng_ctx **ctx) field->record = migratable_record; field->get_value = migratable_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_migratable_to_ctx); diff --git a/lttng-context-mnt-ns.c b/lttng-context-mnt-ns.c index 02940139..539b043b 100644 --- a/lttng-context-mnt-ns.c +++ b/lttng-context-mnt-ns.c @@ -98,7 +98,7 @@ int lttng_add_mnt_ns_to_ctx(struct lttng_ctx **ctx) field->record = mnt_ns_record; field->get_value = mnt_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_mnt_ns_to_ctx); diff --git a/lttng-context-need-reschedule.c b/lttng-context-need-reschedule.c index 9a51f0c6..dc404076 100644 --- a/lttng-context-need-reschedule.c +++ b/lttng-context-need-reschedule.c @@ -68,7 +68,7 @@ int lttng_add_need_reschedule_to_ctx(struct lttng_ctx **ctx) field->record = need_reschedule_record; field->get_value = need_reschedule_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_need_reschedule_to_ctx); diff --git a/lttng-context-net-ns.c b/lttng-context-net-ns.c index 4584991f..06088a51 100644 --- a/lttng-context-net-ns.c +++ b/lttng-context-net-ns.c @@ -97,7 +97,7 @@ int lttng_add_net_ns_to_ctx(struct lttng_ctx **ctx) field->record = net_ns_record; field->get_value = net_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_net_ns_to_ctx); diff --git a/lttng-context-nice.c b/lttng-context-nice.c index 89283d0f..271f2049 100644 --- a/lttng-context-nice.c +++ b/lttng-context-nice.c @@ -68,7 +68,7 @@ int lttng_add_nice_to_ctx(struct lttng_ctx **ctx) field->record = nice_record; field->get_value = nice_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_nice_to_ctx); diff --git a/lttng-context-perf-counters.c b/lttng-context-perf-counters.c index 1c5ab9fb..be26e495 100644 --- a/lttng-context-perf-counters.c +++ b/lttng-context-perf-counters.c @@ -321,7 +321,7 @@ int lttng_add_perf_counter_to_ctx(uint32_t type, field->u.perf_counter = perf_field; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) diff --git a/lttng-context-pid-ns.c b/lttng-context-pid-ns.c index e4d3a3a3..90c21dea 100644 --- a/lttng-context-pid-ns.c +++ b/lttng-context-pid-ns.c @@ -98,7 +98,7 @@ int lttng_add_pid_ns_to_ctx(struct lttng_ctx **ctx) field->record = pid_ns_record; field->get_value = pid_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_pid_ns_to_ctx); diff --git a/lttng-context-pid.c b/lttng-context-pid.c index 3cdc5128..b8155ea8 100644 --- a/lttng-context-pid.c +++ b/lttng-context-pid.c @@ -68,7 +68,7 @@ int lttng_add_pid_to_ctx(struct lttng_ctx **ctx) field->record = pid_record; field->get_value = pid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_pid_to_ctx); diff --git a/lttng-context-ppid.c b/lttng-context-ppid.c index 8935fad0..13a521bd 100644 --- a/lttng-context-ppid.c +++ b/lttng-context-ppid.c @@ -90,7 +90,7 @@ int lttng_add_ppid_to_ctx(struct lttng_ctx **ctx) field->record = ppid_record; field->get_value = ppid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_ppid_to_ctx); diff --git a/lttng-context-preemptible.c b/lttng-context-preemptible.c index 6a40f03a..95a38547 100644 --- a/lttng-context-preemptible.c +++ b/lttng-context-preemptible.c @@ -86,7 +86,7 @@ int lttng_add_preemptible_to_ctx(struct lttng_ctx **ctx) field->record = preemptible_record; field->get_value = preemptible_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_preemptible_to_ctx); diff --git a/lttng-context-prio.c b/lttng-context-prio.c index 9cd843d5..6441b5f1 100644 --- a/lttng-context-prio.c +++ b/lttng-context-prio.c @@ -89,7 +89,7 @@ int lttng_add_prio_to_ctx(struct lttng_ctx **ctx) field->record = prio_record; field->get_value = prio_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_prio_to_ctx); diff --git a/lttng-context-procname.c b/lttng-context-procname.c index 8e4bbf97..fc40f6cf 100644 --- a/lttng-context-procname.c +++ b/lttng-context-procname.c @@ -71,7 +71,7 @@ int lttng_add_procname_to_ctx(struct lttng_ctx **ctx) field->record = procname_record; field->get_value = procname_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_procname_to_ctx); diff --git a/lttng-context-sgid.c b/lttng-context-sgid.c index 4f58a14f..81fa3f10 100644 --- a/lttng-context-sgid.c +++ b/lttng-context-sgid.c @@ -70,7 +70,7 @@ int lttng_add_sgid_to_ctx(struct lttng_ctx **ctx) field->record = sgid_record; field->get_value = sgid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_sgid_to_ctx); diff --git a/lttng-context-suid.c b/lttng-context-suid.c index 4b8477e5..5f1ce3cd 100644 --- a/lttng-context-suid.c +++ b/lttng-context-suid.c @@ -70,7 +70,7 @@ int lttng_add_suid_to_ctx(struct lttng_ctx **ctx) field->record = suid_record; field->get_value = suid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_suid_to_ctx); diff --git a/lttng-context-tid.c b/lttng-context-tid.c index 5f8e68fa..efeca439 100644 --- a/lttng-context-tid.c +++ b/lttng-context-tid.c @@ -71,7 +71,7 @@ int lttng_add_tid_to_ctx(struct lttng_ctx **ctx) field->record = tid_record; field->get_value = tid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_tid_to_ctx); diff --git a/lttng-context-uid.c b/lttng-context-uid.c index c7a2bb18..3942e94a 100644 --- a/lttng-context-uid.c +++ b/lttng-context-uid.c @@ -70,7 +70,7 @@ int lttng_add_uid_to_ctx(struct lttng_ctx **ctx) field->record = uid_record; field->get_value = uid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_uid_to_ctx); diff --git a/lttng-context-user-ns.c b/lttng-context-user-ns.c index ff56f551..45db066d 100644 --- a/lttng-context-user-ns.c +++ b/lttng-context-user-ns.c @@ -82,7 +82,7 @@ int lttng_add_user_ns_to_ctx(struct lttng_ctx **ctx) field->record = user_ns_record; field->get_value = user_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_user_ns_to_ctx); diff --git a/lttng-context-uts-ns.c b/lttng-context-uts-ns.c index 67c42764..26bdfa83 100644 --- a/lttng-context-uts-ns.c +++ b/lttng-context-uts-ns.c @@ -96,7 +96,7 @@ int lttng_add_uts_ns_to_ctx(struct lttng_ctx **ctx) field->record = uts_ns_record; field->get_value = uts_ns_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_uts_ns_to_ctx); diff --git a/lttng-context-vegid.c b/lttng-context-vegid.c index 501b01e6..029658b6 100644 --- a/lttng-context-vegid.c +++ b/lttng-context-vegid.c @@ -70,7 +70,7 @@ int lttng_add_vegid_to_ctx(struct lttng_ctx **ctx) field->record = vegid_record; field->get_value = vegid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vegid_to_ctx); diff --git a/lttng-context-veuid.c b/lttng-context-veuid.c index a8810a77..211a1651 100644 --- a/lttng-context-veuid.c +++ b/lttng-context-veuid.c @@ -70,7 +70,7 @@ int lttng_add_veuid_to_ctx(struct lttng_ctx **ctx) field->record = veuid_record; field->get_value = veuid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_veuid_to_ctx); diff --git a/lttng-context-vgid.c b/lttng-context-vgid.c index 92ac567b..a7b70d84 100644 --- a/lttng-context-vgid.c +++ b/lttng-context-vgid.c @@ -70,7 +70,7 @@ int lttng_add_vgid_to_ctx(struct lttng_ctx **ctx) field->record = vgid_record; field->get_value = vgid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vgid_to_ctx); diff --git a/lttng-context-vpid.c b/lttng-context-vpid.c index 31a2c15e..544215cb 100644 --- a/lttng-context-vpid.c +++ b/lttng-context-vpid.c @@ -83,7 +83,7 @@ int lttng_add_vpid_to_ctx(struct lttng_ctx **ctx) field->record = vpid_record; field->get_value = vpid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vpid_to_ctx); diff --git a/lttng-context-vppid.c b/lttng-context-vppid.c index 43bfcc12..87dd3d0d 100644 --- a/lttng-context-vppid.c +++ b/lttng-context-vppid.c @@ -112,7 +112,7 @@ int lttng_add_vppid_to_ctx(struct lttng_ctx **ctx) field->record = vppid_record; field->get_value = vppid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vppid_to_ctx); diff --git a/lttng-context-vsgid.c b/lttng-context-vsgid.c index 20987dde..b82e4eab 100644 --- a/lttng-context-vsgid.c +++ b/lttng-context-vsgid.c @@ -70,7 +70,7 @@ int lttng_add_vsgid_to_ctx(struct lttng_ctx **ctx) field->record = vsgid_record; field->get_value = vsgid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vsgid_to_ctx); diff --git a/lttng-context-vsuid.c b/lttng-context-vsuid.c index 7cefd535..71a40d59 100644 --- a/lttng-context-vsuid.c +++ b/lttng-context-vsuid.c @@ -70,7 +70,7 @@ int lttng_add_vsuid_to_ctx(struct lttng_ctx **ctx) field->record = vsuid_record; field->get_value = vsuid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vsuid_to_ctx); diff --git a/lttng-context-vtid.c b/lttng-context-vtid.c index a7da7d2e..a2d90bb9 100644 --- a/lttng-context-vtid.c +++ b/lttng-context-vtid.c @@ -83,7 +83,7 @@ int lttng_add_vtid_to_ctx(struct lttng_ctx **ctx) field->record = vtid_record; field->get_value = vtid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vtid_to_ctx); diff --git a/lttng-context-vuid.c b/lttng-context-vuid.c index 427ada71..dd8eab89 100644 --- a/lttng-context-vuid.c +++ b/lttng-context-vuid.c @@ -70,7 +70,7 @@ int lttng_add_vuid_to_ctx(struct lttng_ctx **ctx) field->record = vuid_record; field->get_value = vuid_get_value; lttng_context_update(*ctx); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return 0; } EXPORT_SYMBOL_GPL(lttng_add_vuid_to_ctx); diff --git a/lttng-context.c b/lttng-context.c index 54fa4478..b4a60038 100644 --- a/lttng-context.c +++ b/lttng-context.c @@ -11,7 +11,7 @@ #include #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include diff --git a/lttng-events.c b/lttng-events.c index da949e97..49a181b6 100644 --- a/lttng-events.c +++ b/lttng-events.c @@ -30,7 +30,7 @@ #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include #include @@ -2795,9 +2795,9 @@ end: * Registers a transport which can be used as output to extract the data out of * LTTng. The module calling this registration function must ensure that no * trap-inducing code will be executed by the transport functions. E.g. - * vmalloc_sync_all() must be called between a vmalloc and the moment the memory + * vmalloc_sync_mappings() must be called between a vmalloc and the moment the memory * is made visible to the transport function. This registration acts as a - * vmalloc_sync_all. Therefore, only if the module allocates virtual memory + * vmalloc_sync_mappings. Therefore, only if the module allocates virtual memory * after its registration must it synchronize the TLBs. */ void lttng_transport_register(struct lttng_transport *transport) @@ -2805,9 +2805,9 @@ void lttng_transport_register(struct lttng_transport *transport) /* * Make sure no page fault can be triggered by the module about to be * registered. We deal with this here so we don't have to call - * vmalloc_sync_all() in each module's init. + * vmalloc_sync_mappings() in each module's init. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); mutex_lock(&sessions_mutex); list_add_tail(&transport->node, <tng_transport_list); diff --git a/lttng-ring-buffer-client.h b/lttng-ring-buffer-client.h index f11c606b..ecdc4adf 100644 --- a/lttng-ring-buffer-client.h +++ b/lttng-ring-buffer-client.h @@ -10,7 +10,7 @@ #include #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include #include @@ -766,7 +766,7 @@ static int __init lttng_ring_buffer_client_init(void) * This vmalloc sync all also takes care of the lib ring buffer * vmalloc'd module pages when it is built as a module into LTTng. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); lttng_transport_register(<tng_relay_transport); return 0; } diff --git a/lttng-ring-buffer-metadata-client.h b/lttng-ring-buffer-metadata-client.h index 4c102975..fb71ad22 100644 --- a/lttng-ring-buffer-metadata-client.h +++ b/lttng-ring-buffer-metadata-client.h @@ -9,7 +9,7 @@ #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include @@ -427,7 +427,7 @@ static int __init lttng_ring_buffer_client_init(void) * This vmalloc sync all also takes care of the lib ring buffer * vmalloc'd module pages when it is built as a module into LTTng. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); lttng_transport_register(<tng_relay_transport); return 0; } diff --git a/lttng-syscalls.c b/lttng-syscalls.c index a43c475e..7720f4ea 100644 --- a/lttng-syscalls.c +++ b/lttng-syscalls.c @@ -761,7 +761,7 @@ int lttng_syscalls_register(struct lttng_channel *chan, void *filter) struct lttng_kernel_event ev; int ret; - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); if (!chan->sc_table) { /* create syscall table mapping syscall to events */ diff --git a/probes/lttng-kprobes.c b/probes/lttng-kprobes.c index 40fe0f68..d0a48d28 100644 --- a/probes/lttng-kprobes.c +++ b/probes/lttng-kprobes.c @@ -132,7 +132,7 @@ int lttng_kprobes_register(const char *name, * Well.. kprobes itself puts the page fault handler on the blacklist, * but we can never be too careful. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); ret = register_kprobe(&event->u.kprobe.kp); if (ret) diff --git a/probes/lttng-kretprobes.c b/probes/lttng-kretprobes.c index 40938f2b..133158fe 100644 --- a/probes/lttng-kretprobes.c +++ b/probes/lttng-kretprobes.c @@ -221,7 +221,7 @@ int lttng_kretprobes_register(const char *name, * Well.. kprobes itself puts the page fault handler on the blacklist, * but we can never be too careful. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); ret = register_kretprobe(<tng_krp->krp); if (ret) diff --git a/probes/lttng-tracepoint-event-impl.h b/probes/lttng-tracepoint-event-impl.h index 5f77bf4d..48639152 100644 --- a/probes/lttng-tracepoint-event-impl.h +++ b/probes/lttng-tracepoint-event-impl.h @@ -15,7 +15,7 @@ #include #include #include -#include /* for wrapper_vmalloc_sync_all() */ +#include /* for wrapper_vmalloc_sync_mappings() */ #include #include #include @@ -1414,7 +1414,7 @@ static __used struct lttng_probe_desc TP_ID(__probe_desc___, TRACE_SYSTEM) = { #ifndef TP_MODULE_NOINIT static int TP_ID(__lttng_events_init__, TRACE_SYSTEM)(void) { - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); return lttng_probe_register(&TP_ID(__probe_desc___, TRACE_SYSTEM)); } diff --git a/probes/lttng-uprobes.c b/probes/lttng-uprobes.c index 175c0dd9..82bd4fd4 100644 --- a/probes/lttng-uprobes.c +++ b/probes/lttng-uprobes.c @@ -161,7 +161,7 @@ int lttng_uprobes_add_callsite(struct lttng_event *event, } /* Ensure the memory we just allocated don't trigger page faults. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); uprobe_handler->event = event; uprobe_handler->up_consumer.handler = lttng_uprobes_handler_pre; diff --git a/probes/lttng.c b/probes/lttng.c index 160e0bcd..e571ceea 100644 --- a/probes/lttng.c +++ b/probes/lttng.c @@ -116,7 +116,7 @@ int __init lttng_logger_init(void) { int ret = 0; - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); /* /dev/lttng-logger */ ret = misc_register(&logger_dev); diff --git a/tests/probes/lttng-test.c b/tests/probes/lttng-test.c index cfdacc94..79743b80 100644 --- a/tests/probes/lttng-test.c +++ b/tests/probes/lttng-test.c @@ -97,7 +97,7 @@ int __init lttng_test_init(void) int ret = 0; (void) wrapper_lttng_fixup_sig(THIS_MODULE); - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); lttng_test_filter_event_dentry = proc_create_data(LTTNG_TEST_FILTER_EVENT_FILE, S_IRUGO | S_IWUGO, NULL, diff --git a/wrapper/vmalloc.h b/wrapper/vmalloc.h index 80deacfe..4000fc9c 100644 --- a/wrapper/vmalloc.h +++ b/wrapper/vmalloc.h @@ -21,8 +21,35 @@ #include #include +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)) + +static inline +void wrapper_vmalloc_sync_mappings(void) +{ + void (*vmalloc_sync_mappings_sym)(void); + + vmalloc_sync_mappings_sym = (void *) kallsyms_lookup_funcptr("vmalloc_sync_mappings"); + if (vmalloc_sync_mappings_sym) { + vmalloc_sync_mappings_sym(); + } else { +#ifdef CONFIG_X86 + /* + * Only x86 needs vmalloc_sync_mappings to make sure LTTng does not + * trigger recursive page faults. + */ + printk_once(KERN_WARNING "LTTng: vmalloc_sync_mappings symbol lookup failed.\n"); + printk_once(KERN_WARNING "Page fault handler and NMI tracing might trigger faults.\n"); +#endif + } +} + +#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)) */ + +/* + * Map vmalloc_sync_mappings to vmalloc_sync_all() on kernels before 5.7. + */ static inline -void wrapper_vmalloc_sync_all(void) +void wrapper_vmalloc_sync_mappings(void) { void (*vmalloc_sync_all_sym)(void); @@ -40,13 +67,29 @@ void wrapper_vmalloc_sync_all(void) #endif } } + +#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)) */ + #else +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)) + +static inline +void wrapper_vmalloc_sync_mappings(void) +{ + return vmalloc_sync_mappings(); +} + +#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)) */ + static inline -void wrapper_vmalloc_sync_all(void) +void wrapper_vmalloc_sync_mappings(void) { return vmalloc_sync_all(); } + +#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)) */ + #endif #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)) @@ -61,7 +104,7 @@ void *lttng_kvmalloc_node(unsigned long size, gfp_t flags, int node) * Make sure we don't trigger recursive page faults in the * tracing fast path. */ - wrapper_vmalloc_sync_all(); + wrapper_vmalloc_sync_mappings(); } return ret; } -- 2.34.1