From 6dffa64f09899cc452e4824eabb1d15ddd9ab155 Mon Sep 17 00:00:00 2001 From: Francis Deslauriers Date: Mon, 12 Mar 2018 16:13:32 -0400 Subject: [PATCH] jjb: lava: Reimplement kprobe fuzzing job This commits reimplements the kprobe fuzzing job in Python. The Lava job is split into two stages. The first stage generates the list of instrumentation points to try and saves it to the Lava job. The second stage, reads this file and perform the fuzzing. To save the instrumentation points file, we need to attach it to the job from a Lava test shell that doesn't crash since that would prevent the file from being saved. Because the fuzzing stage is likely to crash, we need to do the generation and saving in a different stage. In the past, we only fuzzed kallsyms symbols. Now we also fuzz random offsets from these symbols and random addresses in the entire address range. We now save the list of instrumentation points before running the fuzzing, so we don't have to print tested symbols along the way. Instead, at each iteration, we print what interval of lines we are currently testing. When we witness a crash, we can go back to the saved file to see what instrumentation points caused it. Signed-off-by: Francis Deslauriers --- .../kprobe-fuzzing-generate-data.yml | 14 +++ lava/system-tests/kprobe-fuzzing-tests.yml | 7 +- scripts/system-tests/kprobe-fuzzing.sh | 59 ---------- scripts/system-tests/lava-submit.py | 18 +++ scripts/system-tests/run-kprobe-fuzzing.py | 109 ++++++++++++++++++ .../run-kprobe-generate-instr-points.py | 89 ++++++++++++++ 6 files changed, 233 insertions(+), 63 deletions(-) create mode 100644 lava/system-tests/kprobe-fuzzing-generate-data.yml delete mode 100755 scripts/system-tests/kprobe-fuzzing.sh create mode 100644 scripts/system-tests/run-kprobe-fuzzing.py create mode 100644 scripts/system-tests/run-kprobe-generate-instr-points.py diff --git a/lava/system-tests/kprobe-fuzzing-generate-data.yml b/lava/system-tests/kprobe-fuzzing-generate-data.yml new file mode 100644 index 0000000..d9d5e49 --- /dev/null +++ b/lava/system-tests/kprobe-fuzzing-generate-data.yml @@ -0,0 +1,14 @@ +metadata: + format: Lava-Test Test Definition 1.0 + name: lttng-fuzzing-kprobe-generate-data + description: "Run kprobe fuzzing data generation" +install: + git-repos: + - url: https://github.com/lttng/lttng-ci + destination: ci + branch: master +run: + steps: + - cd ci/ + - lava-test-case generate-fuzzing-data --shell "python3 ./scripts/system-tests/run-kprobe-generate-instr-points.py $((1 + RANDOM))" + - sync diff --git a/lava/system-tests/kprobe-fuzzing-tests.yml b/lava/system-tests/kprobe-fuzzing-tests.yml index cf65067..56812ba 100644 --- a/lava/system-tests/kprobe-fuzzing-tests.yml +++ b/lava/system-tests/kprobe-fuzzing-tests.yml @@ -8,15 +8,14 @@ install: destination: ci branch: master steps: - - export TMPDIR="/tmp" - cd - ulimit -c unlimited - mkdir -p coredump - echo "$(pwd)/coredump/core.%e.%p.%h.%t" > /proc/sys/kernel/core_pattern run: steps: - - source /root/lttngvenv/activate - cd ci/ - - lava-test-case run-tests --shell "./scripts/system-tests/kprobe-fuzzing.sh" + - lava-test-case run-fuzzing --shell "python3 ./scripts/system-tests/run-kprobe-fuzzing.py /root/instr_points.txt.gz" + - cd .. - tar czf coredump.tar.gz coredump - - lava-test-case-attach run-tests coredump.tar.gz + - lava-test-case-attach run-fuzzing coredump.tar.gz diff --git a/scripts/system-tests/kprobe-fuzzing.sh b/scripts/system-tests/kprobe-fuzzing.sh deleted file mode 100755 index fab987b..0000000 --- a/scripts/system-tests/kprobe-fuzzing.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2017 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -NB_KPROBE_PER_ITER=500 -SESSION_NAME="my_kprobe_session" - -# Silence the script to avoid redirection of kallsyms to fill the screen -set +x -syms=$(awk '{print $3;}' /proc/kallsyms | sort -R) -nb_syms=$(echo "$syms" | wc -l) -set -x - -# Loop over the list of symbols and enable the symbols in groups of -# $NB_KPROBE_PER_ITER -for i in $(seq 0 "$NB_KPROBE_PER_ITER" "$nb_syms"); do - # Print time in UTC at each iteration to easily see when the script - # hangs - date --utc - - # Pick $NB_KPROBE_PER_ITER symbols to instrument, craft enable-event - # command and save them to a file. We craft the commands and executed - # them in two steps so that the pipeline can be done without the bash - # '-x' option that would fill the serial buffer because of the multiple - # pipe redirections. - set +x - echo "$syms" | head -n $((i+NB_KPROBE_PER_ITER)) | tail -n $NB_KPROBE_PER_ITER |awk '{print "lttng enable-event --kernel --function=" $1 " " $1}' > lttng-enable-event.sh - set -x - - # Print what iteration we are at - echo "$i" $((i+NB_KPROBE_PER_ITER)) - - # Destroy previous session and create a new one - lttng create "$SESSION_NAME" - - # Expect commands to fail, turn off early exit of shell script on - # non-zero return value - set +e - source ./lttng-enable-event.sh - set -e - - # Run stress util to generate some kernel activity - stress --cpu 2 --io 4 --vm 2 --vm-bytes 128M --hdd 3 --timeout 5s - - lttng list "$SESSION_NAME" - lttng destroy "$SESSION_NAME" -done diff --git a/scripts/system-tests/lava-submit.py b/scripts/system-tests/lava-submit.py index 94ecba2..4ee7077 100644 --- a/scripts/system-tests/lava-submit.py +++ b/scripts/system-tests/lava-submit.py @@ -267,6 +267,23 @@ def get_kvm_tests_cmd(): } }) return command + +def get_kprobes_generate_data_cmd(): + command = OrderedDict({ + 'command': 'lava_test_shell', + 'parameters': { + 'testdef_repos': [ + { + 'git-repo': 'https://github.com/lttng/lttng-ci.git', + 'revision': 'master', + 'testdef': 'lava/system-tests/kprobe-fuzzing-generate-data.yml' + } + ], + 'timeout': 60 + } + }) + return command + def get_kprobes_test_cmd(): command = OrderedDict({ 'command': 'lava_test_shell', @@ -447,6 +464,7 @@ def main(): return -1 j['actions'].append(get_config_cmd('kvm')) j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit)) + j['actions'].append(get_kprobes_generate_data_cmd()) j['actions'].append(get_kprobes_test_cmd()) j['actions'].append(get_results_cmd(stream_name='tests-kernel')) else: diff --git a/scripts/system-tests/run-kprobe-fuzzing.py b/scripts/system-tests/run-kprobe-fuzzing.py new file mode 100644 index 0000000..ef79077 --- /dev/null +++ b/scripts/system-tests/run-kprobe-fuzzing.py @@ -0,0 +1,109 @@ +# Copyright (C) 2018 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import datetime +import gzip +import os +import pprint +import subprocess +import sys + +NB_KPROBES_PER_ITER=500 + +def load_instr_points(instr_points_archive): + print('Reading instrumentation points from \'{}\'.'.format(instr_points_archive), end='') + sys.stdout.flush() + + with gzip.open(instr_points_archive, 'r') as f: + data = f.read() + print(' Done.') + + return [x.decode('utf-8') for x in data.split()] + +def enable_kprobe_events(instr_points): + print('Enabling events...', end='') + sys.stdout.flush() + + # Use os module directly, because this is a sysfs file and seeking inside + # the file is not supported. The python open() function with the append + # ('a') flag uses lseek(, SEEK_END) to move the write pointer to the end. + fd = os.open('/sys/kernel/debug/tracing/kprobe_events', os.O_WRONLY|os.O_CREAT|os.O_APPEND) + for i, point in enumerate(instr_points): + + kprobe_cmd = 'r:event_{} {}\n'.format(i, point).encode('utf-8') + try: + os.write(fd, kprobe_cmd) + except OSError: + continue + os.close(fd) + print(' Done.') + +def set_kprobe_tracing_state(state): + if state not in (0 ,1): + raise ValueError + + if state == 0: + # Clear the content of the trace. + open('/sys/kernel/debug/tracing/trace', 'w').close() + + try: + with open('/sys/kernel/debug/tracing/events/kprobes/enable', 'w') as enable_kprobe_file: + enable_kprobe_file.write('{}\n'.format(state)) + except IOError: + print('kprobes/enable file does not exist') + +def run_workload(): + print('Running workload...', end='') + sys.stdout.flush() + workload = ['stress', '--cpu', '2', '--io', '4', '--vm', '2', + '--vm-bytes', '128M', '--hdd', '3', '--timeout', '3s'] + try: + with open(os.devnull) as devnull: + subprocess.call(workload, stdout=devnull, stderr=devnull) + except OSError as e: + print("Workload execution failed:", e, file=sys.stderr) + pprint.pprint(workload) + + print(' Done.') + +def mount_tracingfs(): + with open(os.devnull) as devnull: + subprocess.call(['mount', '-t', 'debugfs', 'nodev', '/sys/kernel/debug/'], + stdout=devnull, stderr=devnull) + +def print_dashed_line(): + print('-'*100) + +def main(): + assert(len(sys.argv) == 2) + + instr_point_archive = sys.argv[1] + # Load instrumentation points to disk and attach it to lava test run. + instrumentation_points = load_instr_points(instr_point_archive) + + mount_tracingfs() + + # Loop over the list by enabling ranges of NB_KPROBES_PER_ITER kprobes. + for i in range(int(len(instrumentation_points)/NB_KPROBES_PER_ITER)): + print_dashed_line() + print('Time now: {}, {} to {}'.format(datetime.datetime.now(), i*NB_KPROBES_PER_ITER, (i+1)*NB_KPROBES_PER_ITER)) + set_kprobe_tracing_state(0) + enable_kprobe_events(instrumentation_points[i*NB_KPROBES_PER_ITER:(i+1)*NB_KPROBES_PER_ITER]) + set_kprobe_tracing_state(1) + run_workload() + print('\n') + +if __name__ == "__main__": + main() diff --git a/scripts/system-tests/run-kprobe-generate-instr-points.py b/scripts/system-tests/run-kprobe-generate-instr-points.py new file mode 100644 index 0000000..072ff3d --- /dev/null +++ b/scripts/system-tests/run-kprobe-generate-instr-points.py @@ -0,0 +1,89 @@ +# Copyright (C) 2018 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import datetime +import gzip +import os +import pprint +import random +import subprocess +import sys + +def save_instr_points(instr_points): + + # Save in /root to be persistent across lava slave reboots. + instrumenation_points_arch = '/root/instr_points.txt.gz' + + print('Saving instrumentation points to \'{}\' ...'.format(instrumenation_points_arch), end='') + sys.stdout.flush() + + text = "\n".join(instr_points) + + with gzip.open(instrumenation_points_arch, 'w') as f: + f.write(text.encode('utf-8')) + + # Attach fuzzing data to test case. + events = ['lava-test-case-attach', 'generate-fuzzing-data', instrumenation_points_arch] + + try: + subprocess.call(events) + except OSError as e: + print("Execution failed:", e, file=sys.stderr) + print("Probably not running on the lava worker") + pprint.pprint(events) + print('Done.') + +def main(): + assert(len(sys.argv) == 2) + + seed = int(sys.argv[1]) + print('Random seed: {}'.format(seed)) + + rng = random.Random(seed) + + # Get all the symbols from kallsyms. + with open('/proc/kallsyms') as kallsyms_file: + raw_symbol_list = kallsyms_file.readlines() + + # Keep only the symbol name. + raw_symbol_list = [x.split()[2].strip() for x in raw_symbol_list] + + instrumentation_points = [] + + # Add all symbols. + instrumentation_points.extend(raw_symbol_list) + + # For each symbol, create 2 new instrumentation points by random offsets. + for s in raw_symbol_list: + offsets = rng.sample(range(1, 10), 2) + for offset in offsets: + instrumentation_points.append(s + "+" + str(hex(offset))) + + lower_bound = 0x0 + upper_bound = 0xffffffffffffffff + address_list = [] + + # Add random addresses to the instrumentation points. + for _ in range(1000): + instrumentation_points.append(hex(rng.randint(lower_bound, upper_bound))) + + # Shuffle the entire list. + rng.shuffle(instrumentation_points) + + # Save instrumentation points to disk and attach it to lava test run. + save_instr_points(instrumentation_points) + +if __name__ == "__main__": + main() -- 2.34.1