This commits reimplements the kprobe fuzzing job in Python.
The Lava job is split into two stages. The first stage generates the
list of instrumentation points to try and saves it to the Lava job.
The second stage, reads this file and perform the fuzzing.
To save the instrumentation points file, we need to attach it to the job
from a Lava test shell that doesn't crash since that would prevent the
file from being saved. Because the fuzzing stage is likely to crash, we
need to do the generation and saving in a different stage.
In the past, we only fuzzed kallsyms symbols. Now we also fuzz random
offsets from these symbols and random addresses in the entire address
range.
We now save the list of instrumentation points before running the
fuzzing, so we don't have to print tested symbols along the way.
Instead, at each iteration, we print what interval of lines we are
currently testing. When we witness a crash, we can go back to the saved
file to see what instrumentation points caused it.
Signed-off-by: Francis Deslauriers <francis.deslauriers@efficios.com>
--- /dev/null
+metadata:
+ format: Lava-Test Test Definition 1.0
+ name: lttng-fuzzing-kprobe-generate-data
+ description: "Run kprobe fuzzing data generation"
+install:
+ git-repos:
+ - url: https://github.com/lttng/lttng-ci
+ destination: ci
+ branch: master
+run:
+ steps:
+ - cd ci/
+ - lava-test-case generate-fuzzing-data --shell "python3 ./scripts/system-tests/run-kprobe-generate-instr-points.py $((1 + RANDOM))"
+ - sync
destination: ci
branch: master
steps:
- - export TMPDIR="/tmp"
- cd
- ulimit -c unlimited
- mkdir -p coredump
- echo "$(pwd)/coredump/core.%e.%p.%h.%t" > /proc/sys/kernel/core_pattern
run:
steps:
- - source /root/lttngvenv/activate
- cd ci/
- - lava-test-case run-tests --shell "./scripts/system-tests/kprobe-fuzzing.sh"
+ - lava-test-case run-fuzzing --shell "python3 ./scripts/system-tests/run-kprobe-fuzzing.py /root/instr_points.txt.gz"
+ - cd ..
- tar czf coredump.tar.gz coredump
- - lava-test-case-attach run-tests coredump.tar.gz
+ - lava-test-case-attach run-fuzzing coredump.tar.gz
+++ /dev/null
-#!/bin/bash -xeu
-# Copyright (C) 2017 - Francis Deslauriers <francis.deslauriers@efficios.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-NB_KPROBE_PER_ITER=500
-SESSION_NAME="my_kprobe_session"
-
-# Silence the script to avoid redirection of kallsyms to fill the screen
-set +x
-syms=$(awk '{print $3;}' /proc/kallsyms | sort -R)
-nb_syms=$(echo "$syms" | wc -l)
-set -x
-
-# Loop over the list of symbols and enable the symbols in groups of
-# $NB_KPROBE_PER_ITER
-for i in $(seq 0 "$NB_KPROBE_PER_ITER" "$nb_syms"); do
- # Print time in UTC at each iteration to easily see when the script
- # hangs
- date --utc
-
- # Pick $NB_KPROBE_PER_ITER symbols to instrument, craft enable-event
- # command and save them to a file. We craft the commands and executed
- # them in two steps so that the pipeline can be done without the bash
- # '-x' option that would fill the serial buffer because of the multiple
- # pipe redirections.
- set +x
- echo "$syms" | head -n $((i+NB_KPROBE_PER_ITER)) | tail -n $NB_KPROBE_PER_ITER |awk '{print "lttng enable-event --kernel --function=" $1 " " $1}' > lttng-enable-event.sh
- set -x
-
- # Print what iteration we are at
- echo "$i" $((i+NB_KPROBE_PER_ITER))
-
- # Destroy previous session and create a new one
- lttng create "$SESSION_NAME"
-
- # Expect commands to fail, turn off early exit of shell script on
- # non-zero return value
- set +e
- source ./lttng-enable-event.sh
- set -e
-
- # Run stress util to generate some kernel activity
- stress --cpu 2 --io 4 --vm 2 --vm-bytes 128M --hdd 3 --timeout 5s
-
- lttng list "$SESSION_NAME"
- lttng destroy "$SESSION_NAME"
-done
}
})
return command
+
+def get_kprobes_generate_data_cmd():
+ command = OrderedDict({
+ 'command': 'lava_test_shell',
+ 'parameters': {
+ 'testdef_repos': [
+ {
+ 'git-repo': 'https://github.com/lttng/lttng-ci.git',
+ 'revision': 'master',
+ 'testdef': 'lava/system-tests/kprobe-fuzzing-generate-data.yml'
+ }
+ ],
+ 'timeout': 60
+ }
+ })
+ return command
+
def get_kprobes_test_cmd():
command = OrderedDict({
'command': 'lava_test_shell',
return -1
j['actions'].append(get_config_cmd('kvm'))
j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit))
+ j['actions'].append(get_kprobes_generate_data_cmd())
j['actions'].append(get_kprobes_test_cmd())
j['actions'].append(get_results_cmd(stream_name='tests-kernel'))
else:
--- /dev/null
+# Copyright (C) 2018 - Francis Deslauriers <francis.deslauriers@efficios.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import datetime
+import gzip
+import os
+import pprint
+import subprocess
+import sys
+
+NB_KPROBES_PER_ITER=500
+
+def load_instr_points(instr_points_archive):
+ print('Reading instrumentation points from \'{}\'.'.format(instr_points_archive), end='')
+ sys.stdout.flush()
+
+ with gzip.open(instr_points_archive, 'r') as f:
+ data = f.read()
+ print(' Done.')
+
+ return [x.decode('utf-8') for x in data.split()]
+
+def enable_kprobe_events(instr_points):
+ print('Enabling events...', end='')
+ sys.stdout.flush()
+
+ # Use os module directly, because this is a sysfs file and seeking inside
+ # the file is not supported. The python open() function with the append
+ # ('a') flag uses lseek(, SEEK_END) to move the write pointer to the end.
+ fd = os.open('/sys/kernel/debug/tracing/kprobe_events', os.O_WRONLY|os.O_CREAT|os.O_APPEND)
+ for i, point in enumerate(instr_points):
+
+ kprobe_cmd = 'r:event_{} {}\n'.format(i, point).encode('utf-8')
+ try:
+ os.write(fd, kprobe_cmd)
+ except OSError:
+ continue
+ os.close(fd)
+ print(' Done.')
+
+def set_kprobe_tracing_state(state):
+ if state not in (0 ,1):
+ raise ValueError
+
+ if state == 0:
+ # Clear the content of the trace.
+ open('/sys/kernel/debug/tracing/trace', 'w').close()
+
+ try:
+ with open('/sys/kernel/debug/tracing/events/kprobes/enable', 'w') as enable_kprobe_file:
+ enable_kprobe_file.write('{}\n'.format(state))
+ except IOError:
+ print('kprobes/enable file does not exist')
+
+def run_workload():
+ print('Running workload...', end='')
+ sys.stdout.flush()
+ workload = ['stress', '--cpu', '2', '--io', '4', '--vm', '2',
+ '--vm-bytes', '128M', '--hdd', '3', '--timeout', '3s']
+ try:
+ with open(os.devnull) as devnull:
+ subprocess.call(workload, stdout=devnull, stderr=devnull)
+ except OSError as e:
+ print("Workload execution failed:", e, file=sys.stderr)
+ pprint.pprint(workload)
+
+ print(' Done.')
+
+def mount_tracingfs():
+ with open(os.devnull) as devnull:
+ subprocess.call(['mount', '-t', 'debugfs', 'nodev', '/sys/kernel/debug/'],
+ stdout=devnull, stderr=devnull)
+
+def print_dashed_line():
+ print('-'*100)
+
+def main():
+ assert(len(sys.argv) == 2)
+
+ instr_point_archive = sys.argv[1]
+ # Load instrumentation points to disk and attach it to lava test run.
+ instrumentation_points = load_instr_points(instr_point_archive)
+
+ mount_tracingfs()
+
+ # Loop over the list by enabling ranges of NB_KPROBES_PER_ITER kprobes.
+ for i in range(int(len(instrumentation_points)/NB_KPROBES_PER_ITER)):
+ print_dashed_line()
+ print('Time now: {}, {} to {}'.format(datetime.datetime.now(), i*NB_KPROBES_PER_ITER, (i+1)*NB_KPROBES_PER_ITER))
+ set_kprobe_tracing_state(0)
+ enable_kprobe_events(instrumentation_points[i*NB_KPROBES_PER_ITER:(i+1)*NB_KPROBES_PER_ITER])
+ set_kprobe_tracing_state(1)
+ run_workload()
+ print('\n')
+
+if __name__ == "__main__":
+ main()
--- /dev/null
+# Copyright (C) 2018 - Francis Deslauriers <francis.deslauriers@efficios.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import datetime
+import gzip
+import os
+import pprint
+import random
+import subprocess
+import sys
+
+def save_instr_points(instr_points):
+
+ # Save in /root to be persistent across lava slave reboots.
+ instrumenation_points_arch = '/root/instr_points.txt.gz'
+
+ print('Saving instrumentation points to \'{}\' ...'.format(instrumenation_points_arch), end='')
+ sys.stdout.flush()
+
+ text = "\n".join(instr_points)
+
+ with gzip.open(instrumenation_points_arch, 'w') as f:
+ f.write(text.encode('utf-8'))
+
+ # Attach fuzzing data to test case.
+ events = ['lava-test-case-attach', 'generate-fuzzing-data', instrumenation_points_arch]
+
+ try:
+ subprocess.call(events)
+ except OSError as e:
+ print("Execution failed:", e, file=sys.stderr)
+ print("Probably not running on the lava worker")
+ pprint.pprint(events)
+ print('Done.')
+
+def main():
+ assert(len(sys.argv) == 2)
+
+ seed = int(sys.argv[1])
+ print('Random seed: {}'.format(seed))
+
+ rng = random.Random(seed)
+
+ # Get all the symbols from kallsyms.
+ with open('/proc/kallsyms') as kallsyms_file:
+ raw_symbol_list = kallsyms_file.readlines()
+
+ # Keep only the symbol name.
+ raw_symbol_list = [x.split()[2].strip() for x in raw_symbol_list]
+
+ instrumentation_points = []
+
+ # Add all symbols.
+ instrumentation_points.extend(raw_symbol_list)
+
+ # For each symbol, create 2 new instrumentation points by random offsets.
+ for s in raw_symbol_list:
+ offsets = rng.sample(range(1, 10), 2)
+ for offset in offsets:
+ instrumentation_points.append(s + "+" + str(hex(offset)))
+
+ lower_bound = 0x0
+ upper_bound = 0xffffffffffffffff
+ address_list = []
+
+ # Add random addresses to the instrumentation points.
+ for _ in range(1000):
+ instrumentation_points.append(hex(rng.randint(lower_bound, upper_bound)))
+
+ # Shuffle the entire list.
+ rng.shuffle(instrumentation_points)
+
+ # Save instrumentation points to disk and attach it to lava test run.
+ save_instr_points(instrumentation_points)
+
+if __name__ == "__main__":
+ main()
projects / lttng-ci.git / commitdiff
