scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import base64
  19 import json
  20 import os
  21 import random
  22 import sys
  23 import time
  24 import yaml
  25 import xmlrpc.client
  26 import pprint
  27
  28 from jinja2 import Environment, FileSystemLoader, meta
  29
  30 USERNAME = 'lava-jenkins'
  31 HOSTNAME = 'lava-master-02.internal.efficios.com'
  32 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  33
  34 class TestType():
  35     baremetal_benchmarks=1
  36     baremetal_tests=2
  37     kvm_tests=3
  38     kvm_fuzzing_tests=4
  39     values = {
  40         'baremetal-benchmarks' : baremetal_benchmarks,
  41         'baremetal-tests' : baremetal_tests,
  42         'kvm-tests' : kvm_tests,
  43         'kvm-fuzzing-tests' : kvm_fuzzing_tests,
  44     }
  45
  46 class DeviceType():
  47     x86 = 'x86'
  48     kvm = 'qemu'
  49     values = {
  50         'kvm' : kvm,
  51         'x86' : x86,
  52     }
  53
  54 def get_job_bundle_content(server, job):
  55     try:
  56         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  57         bundle = server.dashboard.get(bundle_sha)
  58     except xmlrpc.client.Fault as f:
  59         print('Error while fetching results bundle', f.faultString)
  60         raise f
  61
  62     return json.loads(bundle['content'])
  63
  64 # Parse the results bundle to see the run-tests testcase
  65 # of the lttng-kernel-tests passed successfully
  66 def check_job_all_test_cases_state_count(server, job):
  67     print("Testcase result:")
  68     content = server.results.get_testjob_results_yaml(str(job))
  69     testcases = yaml.load(content)
  70
  71     passed_tests=0
  72     failed_tests=0
  73     for testcase in testcases:
  74         if testcase['result'] != 'pass':
  75             print("\tFAILED {}\n\t\t See http://{}{}".format(
  76                 testcase['name'],
  77                 HOSTNAME,
  78                 testcase['url']
  79             ))
  80             failed_tests+=1
  81         else:
  82             passed_tests+=1
  83     return (passed_tests, failed_tests)
  84
  85 # Get the benchmark results from the objstore
  86 # save them as CSV files localy
  87 def fetch_benchmark_results(build_id):
  88     testcases = ['processed_results_close.csv',
  89             'processed_results_ioctl.csv',
  90             'processed_results_open_efault.csv',
  91             'processed_results_open_enoent.csv',
  92             'processed_results_dup_close.csv',
  93             'processed_results_raw_syscall_getpid.csv',
  94             'processed_results_lttng_test_filter.csv']
  95     for testcase in testcases:
  96         url = urljoin(OBJSTORE_URL, "{:s}/{:s}".format(build_id, testcase))
  97         urlretrieve(url, testcase)
  98
  99 # Parse the attachment of the testcase to fetch the stdout of the test suite
 100 def print_test_output(server, job):
 101     job_finished, log = server.scheduler.jobs.logs(str(job))
 102     logs = yaml.load(log.data.decode('ascii'))
 103     print_line = False
 104     for line in logs:
 105         if line['lvl'] != 'target':
 106             continue
 107         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 108             print('---- TEST SUITE OUTPUT BEGIN ----')
 109             print_line = True
 110             continue
 111         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 112             print('----- TEST SUITE OUTPUT END -----')
 113             break
 114         if print_line:
 115             print("{} {}".format(line['dt'], line['msg']))
 116
 117 def get_vlttng_cmd(device, lttng_tools_commit, lttng_ust_commit=None):
 118
 119     vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
 120                     ' --override projects.babeltrace.build-env.PYTHON=python3' \
 121                     ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
 122                     ' --profile babeltrace-stable-1.4' \
 123                     ' --profile babeltrace-python' \
 124                     ' --profile lttng-tools-master' \
 125                     ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
 126                     ' --profile lttng-tools-no-man-pages'
 127
 128     if lttng_ust_commit is not None:
 129         vlttng_cmd += ' --profile lttng-ust-master ' \
 130                     ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
 131                     ' --profile lttng-ust-no-man-pages'
 132
 133     vlttng_path = '/tmp/virtenv'
 134
 135     vlttng_cmd += ' ' + vlttng_path
 136
 137     return vlttng_cmd
 138
 139 def main():
 140     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 141     test_type = None
 142     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 143     parser.add_argument('-t', '--type', required=True)
 144     parser.add_argument('-j', '--jobname', required=True)
 145     parser.add_argument('-k', '--kernel', required=True)
 146     parser.add_argument('-lm', '--lmodule', required=True)
 147     parser.add_argument('-tc', '--tools-commit', required=True)
 148     parser.add_argument('-id', '--build-id', required=True)
 149     parser.add_argument('-uc', '--ust-commit', required=False)
 150     parser.add_argument('-d', '--debug', required=False, action='store_true')
 151     args = parser.parse_args()
 152
 153     if args.type not in TestType.values:
 154         print('argument -t/--type {} unrecognized.'.format(args.type))
 155         print('Possible values are:')
 156         for k in TestType.values:
 157             print('\t {}'.format(k))
 158         return -1
 159
 160     lava_api_key = None
 161     if not args.debug:
 162         try:
 163             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 164         except Exception as e:
 165             print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...', e )
 166             return -1
 167
 168     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 169     jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
 170             lstrip_blocks= True)
 171     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 172     template_source = jinja_env.loader.get_source(jinja_env, 'template_lava_job.jinja2')
 173     parsed_content = jinja_env.parse(template_source)
 174     undef = meta.find_undeclared_variables(parsed_content)
 175
 176     test_type = TestType.values[args.type]
 177
 178     if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
 179         device_type = DeviceType.x86
 180     else:
 181         device_type = DeviceType.kvm
 182
 183     vlttng_path = '/tmp/virtenv'
 184
 185     vlttng_cmd = get_vlttng_cmd(device_type, args.tools_commit, args.ust_commit)
 186
 187     context = dict()
 188     context['DeviceType'] = DeviceType
 189     context['TestType'] = TestType
 190
 191     context['job_name'] = args.jobname
 192     context['test_type'] = test_type
 193     context['random_seed'] = random.randint(0, 1000000)
 194     context['device_type'] = device_type
 195
 196     context['vlttng_cmd'] = vlttng_cmd
 197     context['vlttng_path'] = vlttng_path
 198
 199     context['kernel_url'] = args.kernel
 200     context['nfsrootfs_url'] = nfsrootfs
 201     context['lttng_modules_url'] = args.lmodule
 202     context['jenkins_build_id'] = args.build_id
 203
 204     context['kprobe_round_nb'] = 10
 205
 206     render = jinja_template.render(context)
 207
 208     print('Job to be submitted:')
 209
 210     print(render)
 211
 212     if args.debug:
 213         return 0
 214
 215     server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))
 216
 217     jobid = server.scheduler.submit_job(render)
 218
 219     print('Lava jobid:{}'.format(jobid))
 220     print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(jobid))
 221
 222     #Check the status of the job every 30 seconds
 223     jobstatus = server.scheduler.job_state(jobid)['job_state']
 224     running = False
 225     while jobstatus in ['Submitted','Scheduling','Scheduled','Running']:
 226         if not running and jobstatus == 'Running':
 227             print('Job started running')
 228             running = True
 229         time.sleep(30)
 230         jobstatus = server.scheduler.job_state(jobid)['job_state']
 231     print('Job ended with {} status.'.format(jobstatus))
 232
 233     if jobstatus != 'Finished':
 234         return -1
 235
 236     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 237         print_test_output(server, jobid)
 238     elif test_type is TestType.baremetal_benchmarks:
 239         fetch_benchmark_results(args.build_id)
 240
 241     passed, failed=check_job_all_test_cases_state_count(server, jobid)
 242     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 243
 244     if failed == 0:
 245         return 0
 246     else:
 247         return -1
 248
 249 if __name__ == "__main__":
 250     sys.exit(main())