scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import base64
  19 import json
  20 import os
  21 import random
  22 import sys
  23 import time
  24 import yaml
  25 import xmlrpc.client
  26 import pprint
  27 from urllib.parse import urljoin
  28 from urllib.request import urlretrieve
  29
  30 from jinja2 import Environment, FileSystemLoader, meta
  31
  32 USERNAME = 'lava-jenkins'
  33 HOSTNAME = 'lava-master-02.internal.efficios.com'
  34 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  35
  36 class TestType():
  37     baremetal_benchmarks=1
  38     baremetal_tests=2
  39     kvm_tests=3
  40     kvm_fuzzing_tests=4
  41     values = {
  42         'baremetal-benchmarks' : baremetal_benchmarks,
  43         'baremetal-tests' : baremetal_tests,
  44         'kvm-tests' : kvm_tests,
  45         'kvm-fuzzing-tests' : kvm_fuzzing_tests,
  46     }
  47
  48 class DeviceType():
  49     x86 = 'x86'
  50     kvm = 'qemu'
  51     values = {
  52         'kvm' : kvm,
  53         'x86' : x86,
  54     }
  55
  56 def get_job_bundle_content(server, job):
  57     try:
  58         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  59         bundle = server.dashboard.get(bundle_sha)
  60     except xmlrpc.client.Fault as f:
  61         print('Error while fetching results bundle', f.faultString)
  62         raise f
  63
  64     return json.loads(bundle['content'])
  65
  66 # Parse the results bundle to see the run-tests testcase
  67 # of the lttng-kernel-tests passed successfully
  68 def check_job_all_test_cases_state_count(server, job):
  69     print("Testcase result:")
  70     content = server.results.get_testjob_results_yaml(str(job))
  71     testcases = yaml.load(content)
  72
  73     passed_tests=0
  74     failed_tests=0
  75     for testcase in testcases:
  76         if testcase['result'] != 'pass':
  77             print("\tFAILED {}\n\t\t See http://{}{}".format(
  78                 testcase['name'],
  79                 HOSTNAME,
  80                 testcase['url']
  81             ))
  82             failed_tests+=1
  83         else:
  84             passed_tests+=1
  85     return (passed_tests, failed_tests)
  86
  87 # Get the benchmark results from the objstore
  88 # save them as CSV files localy
  89 def fetch_benchmark_results(build_id):
  90     testcases = ['processed_results_close.csv',
  91             'processed_results_ioctl.csv',
  92             'processed_results_open_efault.csv',
  93             'processed_results_open_enoent.csv',
  94             'processed_results_dup_close.csv',
  95             'processed_results_raw_syscall_getpid.csv',
  96             'processed_results_lttng_test_filter.csv']
  97     for testcase in testcases:
  98         url = urljoin(OBJSTORE_URL, "{:s}/{:s}".format(build_id, testcase))
  99         urlretrieve(url, testcase)
 100
 101 # Parse the attachment of the testcase to fetch the stdout of the test suite
 102 def print_test_output(server, job):
 103     job_finished, log = server.scheduler.jobs.logs(str(job))
 104     logs = yaml.load(log.data.decode('ascii'))
 105     print_line = False
 106     for line in logs:
 107         if line['lvl'] != 'target':
 108             continue
 109         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 110             print('---- TEST SUITE OUTPUT BEGIN ----')
 111             print_line = True
 112             continue
 113         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 114             print('----- TEST SUITE OUTPUT END -----')
 115             break
 116         if print_line:
 117             print("{} {}".format(line['dt'], line['msg']))
 118
 119 def get_vlttng_cmd(device, lttng_tools_commit, lttng_ust_commit=None):
 120
 121     vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
 122                     ' --override projects.babeltrace.build-env.PYTHON=python3' \
 123                     ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
 124                     ' --profile babeltrace-stable-1.4' \
 125                     ' --profile babeltrace-python' \
 126                     ' --profile lttng-tools-master' \
 127                     ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
 128                     ' --profile lttng-tools-no-man-pages'
 129
 130     if lttng_ust_commit is not None:
 131         vlttng_cmd += ' --profile lttng-ust-master ' \
 132                     ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
 133                     ' --profile lttng-ust-no-man-pages'
 134
 135     vlttng_path = '/tmp/virtenv'
 136
 137     vlttng_cmd += ' ' + vlttng_path
 138
 139     return vlttng_cmd
 140
 141 def main():
 142     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 143     test_type = None
 144     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 145     parser.add_argument('-t', '--type', required=True)
 146     parser.add_argument('-j', '--jobname', required=True)
 147     parser.add_argument('-k', '--kernel', required=True)
 148     parser.add_argument('-lm', '--lmodule', required=True)
 149     parser.add_argument('-tc', '--tools-commit', required=True)
 150     parser.add_argument('-id', '--build-id', required=True)
 151     parser.add_argument('-uc', '--ust-commit', required=False)
 152     parser.add_argument('-d', '--debug', required=False, action='store_true')
 153     args = parser.parse_args()
 154
 155     if args.type not in TestType.values:
 156         print('argument -t/--type {} unrecognized.'.format(args.type))
 157         print('Possible values are:')
 158         for k in TestType.values:
 159             print('\t {}'.format(k))
 160         return -1
 161
 162     lava_api_key = None
 163     if not args.debug:
 164         try:
 165             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 166         except Exception as e:
 167             print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...', e )
 168             return -1
 169
 170     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 171     jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
 172             lstrip_blocks= True)
 173     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 174     template_source = jinja_env.loader.get_source(jinja_env, 'template_lava_job.jinja2')
 175     parsed_content = jinja_env.parse(template_source)
 176     undef = meta.find_undeclared_variables(parsed_content)
 177
 178     test_type = TestType.values[args.type]
 179
 180     if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
 181         device_type = DeviceType.x86
 182     else:
 183         device_type = DeviceType.kvm
 184
 185     vlttng_path = '/tmp/virtenv'
 186
 187     vlttng_cmd = get_vlttng_cmd(device_type, args.tools_commit, args.ust_commit)
 188
 189     context = dict()
 190     context['DeviceType'] = DeviceType
 191     context['TestType'] = TestType
 192
 193     context['job_name'] = args.jobname
 194     context['test_type'] = test_type
 195     context['random_seed'] = random.randint(0, 1000000)
 196     context['device_type'] = device_type
 197
 198     context['vlttng_cmd'] = vlttng_cmd
 199     context['vlttng_path'] = vlttng_path
 200
 201     context['kernel_url'] = args.kernel
 202     context['nfsrootfs_url'] = nfsrootfs
 203     context['lttng_modules_url'] = args.lmodule
 204     context['jenkins_build_id'] = args.build_id
 205
 206     context['kprobe_round_nb'] = 10
 207
 208     render = jinja_template.render(context)
 209
 210     print('Job to be submitted:')
 211
 212     print(render)
 213
 214     if args.debug:
 215         return 0
 216
 217     server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))
 218
 219     jobid = server.scheduler.submit_job(render)
 220
 221     print('Lava jobid:{}'.format(jobid))
 222     print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(jobid))
 223
 224     #Check the status of the job every 30 seconds
 225     jobstatus = server.scheduler.job_state(jobid)['job_state']
 226     running = False
 227     while jobstatus in ['Submitted','Scheduling','Scheduled','Running']:
 228         if not running and jobstatus == 'Running':
 229             print('Job started running')
 230             running = True
 231         time.sleep(30)
 232         try:
 233             jobstatus = server.scheduler.job_state(jobid)['job_state']
 234         except xmlrpc.client.ProtocolError as e:
 235             print('Protocol error, retring')
 236     print('Job ended with {} status.'.format(jobstatus))
 237
 238     if jobstatus != 'Finished':
 239         return -1
 240
 241     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 242         print_test_output(server, jobid)
 243     elif test_type is TestType.baremetal_benchmarks:
 244         fetch_benchmark_results(args.build_id)
 245
 246     passed, failed=check_job_all_test_cases_state_count(server, jobid)
 247     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 248
 249     if failed == 0:
 250         return 0
 251     else:
 252         return -1
 253
 254 if __name__ == "__main__":
 255     sys.exit(main())