scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python3
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import json
  19 import os
  20 import random
  21 import sys
  22 import time
  23 import xmlrpc.client
  24 from urllib.parse import urljoin
  25 from urllib.request import urlretrieve
  26 import yaml
  27 from jinja2 import Environment, FileSystemLoader
  28
  29 USERNAME = 'lava-jenkins'
  30 HOSTNAME = 'lava-master-02.internal.efficios.com'
  31 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  32
  33
  34 class TestType:
  35     """ Enum like for test type """
  36
  37     baremetal_benchmarks = 1
  38     baremetal_tests = 2
  39     kvm_tests = 3
  40     kvm_fuzzing_tests = 4
  41     values = {
  42         'baremetal-benchmarks': baremetal_benchmarks,
  43         'baremetal-tests': baremetal_tests,
  44         'kvm-tests': kvm_tests,
  45         'kvm-fuzzing-tests': kvm_fuzzing_tests,
  46     }
  47
  48
  49 class DeviceType:
  50     """ Enum like for device type """
  51
  52     x86 = 'x86'
  53     kvm = 'qemu'
  54     values = {'kvm': kvm, 'x86': x86}
  55
  56
  57 def get_job_bundle_content(server, job):
  58     try:
  59         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  60         bundle = server.dashboard.get(bundle_sha)
  61     except xmlrpc.client.Fault as error:
  62         print('Error while fetching results bundle', error.faultString)
  63         raise error
  64
  65     return json.loads(bundle['content'])
  66
  67
  68 def check_job_all_test_cases_state_count(server, job):
  69     """
  70     Parse the results bundle to see the run-tests testcase
  71     of the lttng-kernel-tests passed successfully
  72     """
  73     print("Testcase result:")
  74     content = server.results.get_testjob_results_yaml(str(job))
  75     testcases = yaml.unsafe_load(content)
  76
  77     passed_tests = 0
  78     failed_tests = 0
  79     for testcase in testcases:
  80         if testcase['result'] != 'pass':
  81             print(
  82                 "\tFAILED {}\n\t\t See http://{}{}".format(
  83                     testcase['name'], HOSTNAME, testcase['url']
  84                 )
  85             )
  86             failed_tests += 1
  87         else:
  88             passed_tests += 1
  89     return (passed_tests, failed_tests)
  90
  91
  92 def fetch_benchmark_results(build_id):
  93     """
  94     Get the benchmark results from the objstore
  95     save them as CSV files localy
  96     """
  97     testcases = [
  98         'processed_results_close.csv',
  99         'processed_results_ioctl.csv',
 100         'processed_results_open_efault.csv',
 101         'processed_results_open_enoent.csv',
 102         'processed_results_dup_close.csv',
 103         'processed_results_raw_syscall_getpid.csv',
 104         'processed_results_lttng_test_filter.csv',
 105     ]
 106     for testcase in testcases:
 107         url = urljoin(OBJSTORE_URL, "{:s}/{:s}".format(build_id, testcase))
 108         print('Fetching {}'.format(url))
 109         urlretrieve(url, testcase)
 110
 111
 112 def print_test_output(server, job):
 113     """
 114     Parse the attachment of the testcase to fetch the stdout of the test suite
 115     """
 116     job_finished, log = server.scheduler.jobs.logs(str(job))
 117     logs = yaml.unsafe_load(log.data.decode('ascii'))
 118     print_line = False
 119     for line in logs:
 120         if line['lvl'] != 'target':
 121             continue
 122         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 123             print('---- TEST SUITE OUTPUT BEGIN ----')
 124             print_line = True
 125             continue
 126         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 127             print('----- TEST SUITE OUTPUT END -----')
 128             print_line = False
 129             continue
 130         if print_line:
 131             print("{} {}".format(line['dt'], line['msg']))
 132
 133
 134 def get_vlttng_cmd(
 135     lttng_tools_url, lttng_tools_commit, lttng_ust_url=None, lttng_ust_commit=None
 136 ):
 137     """
 138     Return vlttng cmd to be used in the job template for setup.
 139     """
 140
 141     vlttng_cmd = (
 142         'vlttng --jobs=$(nproc) --profile urcu-master'
 143         ' --override projects.babeltrace.build-env.PYTHON=python3'
 144         ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config'
 145         ' --profile babeltrace-stable-1.4'
 146         ' --profile babeltrace-python'
 147         ' --profile lttng-tools-master'
 148         ' --override projects.lttng-tools.source='
 149         + lttng_tools_url
 150         + ' --override projects.lttng-tools.checkout='
 151         + lttng_tools_commit
 152         + ' --profile lttng-tools-no-man-pages'
 153     )
 154
 155     if lttng_ust_commit is not None:
 156         vlttng_cmd += (
 157             ' --profile lttng-ust-master '
 158             ' --override projects.lttng-ust.source='
 159             + lttng_ust_url
 160             + ' --override projects.lttng-ust.checkout='
 161             + lttng_ust_commit
 162             + ' --profile lttng-ust-no-man-pages'
 163         )
 164
 165     vlttng_path = '/tmp/virtenv'
 166
 167     vlttng_cmd += ' ' + vlttng_path
 168
 169     return vlttng_cmd
 170
 171
 172 def main():
 173     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 174     test_type = None
 175     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 176     parser.add_argument('-t', '--type', required=True)
 177     parser.add_argument('-j', '--jobname', required=True)
 178     parser.add_argument('-k', '--kernel', required=True)
 179     parser.add_argument('-lm', '--lmodule', required=True)
 180     parser.add_argument('-tu', '--tools-url', required=True)
 181     parser.add_argument('-tc', '--tools-commit', required=True)
 182     parser.add_argument('-id', '--build-id', required=True)
 183     parser.add_argument('-uu', '--ust-url', required=False)
 184     parser.add_argument('-uc', '--ust-commit', required=False)
 185     parser.add_argument('-d', '--debug', required=False, action='store_true')
 186     args = parser.parse_args()
 187
 188     if args.type not in TestType.values:
 189         print('argument -t/--type {} unrecognized.'.format(args.type))
 190         print('Possible values are:')
 191         for k in TestType.values:
 192             print('\t {}'.format(k))
 193         return -1
 194
 195     lava_api_key = None
 196     if not args.debug:
 197         try:
 198             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 199         except Exception as error:
 200             print(
 201                 'LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...',
 202                 error,
 203             )
 204             return -1
 205
 206     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 207     jinja_env = Environment(loader=jinja_loader, trim_blocks=True, lstrip_blocks=True)
 208     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 209
 210     test_type = TestType.values[args.type]
 211
 212     if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
 213         device_type = DeviceType.x86
 214     else:
 215         device_type = DeviceType.kvm
 216
 217     vlttng_path = '/tmp/virtenv'
 218
 219     vlttng_cmd = get_vlttng_cmd(
 220         args.tools_url, args.tools_commit, args.ust_url, args.ust_commit
 221     )
 222
 223     context = dict()
 224     context['DeviceType'] = DeviceType
 225     context['TestType'] = TestType
 226
 227     context['job_name'] = args.jobname
 228     context['test_type'] = test_type
 229     context['random_seed'] = random.randint(0, 1000000)
 230     context['device_type'] = device_type
 231
 232     context['vlttng_cmd'] = vlttng_cmd
 233     context['vlttng_path'] = vlttng_path
 234
 235     context['kernel_url'] = args.kernel
 236     context['nfsrootfs_url'] = nfsrootfs
 237     context['lttng_modules_url'] = args.lmodule
 238     context['jenkins_build_id'] = args.build_id
 239
 240     context['kprobe_round_nb'] = 10
 241
 242     render = jinja_template.render(context)
 243
 244     print('Job to be submitted:')
 245
 246     print(render)
 247
 248     if args.debug:
 249         return 0
 250
 251     server = xmlrpc.client.ServerProxy(
 252         'http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME)
 253     )
 254
 255     for attempt in range(10):
 256         try:
 257             jobid = server.scheduler.submit_job(render)
 258         except xmlrpc.client.ProtocolError as error:
 259             print(
 260                 'Protocol error on submit, sleeping and retrying. Attempt #{}'.format(
 261                     attempt
 262                 )
 263             )
 264             time.sleep(5)
 265             continue
 266         else:
 267             break
 268
 269     print('Lava jobid:{}'.format(jobid))
 270     print(
 271         'Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(
 272             jobid
 273         )
 274     )
 275
 276     # Check the status of the job every 30 seconds
 277     jobstatus = server.scheduler.job_state(jobid)['job_state']
 278     running = False
 279     while jobstatus in ['Submitted', 'Scheduling', 'Scheduled', 'Running']:
 280         if not running and jobstatus == 'Running':
 281             print('Job started running')
 282             running = True
 283         time.sleep(30)
 284         try:
 285             jobstatus = server.scheduler.job_state(jobid)['job_state']
 286         except xmlrpc.client.ProtocolError as error:
 287             print('Protocol error, retrying')
 288             continue
 289     print('Job ended with {} status.'.format(jobstatus))
 290
 291     if jobstatus != 'Finished':
 292         return -1
 293
 294     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 295         print_test_output(server, jobid)
 296     elif test_type is TestType.baremetal_benchmarks:
 297         fetch_benchmark_results(args.build_id)
 298
 299     passed, failed = check_job_all_test_cases_state_count(server, jobid)
 300     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 301
 302     if failed != 0:
 303         return -1
 304
 305     return 0
 306
 307
 308 if __name__ == "__main__":
 309     sys.exit(main())