scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python3
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import json
  19 import os
  20 import random
  21 import sys
  22 import time
  23 import xmlrpc.client
  24 from urllib.parse import urljoin
  25 from urllib.request import urlretrieve
  26 import yaml
  27 from jinja2 import Environment, FileSystemLoader
  28
  29 USERNAME = 'lava-jenkins'
  30 HOSTNAME = 'lava-master-02.internal.efficios.com'
  31 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  32
  33
  34 class TestType:
  35     """ Enum like for test type """
  36
  37     baremetal_benchmarks = 1
  38     baremetal_tests = 2
  39     kvm_tests = 3
  40     kvm_fuzzing_tests = 4
  41     values = {
  42         'baremetal-benchmarks': baremetal_benchmarks,
  43         'baremetal-tests': baremetal_tests,
  44         'kvm-tests': kvm_tests,
  45         'kvm-fuzzing-tests': kvm_fuzzing_tests,
  46     }
  47
  48
  49 class DeviceType:
  50     """ Enum like for device type """
  51
  52     x86 = 'x86'
  53     kvm = 'qemu'
  54     values = {'kvm': kvm, 'x86': x86}
  55
  56
  57 def get_job_bundle_content(server, job):
  58     try:
  59         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  60         bundle = server.dashboard.get(bundle_sha)
  61     except xmlrpc.client.Fault as error:
  62         print('Error while fetching results bundle', error.faultString)
  63         raise error
  64
  65     return json.loads(bundle['content'])
  66
  67
  68 def check_job_all_test_cases_state_count(server, job):
  69     """
  70     Parse the results bundle to see the run-tests testcase
  71     of the lttng-kernel-tests passed successfully
  72     """
  73     print("Testcase result:")
  74     content = server.results.get_testjob_results_yaml(str(job))
  75     testcases = yaml.load(content)
  76
  77     passed_tests = 0
  78     failed_tests = 0
  79     for testcase in testcases:
  80         if testcase['result'] != 'pass':
  81             print(
  82                 "\tFAILED {}\n\t\t See http://{}{}".format(
  83                     testcase['name'], HOSTNAME, testcase['url']
  84                 )
  85             )
  86             failed_tests += 1
  87         else:
  88             passed_tests += 1
  89     return (passed_tests, failed_tests)
  90
  91
  92 def fetch_benchmark_results(build_id):
  93     """
  94     Get the benchmark results from the objstore
  95     save them as CSV files localy
  96     """
  97     testcases = [
  98         'processed_results_close.csv',
  99         'processed_results_ioctl.csv',
 100         'processed_results_open_efault.csv',
 101         'processed_results_open_enoent.csv',
 102         'processed_results_dup_close.csv',
 103         'processed_results_raw_syscall_getpid.csv',
 104         'processed_results_lttng_test_filter.csv',
 105     ]
 106     for testcase in testcases:
 107         url = urljoin(OBJSTORE_URL, "{:s}/{:s}".format(build_id, testcase))
 108         print('Fetching {}'.format(url))
 109         urlretrieve(url, testcase)
 110
 111
 112 def print_test_output(server, job):
 113     """
 114     Parse the attachment of the testcase to fetch the stdout of the test suite
 115     """
 116     job_finished, log = server.scheduler.jobs.logs(str(job))
 117     logs = yaml.load(log.data.decode('ascii'))
 118     print_line = False
 119     for line in logs:
 120         if line['lvl'] != 'target':
 121             continue
 122         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 123             print('---- TEST SUITE OUTPUT BEGIN ----')
 124             print_line = True
 125             continue
 126         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 127             print('----- TEST SUITE OUTPUT END -----')
 128             break
 129         if print_line:
 130             print("{} {}".format(line['dt'], line['msg']))
 131
 132
 133 def get_vlttng_cmd(
 134     lttng_tools_url, lttng_tools_commit, lttng_ust_url=None, lttng_ust_commit=None
 135 ):
 136     """
 137     Return vlttng cmd to be used in the job template for setup.
 138     """
 139
 140     vlttng_cmd = (
 141         'vlttng --jobs=$(nproc) --profile urcu-master'
 142         ' --override projects.babeltrace.build-env.PYTHON=python3'
 143         ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config'
 144         ' --profile babeltrace-stable-1.4'
 145         ' --profile babeltrace-python'
 146         ' --profile lttng-tools-master'
 147         ' --override projects.lttng-tools.source='
 148         + lttng_tools_url
 149         + ' --override projects.lttng-tools.checkout='
 150         + lttng_tools_commit
 151         + ' --profile lttng-tools-no-man-pages'
 152     )
 153
 154     if lttng_ust_commit is not None:
 155         vlttng_cmd += (
 156             ' --profile lttng-ust-master '
 157             ' --override projects.lttng-ust.source='
 158             + lttng_ust_url
 159             + ' --override projects.lttng-ust.checkout='
 160             + lttng_ust_commit
 161             + ' --profile lttng-ust-no-man-pages'
 162         )
 163
 164     vlttng_path = '/tmp/virtenv'
 165
 166     vlttng_cmd += ' ' + vlttng_path
 167
 168     return vlttng_cmd
 169
 170
 171 def main():
 172     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 173     test_type = None
 174     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 175     parser.add_argument('-t', '--type', required=True)
 176     parser.add_argument('-j', '--jobname', required=True)
 177     parser.add_argument('-k', '--kernel', required=True)
 178     parser.add_argument('-lm', '--lmodule', required=True)
 179     parser.add_argument('-tu', '--tools-url', required=True)
 180     parser.add_argument('-tc', '--tools-commit', required=True)
 181     parser.add_argument('-id', '--build-id', required=True)
 182     parser.add_argument('-uu', '--ust-url', required=False)
 183     parser.add_argument('-uc', '--ust-commit', required=False)
 184     parser.add_argument('-d', '--debug', required=False, action='store_true')
 185     args = parser.parse_args()
 186
 187     if args.type not in TestType.values:
 188         print('argument -t/--type {} unrecognized.'.format(args.type))
 189         print('Possible values are:')
 190         for k in TestType.values:
 191             print('\t {}'.format(k))
 192         return -1
 193
 194     lava_api_key = None
 195     if not args.debug:
 196         try:
 197             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 198         except Exception as error:
 199             print(
 200                 'LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...',
 201                 error,
 202             )
 203             return -1
 204
 205     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 206     jinja_env = Environment(loader=jinja_loader, trim_blocks=True, lstrip_blocks=True)
 207     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 208
 209     test_type = TestType.values[args.type]
 210
 211     if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
 212         device_type = DeviceType.x86
 213     else:
 214         device_type = DeviceType.kvm
 215
 216     vlttng_path = '/tmp/virtenv'
 217
 218     vlttng_cmd = get_vlttng_cmd(
 219         args.tools_url, args.tools_commit, args.ust_url, args.ust_commit
 220     )
 221
 222     context = dict()
 223     context['DeviceType'] = DeviceType
 224     context['TestType'] = TestType
 225
 226     context['job_name'] = args.jobname
 227     context['test_type'] = test_type
 228     context['random_seed'] = random.randint(0, 1000000)
 229     context['device_type'] = device_type
 230
 231     context['vlttng_cmd'] = vlttng_cmd
 232     context['vlttng_path'] = vlttng_path
 233
 234     context['kernel_url'] = args.kernel
 235     context['nfsrootfs_url'] = nfsrootfs
 236     context['lttng_modules_url'] = args.lmodule
 237     context['jenkins_build_id'] = args.build_id
 238
 239     context['kprobe_round_nb'] = 10
 240
 241     render = jinja_template.render(context)
 242
 243     print('Job to be submitted:')
 244
 245     print(render)
 246
 247     if args.debug:
 248         return 0
 249
 250     server = xmlrpc.client.ServerProxy(
 251         'http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME)
 252     )
 253
 254     for attempt in range(10):
 255         try:
 256             jobid = server.scheduler.submit_job(render)
 257         except xmlrpc.client.ProtocolError as error:
 258             print(
 259                 'Protocol error on submit, sleeping and retrying. Attempt #{}'.format(
 260                     attempt
 261                 )
 262             )
 263             time.sleep(5)
 264             continue
 265         else:
 266             break
 267
 268     print('Lava jobid:{}'.format(jobid))
 269     print(
 270         'Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(
 271             jobid
 272         )
 273     )
 274
 275     # Check the status of the job every 30 seconds
 276     jobstatus = server.scheduler.job_state(jobid)['job_state']
 277     running = False
 278     while jobstatus in ['Submitted', 'Scheduling', 'Scheduled', 'Running']:
 279         if not running and jobstatus == 'Running':
 280             print('Job started running')
 281             running = True
 282         time.sleep(30)
 283         try:
 284             jobstatus = server.scheduler.job_state(jobid)['job_state']
 285         except xmlrpc.client.ProtocolError as error:
 286             print('Protocol error, retrying')
 287             continue
 288     print('Job ended with {} status.'.format(jobstatus))
 289
 290     if jobstatus != 'Finished':
 291         return -1
 292
 293     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 294         print_test_output(server, jobid)
 295     elif test_type is TestType.baremetal_benchmarks:
 296         fetch_benchmark_results(args.build_id)
 297
 298     passed, failed = check_job_all_test_cases_state_count(server, jobid)
 299     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 300
 301     if failed != 0:
 302         return -1
 303
 304     return 0
 305
 306
 307 if __name__ == "__main__":
 308     sys.exit(main())