scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python3
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import json
  19 import os
  20 import random
  21 import re
  22 import sys
  23 import time
  24 import xmlrpc.client
  25 from urllib.parse import urljoin
  26 from urllib.request import urlretrieve
  27 import yaml
  28 from jinja2 import Environment, FileSystemLoader
  29
  30 USERNAME = 'lava-jenkins'
  31 HOSTNAME = 'lava-master-02.internal.efficios.com'
  32 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  33
  34
  35 class TestType:
  36     """ Enum like for test type """
  37
  38     baremetal_tests = 1
  39     kvm_tests = 2
  40     values = {
  41         'baremetal-tests': baremetal_tests,
  42         'kvm-tests': kvm_tests,
  43     }
  44
  45
  46 class DeviceType:
  47     """ Enum like for device type """
  48
  49     x86 = 'x86'
  50     kvm = 'qemu'
  51     values = {'kvm': kvm, 'x86': x86}
  52
  53
  54 def get_job_bundle_content(server, job):
  55     try:
  56         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  57         bundle = server.dashboard.get(bundle_sha)
  58     except xmlrpc.client.Fault as error:
  59         print('Error while fetching results bundle', error.faultString)
  60         raise error
  61
  62     return json.loads(bundle['content'])
  63
  64
  65 def check_job_all_test_cases_state_count(server, job):
  66     """
  67     Parse the results bundle to see the run-tests testcase
  68     of the lttng-kernel-tests passed successfully
  69     """
  70     print("Testcase result:")
  71     content = server.results.get_testjob_results_yaml(str(job))
  72     testcases = yaml.unsafe_load(content)
  73
  74     passed_tests = 0
  75     failed_tests = 0
  76     for testcase in testcases:
  77         if testcase['result'] != 'pass':
  78             print(
  79                 "\tFAILED {}\n\t\t See http://{}{}".format(
  80                     testcase['name'], HOSTNAME, testcase['url']
  81                 )
  82             )
  83             failed_tests += 1
  84         else:
  85             passed_tests += 1
  86     return (passed_tests, failed_tests)
  87
  88
  89 def print_test_output(server, job):
  90     """
  91     Parse the attachment of the testcase to fetch the stdout of the test suite
  92     """
  93     job_finished, log = server.scheduler.jobs.logs(str(job))
  94     logs = yaml.unsafe_load(log.data.decode('ascii'))
  95     print_line = False
  96     for line in logs:
  97         if line['lvl'] != 'target':
  98             continue
  99         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 100             print('---- TEST SUITE OUTPUT BEGIN ----')
 101             print_line = True
 102             continue
 103         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 104             print('----- TEST SUITE OUTPUT END -----')
 105             print_line = False
 106             continue
 107         if print_line:
 108             print("{} {}".format(line['dt'], line['msg']))
 109
 110
 111 def get_vlttng_cmd(
 112     lttng_version, lttng_tools_url, lttng_tools_commit, lttng_ust_url=None, lttng_ust_commit=None
 113 ):
 114     """
 115     Return vlttng cmd to be used in the job template for setup.
 116     """
 117
 118     vlttng_cmd = (
 119         'vlttng --jobs=$(nproc) --profile urcu-master'
 120         ' --override projects.babeltrace.build-env.PYTHON=python3'
 121         ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config'
 122         ' --profile babeltrace-stable-1.4'
 123         ' --profile babeltrace-python'
 124         ' --profile lttng-tools-master'
 125         ' --override projects.lttng-tools.source='
 126         + lttng_tools_url
 127         + ' --override projects.lttng-tools.checkout='
 128         + lttng_tools_commit
 129         + ' --profile lttng-tools-no-man-pages'
 130     )
 131
 132     if lttng_ust_commit is not None:
 133         vlttng_cmd += (
 134             ' --profile lttng-ust-master '
 135             ' --override projects.lttng-ust.source='
 136             + lttng_ust_url
 137             + ' --override projects.lttng-ust.checkout='
 138             + lttng_ust_commit
 139             + ' --profile lttng-ust-no-man-pages'
 140         )
 141
 142
 143     # Get the major and minor version numbers from the lttng version string.
 144     version_match = re.search('stable-(\d).(\d\d)', lttng_version)
 145
 146     if version_match is not None:
 147         major_version = int(version_match.group(1))
 148         minor_version = int(version_match.group(2))
 149     else:
 150         # Setting to zero to make the comparison below easier.
 151         major_version = 0
 152         minor_version = 0
 153
 154     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 11):
 155         vlttng_cmd += (
 156             ' --override projects.lttng-tools.configure+=--enable-test-sdt-uprobe'
 157         )
 158
 159     vlttng_path = '/tmp/virtenv'
 160
 161     vlttng_cmd += ' ' + vlttng_path
 162
 163     return vlttng_cmd
 164
 165
 166 def main():
 167     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 168     test_type = None
 169     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 170     parser.add_argument('-t', '--type', required=True)
 171     parser.add_argument('-lv', '--lttng-version', required=True)
 172     parser.add_argument('-j', '--jobname', required=True)
 173     parser.add_argument('-k', '--kernel', required=True)
 174     parser.add_argument('-lm', '--lmodule', required=True)
 175     parser.add_argument('-tu', '--tools-url', required=True)
 176     parser.add_argument('-tc', '--tools-commit', required=True)
 177     parser.add_argument('-id', '--build-id', required=True)
 178     parser.add_argument('-uu', '--ust-url', required=False)
 179     parser.add_argument('-uc', '--ust-commit', required=False)
 180     parser.add_argument('-d', '--debug', required=False, action='store_true')
 181     args = parser.parse_args()
 182
 183     if args.type not in TestType.values:
 184         print('argument -t/--type {} unrecognized.'.format(args.type))
 185         print('Possible values are:')
 186         for k in TestType.values:
 187             print('\t {}'.format(k))
 188         return -1
 189
 190     lava_api_key = None
 191     if not args.debug:
 192         try:
 193             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 194         except Exception as error:
 195             print(
 196                 'LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...',
 197                 error,
 198             )
 199             return -1
 200
 201     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 202     jinja_env = Environment(loader=jinja_loader, trim_blocks=True, lstrip_blocks=True)
 203     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 204
 205     test_type = TestType.values[args.type]
 206
 207     if test_type is TestType.baremetal_tests:
 208         device_type = DeviceType.x86
 209     else:
 210         device_type = DeviceType.kvm
 211
 212     vlttng_path = '/tmp/virtenv'
 213
 214     vlttng_cmd = get_vlttng_cmd(
 215         args.lttng_version, args.tools_url, args.tools_commit, args.ust_url, args.ust_commit
 216     )
 217
 218     context = dict()
 219     context['DeviceType'] = DeviceType
 220     context['TestType'] = TestType
 221
 222     context['job_name'] = args.jobname
 223     context['test_type'] = test_type
 224     context['random_seed'] = random.randint(0, 1000000)
 225     context['device_type'] = device_type
 226
 227     context['vlttng_cmd'] = vlttng_cmd
 228     context['vlttng_path'] = vlttng_path
 229
 230     context['kernel_url'] = args.kernel
 231     context['nfsrootfs_url'] = nfsrootfs
 232     context['lttng_modules_url'] = args.lmodule
 233     context['jenkins_build_id'] = args.build_id
 234
 235     context['kprobe_round_nb'] = 10
 236
 237     render = jinja_template.render(context)
 238
 239     print('Job to be submitted:')
 240
 241     print(render)
 242
 243     if args.debug:
 244         return 0
 245
 246     server = xmlrpc.client.ServerProxy(
 247         'http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME)
 248     )
 249
 250     for attempt in range(10):
 251         try:
 252             jobid = server.scheduler.submit_job(render)
 253         except xmlrpc.client.ProtocolError as error:
 254             print(
 255                 'Protocol error on submit, sleeping and retrying. Attempt #{}'.format(
 256                     attempt
 257                 )
 258             )
 259             time.sleep(5)
 260             continue
 261         else:
 262             break
 263
 264     print('Lava jobid:{}'.format(jobid))
 265     print(
 266         'Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(
 267             jobid
 268         )
 269     )
 270
 271     # Check the status of the job every 30 seconds
 272     jobstatus = server.scheduler.job_state(jobid)['job_state']
 273     running = False
 274     while jobstatus in ['Submitted', 'Scheduling', 'Scheduled', 'Running']:
 275         if not running and jobstatus == 'Running':
 276             print('Job started running')
 277             running = True
 278         time.sleep(30)
 279         try:
 280             jobstatus = server.scheduler.job_state(jobid)['job_state']
 281         except xmlrpc.client.ProtocolError as error:
 282             print('Protocol error, retrying')
 283             continue
 284     print('Job ended with {} status.'.format(jobstatus))
 285
 286     if jobstatus != 'Finished':
 287         return -1
 288
 289     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 290         print_test_output(server, jobid)
 291
 292     passed, failed = check_job_all_test_cases_state_count(server, jobid)
 293     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 294
 295     if failed != 0:
 296         return -1
 297
 298     return 0
 299
 300
 301 if __name__ == "__main__":
 302     sys.exit(main())