scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python3
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import json
  19 import os
  20 import random
  21 import re
  22 import sys
  23 import time
  24 import xmlrpc.client
  25 from urllib.parse import urljoin
  26 from urllib.request import urlretrieve
  27 import yaml
  28 from jinja2 import Environment, FileSystemLoader
  29
  30 USERNAME = 'lava-jenkins'
  31 HOSTNAME = 'lava-master-02.internal.efficios.com'
  32 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  33
  34 def parse_stable_version(stable_version_string):
  35     # Get the major and minor version numbers from the lttng version string.
  36     version_match = re.search('stable-(\d).(\d\d)', stable_version_string)
  37
  38     if version_match is not None:
  39         major_version = int(version_match.group(1))
  40         minor_version = int(version_match.group(2))
  41     else:
  42         # Setting to zero to make the comparison below easier.
  43         major_version = 0
  44         minor_version = 0
  45     return major_version, minor_version
  46
  47
  48 class TestType:
  49     """ Enum like for test type """
  50
  51     baremetal_tests = 1
  52     kvm_tests = 2
  53     values = {
  54         'baremetal-tests': baremetal_tests,
  55         'kvm-tests': kvm_tests,
  56     }
  57
  58
  59 class DeviceType:
  60     """ Enum like for device type """
  61
  62     x86 = 'x86'
  63     kvm = 'qemu'
  64     values = {'kvm': kvm, 'x86': x86}
  65
  66
  67 def get_job_bundle_content(server, job):
  68     try:
  69         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  70         bundle = server.dashboard.get(bundle_sha)
  71     except xmlrpc.client.Fault as error:
  72         print('Error while fetching results bundle', error.faultString)
  73         raise error
  74
  75     return json.loads(bundle['content'])
  76
  77
  78 def check_job_all_test_cases_state_count(server, job):
  79     """
  80     Parse the results bundle to see the run-tests testcase
  81     of the lttng-kernel-tests passed successfully
  82     """
  83     print("Testcase result:")
  84     content = server.results.get_testjob_results_yaml(str(job))
  85     testcases = yaml.unsafe_load(content)
  86
  87     passed_tests = 0
  88     failed_tests = 0
  89     for testcase in testcases:
  90         if testcase['result'] != 'pass':
  91             print(
  92                 "\tFAILED {}\n\t\t See http://{}{}".format(
  93                     testcase['name'], HOSTNAME, testcase['url']
  94                 )
  95             )
  96             failed_tests += 1
  97         else:
  98             passed_tests += 1
  99     return (passed_tests, failed_tests)
 100
 101
 102 def print_test_output(server, job):
 103     """
 104     Parse the attachment of the testcase to fetch the stdout of the test suite
 105     """
 106     job_finished, log = server.scheduler.jobs.logs(str(job))
 107     logs = yaml.unsafe_load(log.data.decode('ascii'))
 108     print_line = False
 109     for line in logs:
 110         if line['lvl'] != 'target':
 111             continue
 112         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 113             print('---- TEST SUITE OUTPUT BEGIN ----')
 114             print_line = True
 115             continue
 116         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 117             print('----- TEST SUITE OUTPUT END -----')
 118             print_line = False
 119             continue
 120         if print_line:
 121             print("{} {}".format(line['dt'], line['msg']))
 122
 123
 124 def get_vlttng_cmd(
 125     lttng_version, lttng_tools_url, lttng_tools_commit, lttng_ust_url=None, lttng_ust_commit=None
 126 ):
 127     """
 128     Return vlttng cmd to be used in the job template for setup.
 129     """
 130
 131     major_version, minor_version = parse_stable_version(lttng_version)
 132
 133     urcu_profile = ""
 134     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 11):
 135         urcu_profile = "urcu-master"
 136     else:
 137         urcu_profile = "urcu-stable-0.12"
 138
 139     vlttng_cmd = (
 140         'vlttng --jobs=$(nproc) --profile ' + urcu_profile
 141         + ' --override projects.babeltrace.build-env.PYTHON=python3'
 142         ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config'
 143         ' --profile babeltrace-stable-1.5'
 144         ' --profile babeltrace-python'
 145         ' --profile lttng-tools-master'
 146         ' --override projects.lttng-tools.source='
 147         + lttng_tools_url
 148         + ' --override projects.lttng-tools.checkout='
 149         + lttng_tools_commit
 150         + ' --profile lttng-tools-no-man-pages'
 151     )
 152
 153     if lttng_ust_commit is not None:
 154         vlttng_cmd += (
 155             ' --profile lttng-ust-master '
 156             ' --override projects.lttng-ust.source='
 157             + lttng_ust_url
 158             + ' --override projects.lttng-ust.checkout='
 159             + lttng_ust_commit
 160             + ' --profile lttng-ust-no-man-pages'
 161         )
 162
 163
 164     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 11):
 165         vlttng_cmd += (
 166             ' --override projects.lttng-tools.configure+=--enable-test-sdt-uprobe'
 167         )
 168
 169     vlttng_path = '/tmp/virtenv'
 170
 171     vlttng_cmd += ' ' + vlttng_path
 172
 173     return vlttng_cmd
 174
 175
 176 def main():
 177     send_retry_limit = 10
 178     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 179     test_type = None
 180     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 181     parser.add_argument('-t', '--type', required=True)
 182     parser.add_argument('-lv', '--lttng-version', required=True)
 183     parser.add_argument('-j', '--jobname', required=True)
 184     parser.add_argument('-k', '--kernel', required=True)
 185     parser.add_argument('-lm', '--lmodule', required=True)
 186     parser.add_argument('-tu', '--tools-url', required=True)
 187     parser.add_argument('-tc', '--tools-commit', required=True)
 188     parser.add_argument('-id', '--build-id', required=True)
 189     parser.add_argument('-uu', '--ust-url', required=False)
 190     parser.add_argument('-uc', '--ust-commit', required=False)
 191     parser.add_argument('-d', '--debug', required=False, action='store_true')
 192     args = parser.parse_args()
 193
 194     if args.type not in TestType.values:
 195         print('argument -t/--type {} unrecognized.'.format(args.type))
 196         print('Possible values are:')
 197         for k in TestType.values:
 198             print('\t {}'.format(k))
 199         return -1
 200
 201     lava_api_key = None
 202     if not args.debug:
 203         try:
 204             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 205         except Exception as error:
 206             print(
 207                 'LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...',
 208                 error,
 209             )
 210             return -1
 211
 212     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 213     jinja_env = Environment(loader=jinja_loader, trim_blocks=True, lstrip_blocks=True)
 214     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 215
 216     test_type = TestType.values[args.type]
 217
 218     if test_type is TestType.baremetal_tests:
 219         device_type = DeviceType.x86
 220     else:
 221         device_type = DeviceType.kvm
 222
 223     vlttng_path = '/tmp/virtenv'
 224
 225     vlttng_cmd = get_vlttng_cmd(
 226         args.lttng_version, args.tools_url, args.tools_commit, args.ust_url, args.ust_commit
 227     )
 228
 229     if args.lttng_version == "master":
 230         lttng_version_string = "master"
 231     elif args.lttng_version == "canary":
 232         lttng_version_string = "2.10"
 233     else:
 234         major, minor = parse_stable_version(args.lttng_version)
 235         lttng_version_string = str(major) + "." + str(minor)
 236
 237
 238     context = dict()
 239     context['DeviceType'] = DeviceType
 240     context['TestType'] = TestType
 241
 242     context['job_name'] = args.jobname
 243     context['test_type'] = test_type
 244     context['random_seed'] = random.randint(0, 1000000)
 245     context['device_type'] = device_type
 246
 247     context['vlttng_cmd'] = vlttng_cmd
 248     context['vlttng_path'] = vlttng_path
 249     context['lttng_version_string'] = lttng_version_string
 250
 251     context['kernel_url'] = args.kernel
 252     context['nfsrootfs_url'] = nfsrootfs
 253     context['lttng_modules_url'] = args.lmodule
 254     context['jenkins_build_id'] = args.build_id
 255
 256     context['kprobe_round_nb'] = 10
 257
 258     render = jinja_template.render(context)
 259
 260     print('Job to be submitted:')
 261
 262     print(render)
 263
 264     if args.debug:
 265         return 0
 266
 267     server = xmlrpc.client.ServerProxy(
 268         'http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME)
 269     )
 270
 271     for attempt in range(1, send_retry_limit + 1):
 272         try:
 273             jobid = server.scheduler.submit_job(render)
 274         except xmlrpc.client.ProtocolError as error:
 275             print(
 276                 'Protocol error on submit, sleeping and retrying. Attempt #{}'.format(
 277                     attempt
 278                 )
 279             )
 280             time.sleep(5)
 281             continue
 282         else:
 283             break
 284     # Early exit when the maximum number of retry is reached.
 285     if attempt == send_retry_limit:
 286             print(
 287                 'Protocol error on submit, maximum number of retry reached ({})'.format(
 288                     attempt
 289                 )
 290             )
 291             return -1
 292
 293     print('Lava jobid:{}'.format(jobid))
 294     print(
 295         'Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(
 296             jobid
 297         )
 298     )
 299
 300     # Check the status of the job every 30 seconds
 301     jobstatus = server.scheduler.job_state(jobid)['job_state']
 302     running = False
 303     while jobstatus in ['Submitted', 'Scheduling', 'Scheduled', 'Running']:
 304         if not running and jobstatus == 'Running':
 305             print('Job started running')
 306             running = True
 307         time.sleep(30)
 308         try:
 309             jobstatus = server.scheduler.job_state(jobid)['job_state']
 310         except xmlrpc.client.ProtocolError as error:
 311             print('Protocol error, retrying')
 312             continue
 313     print('Job ended with {} status.'.format(jobstatus))
 314
 315     if jobstatus != 'Finished':
 316         return -1
 317
 318     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 319         print_test_output(server, jobid)
 320
 321     passed, failed = check_job_all_test_cases_state_count(server, jobid)
 322     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 323
 324     if failed != 0:
 325         return -1
 326
 327     return 0
 328
 329
 330 if __name__ == "__main__":
 331     sys.exit(main())