scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python3
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import json
  19 import os
  20 import random
  21 import re
  22 import sys
  23 import time
  24 import xmlrpc.client
  25 from urllib.parse import urljoin
  26 from urllib.request import urlretrieve
  27 import yaml
  28 from jinja2 import Environment, FileSystemLoader
  29
  30 USERNAME = 'lava-jenkins'
  31 HOSTNAME = 'lava-master-02.internal.efficios.com'
  32 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  33
  34 def parse_stable_version(stable_version_string):
  35     # Get the major and minor version numbers from the lttng version string.
  36     version_match = re.search('stable-(\d).(\d\d)', stable_version_string)
  37
  38     if version_match is not None:
  39         major_version = int(version_match.group(1))
  40         minor_version = int(version_match.group(2))
  41     else:
  42         # Setting to zero to make the comparison below easier.
  43         major_version = 0
  44         minor_version = 0
  45     return major_version, minor_version
  46
  47
  48 class TestType:
  49     """ Enum like for test type """
  50
  51     baremetal_tests = 1
  52     kvm_tests = 2
  53     values = {
  54         'baremetal-tests': baremetal_tests,
  55         'kvm-tests': kvm_tests,
  56     }
  57
  58
  59 class DeviceType:
  60     """ Enum like for device type """
  61
  62     x86 = 'x86'
  63     kvm = 'qemu'
  64     values = {'kvm': kvm, 'x86': x86}
  65
  66
  67 def get_job_bundle_content(server, job):
  68     try:
  69         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  70         bundle = server.dashboard.get(bundle_sha)
  71     except xmlrpc.client.Fault as error:
  72         print('Error while fetching results bundle', error.faultString)
  73         raise error
  74
  75     return json.loads(bundle['content'])
  76
  77
  78 def check_job_all_test_cases_state_count(server, job):
  79     """
  80     Parse the results bundle to see the run-tests testcase
  81     of the lttng-kernel-tests passed successfully
  82     """
  83     print("Testcase result:")
  84     content = server.results.get_testjob_results_yaml(str(job))
  85     testcases = yaml.unsafe_load(content)
  86
  87     passed_tests = 0
  88     failed_tests = 0
  89     for testcase in testcases:
  90         if testcase['result'] != 'pass':
  91             print(
  92                 "\tFAILED {}\n\t\t See http://{}{}".format(
  93                     testcase['name'], HOSTNAME, testcase['url']
  94                 )
  95             )
  96             failed_tests += 1
  97         else:
  98             passed_tests += 1
  99     return (passed_tests, failed_tests)
 100
 101
 102 def print_test_output(server, job):
 103     """
 104     Parse the attachment of the testcase to fetch the stdout of the test suite
 105     """
 106     job_finished, log = server.scheduler.jobs.logs(str(job))
 107     logs = yaml.unsafe_load(log.data.decode('ascii'))
 108     print_line = False
 109     for line in logs:
 110         if line['lvl'] != 'target':
 111             continue
 112         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 113             print('---- TEST SUITE OUTPUT BEGIN ----')
 114             print_line = True
 115             continue
 116         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 117             print('----- TEST SUITE OUTPUT END -----')
 118             print_line = False
 119             continue
 120         if print_line:
 121             print("{} {}".format(line['dt'], line['msg']))
 122
 123
 124 def get_vlttng_cmd(
 125     lttng_version, lttng_tools_url, lttng_tools_commit, lttng_ust_url=None, lttng_ust_commit=None
 126 ):
 127     """
 128     Return vlttng cmd to be used in the job template for setup.
 129     """
 130
 131     major_version, minor_version = parse_stable_version(lttng_version)
 132
 133     urcu_profile = ""
 134     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 11):
 135         urcu_profile = "urcu-master"
 136     else:
 137         urcu_profile = "urcu-stable-0.12"
 138
 139     # Starting with 2.14, babeltrace2 is the reader for testing.
 140     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 14):
 141         babeltrace_profile = " --profile babeltrace2-stable-2.0 --profile babeltrace2-python"
 142         babeltrace_overrides = " --override projects.babeltrace2.build-env.PYTHON=python3 --override projects.babeltrace2.build-env.PYTHON_CONFIG=python3-config -o projects.babeltrace2.configure+=--disable-man-pages"
 143     else:
 144         babeltrace_profile = " --profile babeltrace-stable-1.5 --profile babeltrace-python"
 145         babeltrace_overrides = " --override projects.babeltrace.build-env.PYTHON=python3 --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config"
 146
 147
 148     vlttng_cmd = (
 149         'vlttng --jobs=$(nproc) --profile ' + urcu_profile
 150         + babeltrace_profile
 151         + babeltrace_overrides
 152         + ' --profile lttng-tools-master'
 153         ' --override projects.lttng-tools.source='
 154         + lttng_tools_url
 155         + ' --override projects.lttng-tools.checkout='
 156         + lttng_tools_commit
 157         + ' --profile lttng-tools-no-man-pages'
 158     )
 159
 160     if lttng_ust_commit is not None:
 161         vlttng_cmd += (
 162             ' --profile lttng-ust-master '
 163             ' --override projects.lttng-ust.source='
 164             + lttng_ust_url
 165             + ' --override projects.lttng-ust.checkout='
 166             + lttng_ust_commit
 167             + ' --profile lttng-ust-no-man-pages'
 168         )
 169
 170
 171     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 11):
 172         vlttng_cmd += (
 173             ' --override projects.lttng-tools.configure+=--enable-test-sdt-uprobe'
 174         )
 175
 176     vlttng_path = '/tmp/virtenv'
 177
 178     vlttng_cmd += ' ' + vlttng_path
 179
 180     return vlttng_cmd
 181
 182
 183 def main():
 184     send_retry_limit = 10
 185     test_type = None
 186     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 187     parser.add_argument('-t', '--type', required=True)
 188     parser.add_argument('-lv', '--lttng-version', required=True)
 189     parser.add_argument('-j', '--jobname', required=True)
 190     parser.add_argument('-k', '--kernel', required=True)
 191     parser.add_argument('-lm', '--lmodule', required=True)
 192     parser.add_argument('-tu', '--tools-url', required=True)
 193     parser.add_argument('-tc', '--tools-commit', required=True)
 194     parser.add_argument('-id', '--build-id', required=True)
 195     parser.add_argument('-uu', '--ust-url', required=False)
 196     parser.add_argument('-uc', '--ust-commit', required=False)
 197     parser.add_argument('-d', '--debug', required=False, action='store_true')
 198     parser.add_argument(
 199         '-r', '--rootfs-url', required=False,
 200         default="https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 201     )
 202     parser.add_argument('--ci-repo', required=False, default='https://github.com/lttng/lttng-ci.git')
 203     parser.add_argument('--ci-branch', required=False, default='master')
 204     args = parser.parse_args()
 205
 206     if args.type not in TestType.values:
 207         print('argument -t/--type {} unrecognized.'.format(args.type))
 208         print('Possible values are:')
 209         for k in TestType.values:
 210             print('\t {}'.format(k))
 211         return -1
 212
 213     lava_api_key = None
 214     if not args.debug:
 215         try:
 216             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 217         except Exception as error:
 218             print(
 219                 'LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...',
 220                 error,
 221             )
 222             return -1
 223
 224     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 225     jinja_env = Environment(loader=jinja_loader, trim_blocks=True, lstrip_blocks=True)
 226     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 227
 228     test_type = TestType.values[args.type]
 229
 230     if test_type is TestType.baremetal_tests:
 231         device_type = DeviceType.x86
 232     else:
 233         device_type = DeviceType.kvm
 234
 235     vlttng_path = '/tmp/virtenv'
 236
 237     vlttng_cmd = get_vlttng_cmd(
 238         args.lttng_version, args.tools_url, args.tools_commit, args.ust_url, args.ust_commit
 239     )
 240
 241     if args.lttng_version == "master":
 242         lttng_version_string = "master"
 243     elif args.lttng_version == "canary":
 244         lttng_version_string = "2.10"
 245     else:
 246         major, minor = parse_stable_version(args.lttng_version)
 247         lttng_version_string = str(major) + "." + str(minor)
 248
 249
 250     context = dict()
 251     context['DeviceType'] = DeviceType
 252     context['TestType'] = TestType
 253
 254     context['job_name'] = args.jobname
 255     context['test_type'] = test_type
 256     context['random_seed'] = random.randint(0, 1000000)
 257     context['device_type'] = device_type
 258
 259     context['vlttng_cmd'] = vlttng_cmd
 260     context['vlttng_path'] = vlttng_path
 261     context['lttng_version_string'] = lttng_version_string
 262
 263     context['kernel_url'] = args.kernel
 264     context['nfsrootfs_url'] = args.rootfs_url
 265     context['lttng_modules_url'] = args.lmodule
 266     context['jenkins_build_id'] = args.build_id
 267
 268     context['kprobe_round_nb'] = 10
 269
 270     context['ci_repo'] = args.ci_repo
 271     context['ci_branch'] = args.ci_branch
 272
 273     render = jinja_template.render(context)
 274
 275     print('Job to be submitted:')
 276
 277     print(render)
 278
 279     if args.debug:
 280         return 0
 281
 282     server = xmlrpc.client.ServerProxy(
 283         'http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME)
 284     )
 285
 286     for attempt in range(1, send_retry_limit + 1):
 287         try:
 288             jobid = server.scheduler.submit_job(render)
 289         except xmlrpc.client.ProtocolError as error:
 290             print(
 291                 'Protocol error on submit, sleeping and retrying. Attempt #{}'.format(
 292                     attempt
 293                 )
 294             )
 295             time.sleep(5)
 296             continue
 297         else:
 298             break
 299     # Early exit when the maximum number of retry is reached.
 300     if attempt == send_retry_limit:
 301             print(
 302                 'Protocol error on submit, maximum number of retry reached ({})'.format(
 303                     attempt
 304                 )
 305             )
 306             return -1
 307
 308     print('Lava jobid:{}'.format(jobid))
 309     print(
 310         'Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(
 311             jobid
 312         )
 313     )
 314
 315     # Check the status of the job every 30 seconds
 316     jobstatus = server.scheduler.job_state(jobid)['job_state']
 317     running = False
 318     while jobstatus in ['Submitted', 'Scheduling', 'Scheduled', 'Running']:
 319         if not running and jobstatus == 'Running':
 320             print('Job started running')
 321             running = True
 322         time.sleep(30)
 323         try:
 324             jobstatus = server.scheduler.job_state(jobid)['job_state']
 325         except xmlrpc.client.ProtocolError as error:
 326             print('Protocol error, retrying')
 327             continue
 328     print('Job ended with {} status.'.format(jobstatus))
 329
 330     if jobstatus != 'Finished':
 331         return -1
 332
 333     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 334         print_test_output(server, jobid)
 335
 336     passed, failed = check_job_all_test_cases_state_count(server, jobid)
 337     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 338
 339     if failed != 0:
 340         return -1
 341
 342     return 0
 343
 344
 345 if __name__ == "__main__":
 346     sys.exit(main())