scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python3
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import json
  19 import os
  20 import random
  21 import re
  22 import sys
  23 import time
  24 import xmlrpc.client
  25 from urllib.parse import urljoin
  26 from urllib.request import urlretrieve
  27 import yaml
  28 from jinja2 import Environment, FileSystemLoader
  29
  30 USERNAME = 'lava-jenkins'
  31 HOSTNAME = os.environ.get('LAVA_HOST', 'lava-master-03.internal.efficios.com')
  32 PROTO = os.environ.get('LAVA_PROTO', 'https')
  33 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  34
  35 def parse_stable_version(stable_version_string):
  36     # Get the major and minor version numbers from the lttng version string.
  37     version_match = re.search('stable-(\d).(\d\d)', stable_version_string)
  38
  39     if version_match is not None:
  40         major_version = int(version_match.group(1))
  41         minor_version = int(version_match.group(2))
  42     else:
  43         # Setting to zero to make the comparison below easier.
  44         major_version = 0
  45         minor_version = 0
  46     return major_version, minor_version
  47
  48
  49 class TestType:
  50     """ Enum like for test type """
  51
  52     baremetal_tests = 1
  53     kvm_tests = 2
  54     values = {
  55         'baremetal-tests': baremetal_tests,
  56         'kvm-tests': kvm_tests,
  57     }
  58
  59
  60 class DeviceType:
  61     """ Enum like for device type """
  62
  63     x86 = 'x86'
  64     kvm = 'qemu'
  65     values = {'kvm': kvm, 'x86': x86}
  66
  67
  68 def get_job_bundle_content(server, job):
  69     try:
  70         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  71         bundle = server.dashboard.get(bundle_sha)
  72     except xmlrpc.client.Fault as error:
  73         print('Error while fetching results bundle', error.faultString)
  74         raise error
  75
  76     return json.loads(bundle['content'])
  77
  78
  79 def check_job_all_test_cases_state_count(server, job):
  80     """
  81     Parse the results bundle to see the run-tests testcase
  82     of the lttng-kernel-tests passed successfully
  83     """
  84     print("Testcase result:")
  85     content = server.results.get_testjob_results_yaml(str(job))
  86     testcases = yaml.load(content)
  87
  88     passed_tests = 0
  89     failed_tests = 0
  90     for testcase in testcases:
  91         if testcase['result'] != 'pass':
  92             print(
  93                 "\tFAILED {}\n\t\t See {}://{}{}".format(
  94                     testcase['name'], PROTO, HOSTNAME, testcase['url']
  95                 )
  96             )
  97             failed_tests += 1
  98         else:
  99             passed_tests += 1
 100     return (passed_tests, failed_tests)
 101
 102
 103 def print_test_output(server, job):
 104     """
 105     Parse the attachment of the testcase to fetch the stdout of the test suite
 106     """
 107     job_finished, log = server.scheduler.jobs.logs(str(job))
 108     logs = yaml.load(log.data.decode('ascii'))
 109     print_line = False
 110     for line in logs:
 111         if line['lvl'] != 'target':
 112             continue
 113         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 114             print('---- TEST SUITE OUTPUT BEGIN ----')
 115             print_line = True
 116             continue
 117         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 118             print('----- TEST SUITE OUTPUT END -----')
 119             print_line = False
 120             continue
 121         if print_line:
 122             print("{} {}".format(line['dt'], line['msg']))
 123
 124
 125 def get_vlttng_cmd(
 126     lttng_version, lttng_tools_url, lttng_tools_commit, lttng_ust_url=None, lttng_ust_commit=None
 127 ):
 128     """
 129     Return vlttng cmd to be used in the job template for setup.
 130     """
 131
 132     major_version, minor_version = parse_stable_version(lttng_version)
 133
 134     urcu_profile = ""
 135     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 11):
 136         urcu_profile = "urcu-master"
 137     else:
 138         urcu_profile = "urcu-stable-0.12"
 139
 140     # Starting with 2.14, babeltrace2 is the reader for testing.
 141     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 14):
 142         babeltrace_profile = " --profile babeltrace2-stable-2.0 --profile babeltrace2-python"
 143         babeltrace_overrides = " --override projects.babeltrace2.build-env.PYTHON=python3 --override projects.babeltrace2.build-env.PYTHON_CONFIG=python3-config -o projects.babeltrace2.configure+=--disable-man-pages"
 144     else:
 145         babeltrace_profile = " --profile babeltrace-stable-1.5 --profile babeltrace-python"
 146         babeltrace_overrides = " --override projects.babeltrace.build-env.PYTHON=python3 --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config"
 147
 148
 149     vlttng_cmd = (
 150         'vlttng --jobs=$(nproc) --profile ' + urcu_profile
 151         + babeltrace_profile
 152         + babeltrace_overrides
 153         + ' --profile lttng-tools-master'
 154         ' --override projects.lttng-tools.source='
 155         + lttng_tools_url
 156         + ' --override projects.lttng-tools.checkout='
 157         + lttng_tools_commit
 158         + ' --profile lttng-tools-no-man-pages'
 159     )
 160
 161     if lttng_ust_commit is not None:
 162         vlttng_cmd += (
 163             ' --profile lttng-ust-master '
 164             ' --override projects.lttng-ust.source='
 165             + lttng_ust_url
 166             + ' --override projects.lttng-ust.checkout='
 167             + lttng_ust_commit
 168             + ' --profile lttng-ust-no-man-pages'
 169         )
 170
 171
 172     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 11):
 173         vlttng_cmd += (
 174             ' --override projects.lttng-tools.configure+=--enable-test-sdt-uprobe'
 175         )
 176
 177     vlttng_path = '/tmp/virtenv'
 178
 179     vlttng_cmd += ' ' + vlttng_path
 180
 181     return vlttng_cmd
 182
 183
 184 def main():
 185     send_retry_limit = 10
 186     test_type = None
 187     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 188     parser.add_argument('-t', '--type', required=True)
 189     parser.add_argument('-lv', '--lttng-version', required=True)
 190     parser.add_argument('-j', '--jobname', required=True)
 191     parser.add_argument('-k', '--kernel', required=True)
 192     parser.add_argument('-lm', '--lmodule', required=True)
 193     parser.add_argument('-tu', '--tools-url', required=True)
 194     parser.add_argument('-tc', '--tools-commit', required=True)
 195     parser.add_argument('-id', '--build-id', required=True)
 196     parser.add_argument('-uu', '--ust-url', required=False)
 197     parser.add_argument('-uc', '--ust-commit', required=False)
 198     parser.add_argument('-d', '--debug', required=False, action='store_true')
 199     parser.add_argument(
 200         '-r', '--rootfs-url', required=False,
 201         default="https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_jammy_2023-05-18.tar.gz"
 202     )
 203     parser.add_argument('--ci-repo', required=False, default='https://github.com/lttng/lttng-ci.git')
 204     parser.add_argument('--ci-branch', required=False, default='master')
 205     args = parser.parse_args()
 206
 207     if args.type not in TestType.values:
 208         print('argument -t/--type {} unrecognized.'.format(args.type))
 209         print('Possible values are:')
 210         for k in TestType.values:
 211             print('\t {}'.format(k))
 212         return -1
 213
 214     lava_api_key = None
 215     if not args.debug:
 216         try:
 217             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 218         except Exception as error:
 219             print(
 220                 'LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...',
 221                 error,
 222             )
 223             return -1
 224
 225     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 226     jinja_env = Environment(loader=jinja_loader, trim_blocks=True, lstrip_blocks=True)
 227     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 228
 229     test_type = TestType.values[args.type]
 230
 231     if test_type is TestType.baremetal_tests:
 232         device_type = DeviceType.x86
 233     else:
 234         device_type = DeviceType.kvm
 235
 236     vlttng_path = '/tmp/virtenv'
 237
 238     vlttng_cmd = get_vlttng_cmd(
 239         args.lttng_version, args.tools_url, args.tools_commit, args.ust_url, args.ust_commit
 240     )
 241
 242     if args.lttng_version == "master":
 243         lttng_version_string = "master"
 244     elif args.lttng_version == "canary":
 245         lttng_version_string = "2.13"
 246     else:
 247         major, minor = parse_stable_version(args.lttng_version)
 248         lttng_version_string = str(major) + "." + str(minor)
 249
 250
 251     context = dict()
 252     context['DeviceType'] = DeviceType
 253     context['TestType'] = TestType
 254
 255     context['job_name'] = args.jobname
 256     context['test_type'] = test_type
 257     context['random_seed'] = random.randint(0, 1000000)
 258     context['device_type'] = device_type
 259
 260     context['vlttng_cmd'] = vlttng_cmd
 261     context['vlttng_path'] = vlttng_path
 262     context['lttng_version_string'] = lttng_version_string
 263
 264     context['kernel_url'] = args.kernel
 265     context['nfsrootfs_url'] = args.rootfs_url
 266     context['lttng_modules_url'] = args.lmodule
 267     context['jenkins_build_id'] = args.build_id
 268
 269     context['kprobe_round_nb'] = 10
 270
 271     context['ci_repo'] = args.ci_repo
 272     context['ci_branch'] = args.ci_branch
 273
 274     render = jinja_template.render(context)
 275
 276     print('Job to be submitted:')
 277
 278     print(render)
 279
 280     if args.debug:
 281         return 0
 282
 283     server = xmlrpc.client.ServerProxy(
 284         '%s://%s:%s@%s/RPC2' % (PROTO, USERNAME, lava_api_key, HOSTNAME)
 285     )
 286
 287     for attempt in range(1, send_retry_limit + 1):
 288         try:
 289             jobid = server.scheduler.submit_job(render)
 290         except xmlrpc.client.ProtocolError as error:
 291             print(
 292                 'Protocol error on submit, sleeping and retrying. Attempt #{}'.format(
 293                     attempt
 294                 )
 295             )
 296             time.sleep(5)
 297             continue
 298         else:
 299             break
 300     # Early exit when the maximum number of retry is reached.
 301     if attempt == send_retry_limit:
 302             print(
 303                 'Protocol error on submit, maximum number of retry reached ({})'.format(
 304                     attempt
 305                 )
 306             )
 307             return -1
 308
 309     print('Lava jobid:{}'.format(jobid))
 310     print(
 311         'Lava job URL: {}://{}/scheduler/job/{}'.format(
 312             PROTO, HOSTNAME, jobid
 313         )
 314     )
 315
 316     # Check the status of the job every 30 seconds
 317     jobstatus = server.scheduler.job_state(jobid)['job_state']
 318     running = False
 319     while jobstatus in ['Submitted', 'Scheduling', 'Scheduled', 'Running']:
 320         if not running and jobstatus == 'Running':
 321             print('Job started running')
 322             running = True
 323         time.sleep(30)
 324         try:
 325             jobstatus = server.scheduler.job_state(jobid)['job_state']
 326         except xmlrpc.client.ProtocolError as error:
 327             print('Protocol error, retrying')
 328             continue
 329     print('Job ended with {} status.'.format(jobstatus))
 330
 331     if jobstatus != 'Finished':
 332         return -1
 333
 334     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 335         print_test_output(server, jobid)
 336
 337     passed, failed = check_job_all_test_cases_state_count(server, jobid)
 338     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 339
 340     if failed != 0:
 341         return -1
 342
 343     return 0
 344
 345
 346 if __name__ == "__main__":
 347     sys.exit(main())