scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python3
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import json
  19 import os
  20 import random
  21 import re
  22 import sys
  23 import time
  24 import xmlrpc.client
  25 from urllib.parse import urljoin
  26 from urllib.request import urlretrieve
  27 import yaml
  28 from jinja2 import Environment, FileSystemLoader
  29
  30 USERNAME = 'lava-jenkins'
  31 HOSTNAME = 'lava-master-02.internal.efficios.com'
  32 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  33
  34 def parse_stable_version(stable_version_string):
  35     # Get the major and minor version numbers from the lttng version string.
  36     version_match = re.search('stable-(\d).(\d\d)', stable_version_string)
  37
  38     if version_match is not None:
  39         major_version = int(version_match.group(1))
  40         minor_version = int(version_match.group(2))
  41     else:
  42         # Setting to zero to make the comparison below easier.
  43         major_version = 0
  44         minor_version = 0
  45     return major_version, minor_version
  46
  47
  48 class TestType:
  49     """ Enum like for test type """
  50
  51     baremetal_tests = 1
  52     kvm_tests = 2
  53     values = {
  54         'baremetal-tests': baremetal_tests,
  55         'kvm-tests': kvm_tests,
  56     }
  57
  58
  59 class DeviceType:
  60     """ Enum like for device type """
  61
  62     x86 = 'x86'
  63     kvm = 'qemu'
  64     values = {'kvm': kvm, 'x86': x86}
  65
  66
  67 def get_job_bundle_content(server, job):
  68     try:
  69         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  70         bundle = server.dashboard.get(bundle_sha)
  71     except xmlrpc.client.Fault as error:
  72         print('Error while fetching results bundle', error.faultString)
  73         raise error
  74
  75     return json.loads(bundle['content'])
  76
  77
  78 def check_job_all_test_cases_state_count(server, job):
  79     """
  80     Parse the results bundle to see the run-tests testcase
  81     of the lttng-kernel-tests passed successfully
  82     """
  83     print("Testcase result:")
  84     content = server.results.get_testjob_results_yaml(str(job))
  85     testcases = yaml.unsafe_load(content)
  86
  87     passed_tests = 0
  88     failed_tests = 0
  89     for testcase in testcases:
  90         if testcase['result'] != 'pass':
  91             print(
  92                 "\tFAILED {}\n\t\t See http://{}{}".format(
  93                     testcase['name'], HOSTNAME, testcase['url']
  94                 )
  95             )
  96             failed_tests += 1
  97         else:
  98             passed_tests += 1
  99     return (passed_tests, failed_tests)
 100
 101
 102 def print_test_output(server, job):
 103     """
 104     Parse the attachment of the testcase to fetch the stdout of the test suite
 105     """
 106     job_finished, log = server.scheduler.jobs.logs(str(job))
 107     logs = yaml.unsafe_load(log.data.decode('ascii'))
 108     print_line = False
 109     for line in logs:
 110         if line['lvl'] != 'target':
 111             continue
 112         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 113             print('---- TEST SUITE OUTPUT BEGIN ----')
 114             print_line = True
 115             continue
 116         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 117             print('----- TEST SUITE OUTPUT END -----')
 118             print_line = False
 119             continue
 120         if print_line:
 121             print("{} {}".format(line['dt'], line['msg']))
 122
 123
 124 def get_vlttng_cmd(
 125     lttng_version, lttng_tools_url, lttng_tools_commit, lttng_ust_url=None, lttng_ust_commit=None
 126 ):
 127     """
 128     Return vlttng cmd to be used in the job template for setup.
 129     """
 130
 131     vlttng_cmd = (
 132         'vlttng --jobs=$(nproc) --profile urcu-master'
 133         ' --override projects.babeltrace.build-env.PYTHON=python3'
 134         ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config'
 135         ' --profile babeltrace-stable-1.5'
 136         ' --profile babeltrace-python'
 137         ' --profile lttng-tools-master'
 138         ' --override projects.lttng-tools.source='
 139         + lttng_tools_url
 140         + ' --override projects.lttng-tools.checkout='
 141         + lttng_tools_commit
 142         + ' --profile lttng-tools-no-man-pages'
 143     )
 144
 145     if lttng_ust_commit is not None:
 146         vlttng_cmd += (
 147             ' --profile lttng-ust-master '
 148             ' --override projects.lttng-ust.source='
 149             + lttng_ust_url
 150             + ' --override projects.lttng-ust.checkout='
 151             + lttng_ust_commit
 152             + ' --profile lttng-ust-no-man-pages'
 153         )
 154
 155     major_version, minor_version = parse_stable_version(lttng_version)
 156
 157     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 11):
 158         vlttng_cmd += (
 159             ' --override projects.lttng-tools.configure+=--enable-test-sdt-uprobe'
 160         )
 161
 162     vlttng_path = '/tmp/virtenv'
 163
 164     vlttng_cmd += ' ' + vlttng_path
 165
 166     return vlttng_cmd
 167
 168
 169 def main():
 170     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 171     test_type = None
 172     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 173     parser.add_argument('-t', '--type', required=True)
 174     parser.add_argument('-lv', '--lttng-version', required=True)
 175     parser.add_argument('-j', '--jobname', required=True)
 176     parser.add_argument('-k', '--kernel', required=True)
 177     parser.add_argument('-lm', '--lmodule', required=True)
 178     parser.add_argument('-tu', '--tools-url', required=True)
 179     parser.add_argument('-tc', '--tools-commit', required=True)
 180     parser.add_argument('-id', '--build-id', required=True)
 181     parser.add_argument('-uu', '--ust-url', required=False)
 182     parser.add_argument('-uc', '--ust-commit', required=False)
 183     parser.add_argument('-d', '--debug', required=False, action='store_true')
 184     args = parser.parse_args()
 185
 186     if args.type not in TestType.values:
 187         print('argument -t/--type {} unrecognized.'.format(args.type))
 188         print('Possible values are:')
 189         for k in TestType.values:
 190             print('\t {}'.format(k))
 191         return -1
 192
 193     lava_api_key = None
 194     if not args.debug:
 195         try:
 196             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 197         except Exception as error:
 198             print(
 199                 'LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...',
 200                 error,
 201             )
 202             return -1
 203
 204     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 205     jinja_env = Environment(loader=jinja_loader, trim_blocks=True, lstrip_blocks=True)
 206     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 207
 208     test_type = TestType.values[args.type]
 209
 210     if test_type is TestType.baremetal_tests:
 211         device_type = DeviceType.x86
 212     else:
 213         device_type = DeviceType.kvm
 214
 215     vlttng_path = '/tmp/virtenv'
 216
 217     vlttng_cmd = get_vlttng_cmd(
 218         args.lttng_version, args.tools_url, args.tools_commit, args.ust_url, args.ust_commit
 219     )
 220
 221     if args.lttng_version == "master":
 222         lttng_version_string = "master"
 223     elif args.lttng_version == "canary":
 224         lttng_version_string = "2.10"
 225     else:
 226         major, minor = parse_stable_version(args.lttng_version)
 227         lttng_version_string = str(major) + "." + str(minor)
 228
 229
 230     context = dict()
 231     context['DeviceType'] = DeviceType
 232     context['TestType'] = TestType
 233
 234     context['job_name'] = args.jobname
 235     context['test_type'] = test_type
 236     context['random_seed'] = random.randint(0, 1000000)
 237     context['device_type'] = device_type
 238
 239     context['vlttng_cmd'] = vlttng_cmd
 240     context['vlttng_path'] = vlttng_path
 241     context['lttng_version_string'] = lttng_version_string
 242
 243     context['kernel_url'] = args.kernel
 244     context['nfsrootfs_url'] = nfsrootfs
 245     context['lttng_modules_url'] = args.lmodule
 246     context['jenkins_build_id'] = args.build_id
 247
 248     context['kprobe_round_nb'] = 10
 249
 250     render = jinja_template.render(context)
 251
 252     print('Job to be submitted:')
 253
 254     print(render)
 255
 256     if args.debug:
 257         return 0
 258
 259     server = xmlrpc.client.ServerProxy(
 260         'http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME)
 261     )
 262
 263     for attempt in range(10):
 264         try:
 265             jobid = server.scheduler.submit_job(render)
 266         except xmlrpc.client.ProtocolError as error:
 267             print(
 268                 'Protocol error on submit, sleeping and retrying. Attempt #{}'.format(
 269                     attempt
 270                 )
 271             )
 272             time.sleep(5)
 273             continue
 274         else:
 275             break
 276
 277     print('Lava jobid:{}'.format(jobid))
 278     print(
 279         'Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(
 280             jobid
 281         )
 282     )
 283
 284     # Check the status of the job every 30 seconds
 285     jobstatus = server.scheduler.job_state(jobid)['job_state']
 286     running = False
 287     while jobstatus in ['Submitted', 'Scheduling', 'Scheduled', 'Running']:
 288         if not running and jobstatus == 'Running':
 289             print('Job started running')
 290             running = True
 291         time.sleep(30)
 292         try:
 293             jobstatus = server.scheduler.job_state(jobid)['job_state']
 294         except xmlrpc.client.ProtocolError as error:
 295             print('Protocol error, retrying')
 296             continue
 297     print('Job ended with {} status.'.format(jobstatus))
 298
 299     if jobstatus != 'Finished':
 300         return -1
 301
 302     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 303         print_test_output(server, jobid)
 304
 305     passed, failed = check_job_all_test_cases_state_count(server, jobid)
 306     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 307
 308     if failed != 0:
 309         return -1
 310
 311     return 0
 312
 313
 314 if __name__ == "__main__":
 315     sys.exit(main())