scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import base64
  19 import json
  20 import os
  21 import random
  22 import sys
  23 import time
  24 import yaml
  25 import xmlrpc.client
  26 import pprint
  27
  28 from jinja2 import Environment, FileSystemLoader, meta
  29
  30 USERNAME = 'lava-jenkins'
  31 HOSTNAME = 'lava-master-02.internal.efficios.com'
  32
  33 class TestType():
  34     baremetal_benchmarks=1
  35     baremetal_tests=2
  36     kvm_tests=3
  37     kvm_fuzzing_tests=4
  38     values = {
  39         'baremetal-benchmarks' : baremetal_benchmarks,
  40         'baremetal-tests' : baremetal_tests,
  41         'kvm-tests' : kvm_tests,
  42         'kvm-fuzzin-tests' : kvm_fuzzing_tests,
  43     }
  44
  45 class DeviceType():
  46     x86 = 'x86'
  47     kvm = 'qemu'
  48     values = {
  49         'kvm' : kvm,
  50         'x86' : x86,
  51     }
  52
  53 def get_packages():
  54     return ['bsdtar', 'psmisc', 'wget', 'python3', 'python3-pip', \
  55             'libglib2.0-dev', 'libffi-dev', 'elfutils', 'libdw-dev', \
  56             'libelf-dev', 'libmount-dev', 'libxml2', 'libpfm4-dev', \
  57             'libnuma-dev', 'python3-dev', 'swig', 'stress']
  58
  59 def get_job_bundle_content(server, job):
  60     try:
  61         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  62         bundle = server.dashboard.get(bundle_sha)
  63     except xmlrpc.client.Fault as f:
  64         print('Error while fetching results bundle', f.faultString)
  65         raise f
  66
  67     return json.loads(bundle['content'])
  68
  69 # Parse the results bundle to see the run-tests testcase
  70 # of the lttng-kernel-tests passed successfully
  71 def check_job_all_test_cases_state_count(server, job):
  72     content = get_job_bundle_content(server, job)
  73
  74     # FIXME:Those tests are part of the boot actions and fail randomly but
  75     # doesn't affect the behaviour of the tests. We should update our Lava
  76     # installation and try to reproduce it. This error was encountered on
  77     # Ubuntu 16.04.
  78     tests_known_to_fail=['mount', 'df', 'ls', 'ip', 'wait_for_test_image_prompt']
  79
  80     passed_tests=0
  81     failed_tests=0
  82     for run in content['test_runs']:
  83         for result in run['test_results']:
  84             if 'test_case_id' in result :
  85                 if result['result'] in 'pass':
  86                     passed_tests+=1
  87                 elif result['test_case_id'] in tests_known_to_fail:
  88                     pass
  89                 else:
  90                     failed_tests+=1
  91     return (passed_tests, failed_tests)
  92
  93 # Get the benchmark results from the lava bundle
  94 # save them as CSV files localy
  95 def fetch_benchmark_results(server, job):
  96     content = get_job_bundle_content(server, job)
  97     testcases = ['processed_results_close.csv',
  98             'processed_results_ioctl.csv',
  99             'processed_results_open_efault.csv',
 100             'processed_results_open_enoent.csv',
 101             'processed_results_dup_close.csv',
 102             'processed_results_raw_syscall_getpid.csv',
 103             'processed_results_lttng_test_filter.csv']
 104
 105     # The result bundle is a large JSON containing the results of every testcase
 106     # of the LAVA job as well as the files that were attached during the run.
 107     # We need to iterate over this JSON to get the base64 representation of the
 108     # benchmark results produced during the run.
 109     for run in content['test_runs']:
 110         # We only care of the benchmark testcases
 111         if 'benchmark-' in run['test_id']:
 112             if 'test_results' in run:
 113                 for res in run['test_results']:
 114                     if 'attachments' in res:
 115                         for a in res['attachments']:
 116                             # We only save the results file
 117                             if a['pathname'] in testcases:
 118                                 with open(a['pathname'],'wb') as f:
 119                                     # Convert the b64 representation of the
 120                                     # result file and write it to a file
 121                                     # in the current working directory
 122                                     f.write(base64.b64decode(a['content']))
 123
 124 # Parse the attachment of the testcase to fetch the stdout of the test suite
 125 def print_test_output(server, job):
 126     content = get_job_bundle_content(server, job)
 127     found = False
 128
 129     for run in content['test_runs']:
 130         if run['test_id'] in 'lttng-kernel-test':
 131             for attachment in run['attachments']:
 132                 if attachment['pathname'] in 'stdout.log':
 133
 134                     # Decode the base64 file and split on newlines to iterate
 135                     # on list
 136                     testoutput = str(base64.b64decode(bytes(attachment['content'], encoding='UTF-8')))
 137
 138                     testoutput = testoutput.replace('\\n', '\n')
 139
 140                     # Create a generator to iterate on the lines and keeping
 141                     # the state of the iterator across the two loops.
 142                     testoutput_iter = iter(testoutput.split('\n'))
 143                     for line in testoutput_iter:
 144
 145                         # Find the header of the test case and start printing
 146                         # from there
 147                         if 'LAVA_SIGNAL_STARTTC run-tests' in line:
 148                             print('---- TEST SUITE OUTPUT BEGIN ----')
 149                             for line in testoutput_iter:
 150                                 if 'LAVA_SIGNAL_ENDTC run-tests' not in line:
 151                                     print(line)
 152                                 else:
 153                                     # Print until we reach the end of the
 154                                     # section
 155                                     break
 156
 157                             print('----- TEST SUITE OUTPUT END -----')
 158                             break
 159
 160 def get_vlttng_cmd(device, lttng_tools_commit, lttng_ust_commit=None, vlttng_path="/tmp/virtenv"):
 161
 162     vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
 163                     ' --override projects.babeltrace.build-env.PYTHON=python3' \
 164                     ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
 165                     ' --profile babeltrace-stable-1.4' \
 166                     ' --profile babeltrace-python' \
 167                     ' --profile lttng-tools-master' \
 168                     ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
 169                     ' --profile lttng-tools-no-man-pages'
 170
 171     if lttng_ust_commit is not None:
 172         vlttng_cmd += ' --profile lttng-ust-master ' \
 173                     ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
 174                     ' --profile lttng-ust-no-man-pages'
 175
 176     vlttng_cmd += ' ' + vlttng_path
 177
 178     return vlttng_cmd
 179
 180 def main():
 181     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_trusty_2016-02-23-1134.tar.gz"
 182     vlttng_path = '/tmp/virtenv'
 183     test_type = None
 184     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 185     parser.add_argument('-t', '--type', required=True)
 186     parser.add_argument('-j', '--jobname', required=True)
 187     parser.add_argument('-k', '--kernel', required=True)
 188     parser.add_argument('-lm', '--lmodule', required=True)
 189     parser.add_argument('-tc', '--tools-commit', required=True)
 190     parser.add_argument('-uc', '--ust-commit', required=False)
 191     parser.add_argument('-d', '--debug', required=False, action='store_true')
 192     args = parser.parse_args()
 193
 194     if args.type not in TestType.values:
 195         print('argument -t/--type {} unrecognized.'.format(args.type))
 196         print('Possible values are:')
 197         for k in TestType.values:
 198             print('\t {}'.format(k))
 199         return -1
 200
 201     lava_api_key = None
 202     if not args.debug:
 203         try:
 204             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 205         except Exception as e:
 206             print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...', e )
 207             return -1
 208
 209     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 210     jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
 211             lstrip_blocks= True)
 212     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 213     template_source = jinja_env.loader.get_source(jinja_env, 'template_lava_job.jinja2')
 214     parsed_content = jinja_env.parse(template_source)
 215     undef = meta.find_undeclared_variables(parsed_content)
 216
 217     test_type = TestType.values[args.type]
 218
 219     if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
 220         device_type = DeviceType.x86
 221
 222     else:
 223         device_type = DeviceType.kvm
 224
 225     vlttng_cmd = get_vlttng_cmd(device_type, args.tools_commit,
 226             args.ust_commit, vlttng_path=vlttng_path)
 227
 228     context = dict()
 229     context['DeviceType'] = DeviceType
 230     context['TestType'] = TestType
 231
 232     context['job_name'] = args.jobname
 233     context['test_type'] = test_type
 234     context['packages'] = get_packages()
 235     context['random_seed'] = random.randint(0, 1000000)
 236     context['device_type'] = device_type
 237
 238     context['vlttng_cmd'] = vlttng_cmd
 239     context['vlttng_path'] = vlttng_path
 240
 241     context['kernel_url'] = args.kernel
 242     context['nfsrootfs_url'] = nfsrootfs
 243     context['lttng_modules_url'] = args.lmodule
 244
 245     context['kprobe_round_nb'] = 10
 246
 247     render = jinja_template.render(context)
 248
 249     print('Current context:')
 250     pprint.pprint(context, indent=4)
 251     print('Job to be submitted:')
 252
 253     print(render)
 254
 255     if args.debug:
 256         return 0
 257
 258     server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))
 259
 260     jobid = server.scheduler.submit_job(render)
 261
 262     print('Lava jobid:{}'.format(jobid))
 263     print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}/log_file'.format(jobid))
 264
 265     #Check the status of the job every 30 seconds
 266     jobstatus = server.scheduler.job_status(jobid)['job_status']
 267     not_running = False
 268     while jobstatus in 'Submitted' or jobstatus in 'Running':
 269         if not_running is False and jobstatus in 'Running':
 270             print('Job started running')
 271             not_running = True
 272         time.sleep(30)
 273         jobstatus = server.scheduler.job_status(jobid)['job_status']
 274
 275 #    Do not fetch result for now
 276 #    if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 277 #        print_test_output(server, jobid)
 278 #    elif test_type is TestType.baremetal_benchmarks:
 279 #        fetch_benchmark_results(server, jobid)
 280
 281     print('Job ended with {} status.'.format(jobstatus))
 282     if jobstatus not in 'Complete':
 283         return -1
 284     else:
 285         passed, failed=check_job_all_test_cases_state_count(server, jobid)
 286         print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 287
 288         if failed == 0:
 289             return 0
 290         else:
 291             return -1
 292
 293 if __name__ == "__main__":
 294     sys.exit(main())