[lttng-ci.git] / scripts / system-tests / lava2-submit.py

#!/usr/bin/python
# Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import argparse
import base64
import json
import os
import random
import sys
import time
import yaml
import xmlrpc.client
import pprint

from jinja2 import Environment, FileSystemLoader, meta

USERNAME = 'lava-jenkins'
HOSTNAME = 'lava-master-02.internal.efficios.com'

class TestType():
    baremetal_benchmarks=1
    baremetal_tests=2
    kvm_tests=3
    kvm_fuzzing_tests=4
    values = {
        'baremetal-benchmarks' : baremetal_benchmarks,
        'baremetal-tests' : baremetal_tests,
        'kvm-tests' : kvm_tests,
        'kvm-fuzzin-tests' : kvm_fuzzing_tests,
    }

class DeviceType():
    x86 = 'x86'
    kvm = 'qemu'
    values = {
        'kvm' : kvm,
        'x86' : x86,
    }

def get_job_bundle_content(server, job):
    try:
        bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
        bundle = server.dashboard.get(bundle_sha)
    except xmlrpc.client.Fault as f:
        print('Error while fetching results bundle', f.faultString)
        raise f

    return json.loads(bundle['content'])

# Parse the results bundle to see the run-tests testcase
# of the lttng-kernel-tests passed successfully
def check_job_all_test_cases_state_count(server, job):
    content = get_job_bundle_content(server, job)

    # FIXME:Those tests are part of the boot actions and fail randomly but
    # doesn't affect the behaviour of the tests. We should update our Lava
    # installation and try to reproduce it. This error was encountered on
    # Ubuntu 16.04.
    tests_known_to_fail=['mount', 'df', 'ls', 'ip', 'wait_for_test_image_prompt']

    passed_tests=0
    failed_tests=0
    for run in content['test_runs']:
        for result in run['test_results']:
            if 'test_case_id' in result :
                if result['result'] in 'pass':
                    passed_tests+=1
                elif result['test_case_id'] in tests_known_to_fail:
                    pass
                else:
                    failed_tests+=1
    return (passed_tests, failed_tests)

# Get the benchmark results from the lava bundle
# save them as CSV files localy
def fetch_benchmark_results(server, job):
    content = get_job_bundle_content(server, job)
    testcases = ['processed_results_close.csv',
            'processed_results_ioctl.csv',
            'processed_results_open_efault.csv',
            'processed_results_open_enoent.csv',
            'processed_results_dup_close.csv',
            'processed_results_raw_syscall_getpid.csv',
            'processed_results_lttng_test_filter.csv']

    # The result bundle is a large JSON containing the results of every testcase
    # of the LAVA job as well as the files that were attached during the run.
    # We need to iterate over this JSON to get the base64 representation of the
    # benchmark results produced during the run.
    for run in content['test_runs']:
        # We only care of the benchmark testcases
        if 'benchmark-' in run['test_id']:
            if 'test_results' in run:
                for res in run['test_results']:
                    if 'attachments' in res:
                        for a in res['attachments']:
                            # We only save the results file
                            if a['pathname'] in testcases:
                                with open(a['pathname'],'wb') as f:
                                    # Convert the b64 representation of the
                                    # result file and write it to a file
                                    # in the current working directory
                                    f.write(base64.b64decode(a['content']))

# Parse the attachment of the testcase to fetch the stdout of the test suite
def print_test_output(server, job):
    content = get_job_bundle_content(server, job)
    found = False

    for run in content['test_runs']:
        if run['test_id'] in 'lttng-kernel-test':
            for attachment in run['attachments']:
                if attachment['pathname'] in 'stdout.log':

                    # Decode the base64 file and split on newlines to iterate
                    # on list
                    testoutput = str(base64.b64decode(bytes(attachment['content'], encoding='UTF-8')))

                    testoutput = testoutput.replace('\\n', '\n')

                    # Create a generator to iterate on the lines and keeping
                    # the state of the iterator across the two loops.
                    testoutput_iter = iter(testoutput.split('\n'))
                    for line in testoutput_iter:

                        # Find the header of the test case and start printing
                        # from there
                        if 'LAVA_SIGNAL_STARTTC run-tests' in line:
                            print('---- TEST SUITE OUTPUT BEGIN ----')
                            for line in testoutput_iter:
                                if 'LAVA_SIGNAL_ENDTC run-tests' not in line:
                                    print(line)
                                else:
                                    # Print until we reach the end of the
                                    # section
                                    break

                            print('----- TEST SUITE OUTPUT END -----')
                            break

def get_vlttng_cmd(device, lttng_tools_commit, lttng_ust_commit=None):

    vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
                    ' --override projects.babeltrace.build-env.PYTHON=python3' \
                    ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
                    ' --profile babeltrace-stable-1.4' \
                    ' --profile babeltrace-python' \
                    ' --profile lttng-tools-master' \
                    ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
                    ' --profile lttng-tools-no-man-pages'

    if lttng_ust_commit is not None:
        vlttng_cmd += ' --profile lttng-ust-master ' \
                    ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
                    ' --profile lttng-ust-no-man-pages'

    if device is DeviceType.kvm:
        vlttng_path = '/root/virtenv'
    else:
        vlttng_path = '/tmp/virtenv'

    vlttng_cmd += ' ' + vlttng_path

    return vlttng_cmd

def main():
    nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_bionic_2018-11-29.tar.gz"
    test_type = None
    parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
    parser.add_argument('-t', '--type', required=True)
    parser.add_argument('-j', '--jobname', required=True)
    parser.add_argument('-k', '--kernel', required=True)
    parser.add_argument('-lm', '--lmodule', required=True)
    parser.add_argument('-tc', '--tools-commit', required=True)
    parser.add_argument('-id', '--build-id', required=True)
    parser.add_argument('-uc', '--ust-commit', required=False)
    parser.add_argument('-d', '--debug', required=False, action='store_true')
    args = parser.parse_args()

    if args.type not in TestType.values:
        print('argument -t/--type {} unrecognized.'.format(args.type))
        print('Possible values are:')
        for k in TestType.values:
            print('\t {}'.format(k))
        return -1

    lava_api_key = None
    if not args.debug:
        try:
            lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
        except Exception as e:
            print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...', e )
            return -1

    jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
    jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
            lstrip_blocks= True)
    jinja_template = jinja_env.get_template('template_lava_job.jinja2')
    template_source = jinja_env.loader.get_source(jinja_env, 'template_lava_job.jinja2')
    parsed_content = jinja_env.parse(template_source)
    undef = meta.find_undeclared_variables(parsed_content)

    test_type = TestType.values[args.type]

    if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
        device_type = DeviceType.x86
        vlttng_path = '/tmp/virtenv'

    else:
        device_type = DeviceType.kvm
        vlttng_path = '/root/virtenv'

    vlttng_cmd = get_vlttng_cmd(device_type, args.tools_commit, args.ust_commit)

    context = dict()
    context['DeviceType'] = DeviceType
    context['TestType'] = TestType

    context['job_name'] = args.jobname
    context['test_type'] = test_type
    context['random_seed'] = random.randint(0, 1000000)
    context['device_type'] = device_type

    context['vlttng_cmd'] = vlttng_cmd
    context['vlttng_path'] = vlttng_path

    context['kernel_url'] = args.kernel
    context['nfsrootfs_url'] = nfsrootfs
    context['lttng_modules_url'] = args.lmodule
    context['jenkins_build_id'] = args.build_id

    context['kprobe_round_nb'] = 10

    render = jinja_template.render(context)

    print('Current context:')
    pprint.pprint(context, indent=4)
    print('Job to be submitted:')

    print(render)

    if args.debug:
        return 0

    server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))

    jobid = server.scheduler.submit_job(render)

    print('Lava jobid:{}'.format(jobid))
    print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}/log_file'.format(jobid))

    #Check the status of the job every 30 seconds
    jobstatus = server.scheduler.job_status(jobid)['job_status']
    not_running = False
    while jobstatus in 'Submitted' or jobstatus in 'Running':
        if not_running is False and jobstatus in 'Running':
            print('Job started running')
            not_running = True
        time.sleep(30)
        jobstatus = server.scheduler.job_status(jobid)['job_status']

#    Do not fetch result for now
#    if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
#        print_test_output(server, jobid)
#    elif test_type is TestType.baremetal_benchmarks:
#        fetch_benchmark_results(server, jobid)

    print('Job ended with {} status.'.format(jobstatus))
    if jobstatus not in 'Complete':
        return -1
    else:
        passed, failed=check_job_all_test_cases_state_count(server, jobid)
        print('With {} passed and {} failed Lava test cases.'.format(passed, failed))

        if failed == 0:
            return 0
        else:
            return -1

if __name__ == "__main__":
    sys.exit(main())
Commit	Line	Data
878b4840 JR	1	#!/usr/bin/python
	2	# Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
	3	#
	4	# This program is free software: you can redistribute it and/or modify
	5	# it under the terms of the GNU General Public License as published by
	6	# the Free Software Foundation, either version 3 of the License, or
	7	# (at your option) any later version.
	8	#
	9	# This program is distributed in the hope that it will be useful,
	10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	# GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License
	15	# along with this program. If not, see <http://www.gnu.org/licenses/>.
	16
	17	import argparse
	18	import base64
	19	import json
	20	import os
	21	import random
	22	import sys
	23	import time
4cb5cc4f	24	import yaml
878b4840	25	import xmlrpc.client
ef84c6ec JR	26	import pprint
ef84c6ec JR	27
4cb5cc4f	28	from jinja2 import Environment, FileSystemLoader, meta
878b4840	29
ef84c6ec JR	30	USERNAME = 'lava-jenkins'
ef84c6ec JR	31	HOSTNAME = 'lava-master-02.internal.efficios.com'
878b4840 JR	32
	33	class TestType():
	34	baremetal_benchmarks=1
	35	baremetal_tests=2
	36	kvm_tests=3
	37	kvm_fuzzing_tests=4
	38	values = {
	39	'baremetal-benchmarks' : baremetal_benchmarks,
	40	'baremetal-tests' : baremetal_tests,
	41	'kvm-tests' : kvm_tests,
	42	'kvm-fuzzin-tests' : kvm_fuzzing_tests,
	43	}
	44
4cb5cc4f JR	45	class DeviceType():
4cb5cc4f JR	46	x86 = 'x86'
f9a184a9	47	kvm = 'qemu'
4cb5cc4f JR	48	values = {
	49	'kvm' : kvm,
	50	'x86' : x86,
	51	}
	52
878b4840 JR	53	def get_job_bundle_content(server, job):
	54	try:
	55	bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
	56	bundle = server.dashboard.get(bundle_sha)
	57	except xmlrpc.client.Fault as f:
	58	print('Error while fetching results bundle', f.faultString)
	59	raise f
	60
	61	return json.loads(bundle['content'])
	62
	63	# Parse the results bundle to see the run-tests testcase
	64	# of the lttng-kernel-tests passed successfully
	65	def check_job_all_test_cases_state_count(server, job):
	66	content = get_job_bundle_content(server, job)
	67
	68	# FIXME:Those tests are part of the boot actions and fail randomly but
	69	# doesn't affect the behaviour of the tests. We should update our Lava
	70	# installation and try to reproduce it. This error was encountered on
	71	# Ubuntu 16.04.
	72	tests_known_to_fail=['mount', 'df', 'ls', 'ip', 'wait_for_test_image_prompt']
	73
	74	passed_tests=0
	75	failed_tests=0
	76	for run in content['test_runs']:
	77	for result in run['test_results']:
	78	if 'test_case_id' in result :
	79	if result['result'] in 'pass':
	80	passed_tests+=1
	81	elif result['test_case_id'] in tests_known_to_fail:
	82	pass
	83	else:
	84	failed_tests+=1
	85	return (passed_tests, failed_tests)
	86
	87	# Get the benchmark results from the lava bundle
	88	# save them as CSV files localy
	89	def fetch_benchmark_results(server, job):
	90	content = get_job_bundle_content(server, job)
	91	testcases = ['processed_results_close.csv',
	92	'processed_results_ioctl.csv',
	93	'processed_results_open_efault.csv',
	94	'processed_results_open_enoent.csv',
	95	'processed_results_dup_close.csv',
	96	'processed_results_raw_syscall_getpid.csv',
	97	'processed_results_lttng_test_filter.csv']
	98
	99	# The result bundle is a large JSON containing the results of every testcase
	100	# of the LAVA job as well as the files that were attached during the run.
	101	# We need to iterate over this JSON to get the base64 representation of the
	102	# benchmark results produced during the run.
	103	for run in content['test_runs']:
	104	# We only care of the benchmark testcases
	105	if 'benchmark-' in run['test_id']:
	106	if 'test_results' in run:
	107	for res in run['test_results']:
	108	if 'attachments' in res:
	109	for a in res['attachments']:
	110	# We only save the results file
	111	if a['pathname'] in testcases:
	112	with open(a['pathname'],'wb') as f:
	113	# Convert the b64 representation of the
	114	# result file and write it to a file
	115	# in the current working directory
	116	f.write(base64.b64decode(a['content']))
117
118	# Parse the attachment of the testcase to fetch the stdout of the test suite
119	def print_test_output(server, job):
120	content = get_job_bundle_content(server, job)
121	found = False
122
123	for run in content['test_runs']:
124	if run['test_id'] in 'lttng-kernel-test':
125	for attachment in run['attachments']:
126	if attachment['pathname'] in 'stdout.log':
127
128	# Decode the base64 file and split on newlines to iterate
129	# on list
130	testoutput = str(base64.b64decode(bytes(attachment['content'], encoding='UTF-8')))
131
132	testoutput = testoutput.replace('\\n', '\n')
133
134	# Create a generator to iterate on the lines and keeping
135	# the state of the iterator across the two loops.
136	testoutput_iter = iter(testoutput.split('\n'))
137	for line in testoutput_iter:
138
139	# Find the header of the test case and start printing
140	# from there
141	if 'LAVA_SIGNAL_STARTTC run-tests' in line:
142	print('---- TEST SUITE OUTPUT BEGIN ----')
143	for line in testoutput_iter:
144	if 'LAVA_SIGNAL_ENDTC run-tests' not in line:
145	print(line)
146	else:
147	# Print until we reach the end of the
148	# section
149	break
150
151	print('----- TEST SUITE OUTPUT END -----')
152	break
153
c11ec858	154	def get_vlttng_cmd(device, lttng_tools_commit, lttng_ust_commit=None):
878b4840 JR	155
	156	vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
	157	' --override projects.babeltrace.build-env.PYTHON=python3' \
	158	' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
	159	' --profile babeltrace-stable-1.4' \
	160	' --profile babeltrace-python' \
	161	' --profile lttng-tools-master' \
	162	' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
	163	' --profile lttng-tools-no-man-pages'
	164
	165	if lttng_ust_commit is not None:
	166	vlttng_cmd += ' --profile lttng-ust-master ' \
	167	' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
	168	' --profile lttng-ust-no-man-pages'
	169
c11ec858 JR	170	if device is DeviceType.kvm:
	171	vlttng_path = '/root/virtenv'
	172	else:
	173	vlttng_path = '/tmp/virtenv'
	174
4cb5cc4f	175	vlttng_cmd += ' ' + vlttng_path
878b4840	176
4cb5cc4f	177	return vlttng_cmd
878b4840 JR	178
878b4840 JR	179	def main():
ca08dcdf	180	nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_bionic_2018-11-29.tar.gz"
878b4840 JR	181	test_type = None
	182	parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
	183	parser.add_argument('-t', '--type', required=True)
	184	parser.add_argument('-j', '--jobname', required=True)
	185	parser.add_argument('-k', '--kernel', required=True)
878b4840 JR	186	parser.add_argument('-lm', '--lmodule', required=True)
878b4840 JR	187	parser.add_argument('-tc', '--tools-commit', required=True)
6b35e57c	188	parser.add_argument('-id', '--build-id', required=True)
878b4840	189	parser.add_argument('-uc', '--ust-commit', required=False)
f23dc688	190	parser.add_argument('-d', '--debug', required=False, action='store_true')
878b4840 JR	191	args = parser.parse_args()
	192
	193	if args.type not in TestType.values:
	194	print('argument -t/--type {} unrecognized.'.format(args.type))
	195	print('Possible values are:')
	196	for k in TestType.values:
	197	print('\t {}'.format(k))
	198	return -1
878b4840 JR	199
878b4840 JR	200	lava_api_key = None
f23dc688 JR	201	if not args.debug:
f23dc688 JR	202	try:
ef84c6ec	203	lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
f23dc688	204	except Exception as e:
ef84c6ec	205	print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...', e )
f23dc688	206	return -1
878b4840	207
4cb5cc4f JR	208	jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
	209	jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
	210	lstrip_blocks= True)
	211	jinja_template = jinja_env.get_template('template_lava_job.jinja2')
	212	template_source = jinja_env.loader.get_source(jinja_env, 'template_lava_job.jinja2')
	213	parsed_content = jinja_env.parse(template_source)
	214	undef = meta.find_undeclared_variables(parsed_content)
	215
	216	test_type = TestType.values[args.type]
	217
	218	if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
	219	device_type = DeviceType.x86
c11ec858	220	vlttng_path = '/tmp/virtenv'
4cb5cc4f	221
878b4840	222	else:
4cb5cc4f	223	device_type = DeviceType.kvm
c11ec858	224	vlttng_path = '/root/virtenv'
4cb5cc4f	225
c11ec858	226	vlttng_cmd = get_vlttng_cmd(device_type, args.tools_commit, args.ust_commit)
4cb5cc4f JR	227
	228	context = dict()
	229	context['DeviceType'] = DeviceType
	230	context['TestType'] = TestType
	231
	232	context['job_name'] = args.jobname
	233	context['test_type'] = test_type
4cb5cc4f JR	234	context['random_seed'] = random.randint(0, 1000000)
	235	context['device_type'] = device_type
	236
	237	context['vlttng_cmd'] = vlttng_cmd
	238	context['vlttng_path'] = vlttng_path
	239
	240	context['kernel_url'] = args.kernel
	241	context['nfsrootfs_url'] = nfsrootfs
	242	context['lttng_modules_url'] = args.lmodule
6b35e57c	243	context['jenkins_build_id'] = args.build_id
4cb5cc4f JR	244
	245	context['kprobe_round_nb'] = 10
	246
ef84c6ec JR	247	render = jinja_template.render(context)
	248
	249	print('Current context:')
	250	pprint.pprint(context, indent=4)
	251	print('Job to be submitted:')
	252
	253	print(render)
878b4840	254
f23dc688	255	if args.debug:
f23dc688 JR	256	return 0
f23dc688 JR	257
878b4840 JR	258	server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))
878b4840 JR	259
ef84c6ec	260	jobid = server.scheduler.submit_job(render)
878b4840 JR	261
878b4840 JR	262	print('Lava jobid:{}'.format(jobid))
ef84c6ec	263	print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}/log_file'.format(jobid))
878b4840 JR	264
	265	#Check the status of the job every 30 seconds
	266	jobstatus = server.scheduler.job_status(jobid)['job_status']
	267	not_running = False
	268	while jobstatus in 'Submitted' or jobstatus in 'Running':
	269	if not_running is False and jobstatus in 'Running':
	270	print('Job started running')
	271	not_running = True
	272	time.sleep(30)
	273	jobstatus = server.scheduler.job_status(jobid)['job_status']
	274
ef84c6ec JR	275	# Do not fetch result for now
	276	# if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
	277	# print_test_output(server, jobid)
	278	# elif test_type is TestType.baremetal_benchmarks:
	279	# fetch_benchmark_results(server, jobid)
878b4840 JR	280
	281	print('Job ended with {} status.'.format(jobstatus))
	282	if jobstatus not in 'Complete':
	283	return -1
	284	else:
	285	passed, failed=check_job_all_test_cases_state_count(server, jobid)
	286	print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
	287
	288	if failed == 0:
	289	return 0
	290	else:
	291	return -1
	292
	293	if __name__ == "__main__":
	294	sys.exit(main())