[lttng-ci.git] / scripts / lttng-baremetal-tests / lava-submit.py

#!/usr/bin/python
# Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import argparse
import base64
import json
import os
import sys
import time
import xmlrpclib
from collections import OrderedDict
from enum import Enum

USERNAME = 'frdeso'
HOSTNAME = 'lava-master.internal.efficios.com'
SCP_PATH = 'scp://jenkins-lava@storage.internal.efficios.com'

class TestType(Enum):
    benchmarks=1
    tests=2

def get_job_bundle_content(server, job):
    bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
    bundle = server.dashboard.get(bundle_sha)

    return json.loads(bundle['content'])

# Parse the results bundle to see the run-tests testcase
# of the lttng-kernel-tests passed successfully
def check_job_all_test_cases_state_count(server, job):
    content = get_job_bundle_content(server, job)

    passed_tests=0
    failed_tests=0
    for run in content['test_runs']:
        for result in run['test_results']:
            if 'test_case_id' in result:
                if result['result'] in 'pass':
                    passed_tests+=1
                else:
                    failed_tests+=1
    return (passed_tests, failed_tests)

# Parse the attachment of the testcase to fetch the stdout of the test suite
def print_test_output(server, job):
    content = get_job_bundle_content(server, job)
    found = False

    for run in content['test_runs']:
        if run['test_id'] in 'lttng-kernel-test':
            for attachment in run['attachments']:
                if attachment['pathname'] in 'stdout.log':

                    # Decode the base64 file and split on newlines to iterate
                    # on list
                    testoutput = base64.b64decode(attachment['content']).split('\n')

                    # Create a generator to iterate on the lines and keeping
                    # the state of the iterator across the two loops.
                    testoutput_iter = iter(testoutput)
                    for line in testoutput_iter:

                        # Find the header of the test case and start printing
                        # from there
                        if 'LAVA_SIGNAL_STARTTC run-tests' in line:
                            found = True
                            print('---- TEST SUITE OUTPUT BEGIN ----')
                            for line in testoutput_iter:
                                if 'LAVA_SIGNAL_ENDTC run-tests' not in line:
                                    print(line)
                                else:
                                    # Print until we reach the end of the
                                    # section
                                    break

                        if found is True:
                            print('----- TEST SUITE OUTPUT END -----')
                            break

def create_new_job(name, build_device):
    job = OrderedDict({
        'health_check': False,
        'job_name': name,
        'device_type':build_device,
        'tags': [ ],
        'timeout': 18000,
        'actions': []
    })
    if build_device in 'x86':
        job['tags'].append('dev-sda1')

    return job

def get_boot_cmd():
    command = OrderedDict({
        'command': 'boot_image'
        })
    return command

def get_config_cmd(build_device):
    packages=['bsdtar', 'psmisc', 'wget', 'python3', 'python3-pip', \
            'libglib2.0-dev', 'libffi-dev', 'elfutils', 'libdw-dev', \
            'libelf-dev', 'libmount-dev', 'libxml2', 'python3-pandas', \
            'python3-numpy']
    command = OrderedDict({
        'command': 'lava_command_run',
        'parameters': {
            'commands': [
                'ifup eth0',
                'route -n',
                'cat /etc/resolv.conf',
                'echo nameserver 172.18.0.12 > /etc/resolv.conf',
                'groupadd tracing'
                ]
            }
        })
    if build_device in 'x86':
        command['parameters']['commands'].extend([
                    'mount /dev/sda1 /tmp',
                    'rm -rf /tmp/*'])

    command['parameters']['commands'].extend([
                    'depmod -a',
                    'locale-gen en_US.UTF-8',
                    'apt-get update',
                    'apt-get install -y {}'.format(' '.join(packages))
                ])
    return command

def get_benchmarks_cmd():
    command = OrderedDict({
        'command': 'lava_test_shell',
        'parameters': {
            'testdef_repos': [
                {
                    'git-repo': 'https://github.com/lttng/lttng-ci.git',
                    'revision': 'master',
                    'testdef': 'lava/baremetal-tests/failing-close.yml'
                },
                {
                    'git-repo': 'https://github.com/lttng/lttng-ci.git',
                    'revision': 'master',
                    'testdef': 'lava/baremetal-tests/failing-open-efault.yml'
                },
                {
                    'git-repo': 'https://github.com/lttng/lttng-ci.git',
                    'revision': 'master',
                    'testdef': 'lava/baremetal-tests/failing-open-enoent.yml'
                }
                ],
            'timeout': 18000
            }
        })
    return command

def get_tests_cmd():
    command = OrderedDict({
        'command': 'lava_test_shell',
        'parameters': {
            'testdef_repos': [
                {
                    'git-repo': 'https://github.com/lttng/lttng-ci.git',
                    'revision': 'master',
                    'testdef': 'lava/baremetal-tests/kernel-tests.yml'
                }
                ],
            'timeout': 18000
            }
        })
    return command

def get_results_cmd(stream_name):
    command = OrderedDict({
            'command': 'submit_results',
            'parameters': {
                'server': 'http://lava-master.internal.efficios.com/RPC2/'
            }
        })
    command['parameters']['stream']='/anonymous/'+stream_name+'/'
    return command

def get_deploy_cmd_kvm(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path):
    command = OrderedDict({
            'command': 'deploy_kernel',
            'metadata': {},
            'parameters': {
                'customize': {},
                'kernel': None,
                'rootfs': 'file:///var/lib/lava-server/default/media/images/trusty-grub.img.gz',
                'target_type': 'ubuntu'
                }
            })

    command['parameters']['customize'][SCP_PATH+linux_modules_path]=['rootfs:/','archive']
    command['parameters']['customize'][SCP_PATH+lttng_modules_path]=['rootfs:/','archive']
    command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
    command['metadata']['jenkins_jobname'] = jenkins_job

    return command

def get_deploy_cmd_x86(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path, nb_iter=None):
    command = OrderedDict({
            'command': 'deploy_kernel',
            'metadata': {},
            'parameters': {
                'overlays': [],
                'kernel': None,
                'nfsrootfs': str(SCP_PATH+'/storage/jenkins-lava/rootfs/rootfs_amd64_trusty_2016-02-23-1134.tar.gz'),
                'target_type': 'ubuntu'
                }
            })

    command['parameters']['overlays'].append( str(SCP_PATH+linux_modules_path))
    command['parameters']['overlays'].append( str(SCP_PATH+lttng_modules_path))
    command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
    command['metadata']['jenkins_jobname'] = jenkins_job
    if nb_iter is not None:
        command['metadata']['nb_iterations'] = nb_iter

    return command


def get_env_setup_cmd(build_device, lttng_tools_commit, lttng_ust_commit=None):
    command = OrderedDict({
        'command': 'lava_command_run',
        'parameters': {
            'commands': [
                'git clone https://github.com/frdeso/syscall-bench-it.git bm',
                'pip3 install vlttng',
                        ],
            'timeout': 18000
            }
        })

    vlttng_cmd = 'vlttng --jobs=16 --profile urcu-master' \
                    ' --profile babeltrace-stable-1.4 ' \
                    ' --profile lttng-tools-master' \
                    ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
                    ' --profile lttng-tools-no-man-pages'

    if lttng_ust_commit is not None:
        vlttng_cmd += ' --profile lttng-ust-master ' \
                    ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
                    ' --profile lttng-ust-no-man-pages'

    virtenv_path = None
    if build_device in 'kvm':
        virtenv_path = '/root/virtenv'
    else:
        virtenv_path = '/tmp/virtenv'

    vlttng_cmd += ' '+virtenv_path

    command['parameters']['commands'].append(vlttng_cmd)
    command['parameters']['commands'].append('ln -s '+virtenv_path+' /root/lttngvenv')
    command['parameters']['commands'].append('sync')

    return command

def main():
    test_type = None
    parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
    parser.add_argument('-t', '--type', required=True)
    parser.add_argument('-j', '--jobname', required=True)
    parser.add_argument('-k', '--kernel', required=True)
    parser.add_argument('-km', '--kmodule', required=True)
    parser.add_argument('-lm', '--lmodule', required=True)
    parser.add_argument('-l', '--lava-key', required=True)
    parser.add_argument('-tc', '--tools-commit', required=True)
    parser.add_argument('-uc', '--ust-commit', required=False)
    args = parser.parse_args()

    if args.type in 'benchmarks':
        test_type = TestType.benchmarks
    elif args.type in 'tests':
        test_type = TestType.tests
    else:
        print('argument -t/--type {} unrecognized. Exiting...'.format(args.type))
        return -1

    if test_type is TestType.benchmarks:
        j = create_new_job(args.jobname, build_device='x86')
        j['actions'].append(get_deploy_cmd_x86(args.jobname, args.kernel, args.kmodule, args.lmodule))
    elif test_type  is TestType.tests:
        j = create_new_job(args.jobname, build_device='kvm')
        j['actions'].append(get_deploy_cmd_kvm(args.jobname, args.kernel, args.kmodule, args.lmodule))

    j['actions'].append(get_boot_cmd())

    if test_type is TestType.benchmarks:
        j['actions'].append(get_config_cmd('x86'))
        j['actions'].append(get_env_setup_cmd('x86', args.tools_commit))
        j['actions'].append(get_benchmarks_cmd())
        j['actions'].append(get_results_cmd(stream_name='benchmark-kernel'))
    elif test_type  is TestType.tests:
        if args.ust_commit is None:
            print('Tests runs need -uc/--ust-commit options. Exiting...')
            return -1
        j['actions'].append(get_config_cmd('kvm'))
        j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit))
        j['actions'].append(get_tests_cmd())
        j['actions'].append(get_results_cmd(stream_name='tests-kernel'))
    else:
        assert False, 'Unknown test type'

    server = xmlrpclib.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, args.lava_key, HOSTNAME))

    jobid = server.scheduler.submit_job(json.dumps(j))

    print('Lava jobid:{}'.format(jobid))

    #Check the status of the job every 30 seconds
    jobstatus = server.scheduler.job_status(jobid)['job_status']
    while jobstatus in 'Submitted' or jobstatus in 'Running':
        time.sleep(30)
        jobstatus = server.scheduler.job_status(jobid)['job_status']

    print('Job ended with {} status.'.format(jobstatus))
    if jobstatus not in 'Complete':
        return -1

    passed, failed=check_job_all_test_cases_state_count(server, jobid)

    print('With {} passed and {} failed Lava test cases.'.format(passed, failed))

    if test_type is TestType.tests:
        print_test_output(server, jobid)

    if failed == 0:
        return 0
    else:
        return -1

if __name__ == "__main__":
    sys.exit(main())
Commit	Line	Data
b3d73c46 FD	1	#!/usr/bin/python
	2	# Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
	3	#
	4	# This program is free software: you can redistribute it and/or modify
	5	# it under the terms of the GNU General Public License as published by
	6	# the Free Software Foundation, either version 3 of the License, or
	7	# (at your option) any later version.
	8	#
	9	# This program is distributed in the hope that it will be useful,
	10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	# GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License
	15	# along with this program. If not, see <http://www.gnu.org/licenses/>.
	16
	17	import argparse
	18	import base64
	19	import json
	20	import os
	21	import sys
	22	import time
	23	import xmlrpclib
	24	from collections import OrderedDict
	25	from enum import Enum
	26
	27	USERNAME = 'frdeso'
	28	HOSTNAME = 'lava-master.internal.efficios.com'
	29	SCP_PATH = 'scp://jenkins-lava@storage.internal.efficios.com'
	30
	31	class TestType(Enum):
	32	benchmarks=1
	33	tests=2
	34
	35	def get_job_bundle_content(server, job):
	36	bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
	37	bundle = server.dashboard.get(bundle_sha)
	38
	39	return json.loads(bundle['content'])
	40
	41	# Parse the results bundle to see the run-tests testcase
	42	# of the lttng-kernel-tests passed successfully
	43	def check_job_all_test_cases_state_count(server, job):
	44	content = get_job_bundle_content(server, job)
	45
	46	passed_tests=0
	47	failed_tests=0
	48	for run in content['test_runs']:
	49	for result in run['test_results']:
	50	if 'test_case_id' in result:
	51	if result['result'] in 'pass':
	52	passed_tests+=1
	53	else:
	54	failed_tests+=1
	55	return (passed_tests, failed_tests)
	56
	57	# Parse the attachment of the testcase to fetch the stdout of the test suite
	58	def print_test_output(server, job):
	59	content = get_job_bundle_content(server, job)
	60	found = False
	61
	62	for run in content['test_runs']:
	63	if run['test_id'] in 'lttng-kernel-test':
	64	for attachment in run['attachments']:
65	if attachment['pathname'] in 'stdout.log':
66
67	# Decode the base64 file and split on newlines to iterate
68	# on list
69	testoutput = base64.b64decode(attachment['content']).split('\n')
70
71	# Create a generator to iterate on the lines and keeping
72	# the state of the iterator across the two loops.
73	testoutput_iter = iter(testoutput)
74	for line in testoutput_iter:
75
76	# Find the header of the test case and start printing
77	# from there
78	if 'LAVA_SIGNAL_STARTTC run-tests' in line:
79	found = True
80	print('---- TEST SUITE OUTPUT BEGIN ----')
81	for line in testoutput_iter:
82	if 'LAVA_SIGNAL_ENDTC run-tests' not in line:
83	print(line)
84	else:
85	# Print until we reach the end of the
86	# section
87	break
88
89	if found is True:
90	print('----- TEST SUITE OUTPUT END -----')
91	break
92
dc9700c9	93	def create_new_job(name, build_device):
b3d73c46 FD	94	job = OrderedDict({
	95	'health_check': False,
	96	'job_name': name,
dc9700c9 FD	97	'device_type':build_device,
dc9700c9 FD	98	'tags': [ ],
b3d73c46 FD	99	'timeout': 18000,
	100	'actions': []
	101	})
dc9700c9 FD	102	if build_device in 'x86':
	103	job['tags'].append('dev-sda1')
	104
b3d73c46 FD	105	return job
	106
	107	def get_boot_cmd():
	108	command = OrderedDict({
	109	'command': 'boot_image'
	110	})
	111	return command
	112
dc9700c9	113	def get_config_cmd(build_device):
b3d73c46 FD	114	packages=['bsdtar', 'psmisc', 'wget', 'python3', 'python3-pip', \
	115	'libglib2.0-dev', 'libffi-dev', 'elfutils', 'libdw-dev', \
	116	'libelf-dev', 'libmount-dev', 'libxml2', 'python3-pandas', \
	117	'python3-numpy']
	118	command = OrderedDict({
	119	'command': 'lava_command_run',
	120	'parameters': {
	121	'commands': [
	122	'ifup eth0',
	123	'route -n',
	124	'cat /etc/resolv.conf',
	125	'echo nameserver 172.18.0.12 > /etc/resolv.conf',
dc9700c9	126	'groupadd tracing'
b3d73c46 FD	127	]
	128	}
	129	})
dc9700c9 FD	130	if build_device in 'x86':
	131	command['parameters']['commands'].extend([
	132	'mount /dev/sda1 /tmp',
	133	'rm -rf /tmp/*'])
	134
	135	command['parameters']['commands'].extend([
	136	'depmod -a',
	137	'locale-gen en_US.UTF-8',
	138	'apt-get update',
	139	'apt-get install -y {}'.format(' '.join(packages))
	140	])
b3d73c46 FD	141	return command
	142
	143	def get_benchmarks_cmd():
	144	command = OrderedDict({
	145	'command': 'lava_test_shell',
	146	'parameters': {
	147	'testdef_repos': [
	148	{
	149	'git-repo': 'https://github.com/lttng/lttng-ci.git',
	150	'revision': 'master',
	151	'testdef': 'lava/baremetal-tests/failing-close.yml'
	152	},
	153	{
	154	'git-repo': 'https://github.com/lttng/lttng-ci.git',
	155	'revision': 'master',
	156	'testdef': 'lava/baremetal-tests/failing-open-efault.yml'
	157	},
	158	{
	159	'git-repo': 'https://github.com/lttng/lttng-ci.git',
	160	'revision': 'master',
	161	'testdef': 'lava/baremetal-tests/failing-open-enoent.yml'
	162	}
	163	],
	164	'timeout': 18000
	165	}
	166	})
	167	return command
	168
	169	def get_tests_cmd():
	170	command = OrderedDict({
	171	'command': 'lava_test_shell',
	172	'parameters': {
	173	'testdef_repos': [
	174	{
	175	'git-repo': 'https://github.com/lttng/lttng-ci.git',
	176	'revision': 'master',
	177	'testdef': 'lava/baremetal-tests/kernel-tests.yml'
	178	}
	179	],
	180	'timeout': 18000
	181	}
	182	})
	183	return command
	184
	185	def get_results_cmd(stream_name):
	186	command = OrderedDict({
	187	'command': 'submit_results',
	188	'parameters': {
	189	'server': 'http://lava-master.internal.efficios.com/RPC2/'
	190	}
	191	})
	192	command['parameters']['stream']='/anonymous/'+stream_name+'/'
	193	return command
	194
dc9700c9 FD	195	def get_deploy_cmd_kvm(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path):
	196	command = OrderedDict({
	197	'command': 'deploy_kernel',
	198	'metadata': {},
	199	'parameters': {
	200	'customize': {},
	201	'kernel': None,
	202	'rootfs': 'file:///var/lib/lava-server/default/media/images/trusty-grub.img.gz',
	203	'target_type': 'ubuntu'
	204	}
	205	})
	206
	207	command['parameters']['customize'][SCP_PATH+linux_modules_path]=['rootfs:/','archive']
	208	command['parameters']['customize'][SCP_PATH+lttng_modules_path]=['rootfs:/','archive']
	209	command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
	210	command['metadata']['jenkins_jobname'] = jenkins_job
	211
	212	return command
	213
	214	def get_deploy_cmd_x86(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path, nb_iter=None):
b3d73c46 FD	215	command = OrderedDict({
	216	'command': 'deploy_kernel',
	217	'metadata': {},
	218	'parameters': {
	219	'overlays': [],
	220	'kernel': None,
	221	'nfsrootfs': str(SCP_PATH+'/storage/jenkins-lava/rootfs/rootfs_amd64_trusty_2016-02-23-1134.tar.gz'),
	222	'target_type': 'ubuntu'
	223	}
	224	})
	225
	226	command['parameters']['overlays'].append( str(SCP_PATH+linux_modules_path))
	227	command['parameters']['overlays'].append( str(SCP_PATH+lttng_modules_path))
	228	command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
dc9700c9	229	command['metadata']['jenkins_jobname'] = jenkins_job
b3d73c46 FD	230	if nb_iter is not None:
	231	command['metadata']['nb_iterations'] = nb_iter
	232
	233	return command
	234
	235
dc9700c9	236	def get_env_setup_cmd(build_device, lttng_tools_commit, lttng_ust_commit=None):
b3d73c46 FD	237	command = OrderedDict({
	238	'command': 'lava_command_run',
	239	'parameters': {
	240	'commands': [
	241	'git clone https://github.com/frdeso/syscall-bench-it.git bm',
	242	'pip3 install vlttng',
	243	],
	244	'timeout': 18000
	245	}
	246	})
	247
	248	vlttng_cmd = 'vlttng --jobs=16 --profile urcu-master' \
	249	' --profile babeltrace-stable-1.4 ' \
	250	' --profile lttng-tools-master' \
	251	' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
	252	' --profile lttng-tools-no-man-pages'
	253
	254	if lttng_ust_commit is not None:
	255	vlttng_cmd += ' --profile lttng-ust-master ' \
	256	' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
	257	' --profile lttng-ust-no-man-pages'
	258
dc9700c9 FD	259	virtenv_path = None
	260	if build_device in 'kvm':
	261	virtenv_path = '/root/virtenv'
	262	else:
	263	virtenv_path = '/tmp/virtenv'
	264
	265	vlttng_cmd += ' '+virtenv_path
b3d73c46 FD	266
b3d73c46 FD	267	command['parameters']['commands'].append(vlttng_cmd)
dc9700c9 FD	268	command['parameters']['commands'].append('ln -s '+virtenv_path+' /root/lttngvenv')
	269	command['parameters']['commands'].append('sync')
	270
b3d73c46 FD	271	return command
	272
	273	def main():
	274	test_type = None
	275	parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
	276	parser.add_argument('-t', '--type', required=True)
	277	parser.add_argument('-j', '--jobname', required=True)
	278	parser.add_argument('-k', '--kernel', required=True)
	279	parser.add_argument('-km', '--kmodule', required=True)
	280	parser.add_argument('-lm', '--lmodule', required=True)
	281	parser.add_argument('-l', '--lava-key', required=True)
	282	parser.add_argument('-tc', '--tools-commit', required=True)
	283	parser.add_argument('-uc', '--ust-commit', required=False)
	284	args = parser.parse_args()
	285
b3d73c46 FD	286	if args.type in 'benchmarks':
	287	test_type = TestType.benchmarks
	288	elif args.type in 'tests':
	289	test_type = TestType.tests
	290	else:
	291	print('argument -t/--type {} unrecognized. Exiting...'.format(args.type))
	292	return -1
	293
	294	if test_type is TestType.benchmarks:
dc9700c9 FD	295	j = create_new_job(args.jobname, build_device='x86')
	296	j['actions'].append(get_deploy_cmd_x86(args.jobname, args.kernel, args.kmodule, args.lmodule))
	297	elif test_type is TestType.tests:
	298	j = create_new_job(args.jobname, build_device='kvm')
	299	j['actions'].append(get_deploy_cmd_kvm(args.jobname, args.kernel, args.kmodule, args.lmodule))
	300
	301	j['actions'].append(get_boot_cmd())
	302
	303	if test_type is TestType.benchmarks:
	304	j['actions'].append(get_config_cmd('x86'))
	305	j['actions'].append(get_env_setup_cmd('x86', args.tools_commit))
b3d73c46 FD	306	j['actions'].append(get_benchmarks_cmd())
	307	j['actions'].append(get_results_cmd(stream_name='benchmark-kernel'))
	308	elif test_type is TestType.tests:
	309	if args.ust_commit is None:
	310	print('Tests runs need -uc/--ust-commit options. Exiting...')
	311	return -1
dc9700c9 FD	312	j['actions'].append(get_config_cmd('kvm'))
dc9700c9 FD	313	j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit))
b3d73c46 FD	314	j['actions'].append(get_tests_cmd())
	315	j['actions'].append(get_results_cmd(stream_name='tests-kernel'))
	316	else:
	317	assert False, 'Unknown test type'
	318
	319	server = xmlrpclib.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, args.lava_key, HOSTNAME))
	320
	321	jobid = server.scheduler.submit_job(json.dumps(j))
	322
21e89f7e FD	323	print('Lava jobid:{}'.format(jobid))
21e89f7e FD	324
b3d73c46 FD	325	#Check the status of the job every 30 seconds
	326	jobstatus = server.scheduler.job_status(jobid)['job_status']
	327	while jobstatus in 'Submitted' or jobstatus in 'Running':
	328	time.sleep(30)
	329	jobstatus = server.scheduler.job_status(jobid)['job_status']
	330
	331	print('Job ended with {} status.'.format(jobstatus))
	332	if jobstatus not in 'Complete':
	333	return -1
	334
	335	passed, failed=check_job_all_test_cases_state_count(server, jobid)
	336
21e89f7e	337	print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
b3d73c46 FD	338
	339	if test_type is TestType.tests:
	340	print_test_output(server, jobid)
	341
21e89f7e	342	if failed == 0:
b3d73c46 FD	343	return 0
	344	else:
	345	return -1
	346
	347	if __name__ == "__main__":
	348	sys.exit(main())