[lttng-ci.git] / scripts / lttng-baremetal-tests / lava-submit.py

#!/usr/bin/python
# Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import argparse
import base64
import json
import os
import sys
import time
import xmlrpclib
from collections import OrderedDict
from enum import Enum

USERNAME = 'frdeso'
HOSTNAME = 'lava-master.internal.efficios.com'
SCP_PATH = 'scp://jenkins-lava@storage.internal.efficios.com'

class TestType(Enum):
    benchmarks=1
    tests=2

def get_job_bundle_content(server, job):
    bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
    bundle = server.dashboard.get(bundle_sha)

    return json.loads(bundle['content'])

# Parse the results bundle to see the run-tests testcase
# of the lttng-kernel-tests passed successfully
def check_job_all_test_cases_state_count(server, job):
    content = get_job_bundle_content(server, job)

    passed_tests=0
    failed_tests=0
    for run in content['test_runs']:
        for result in run['test_results']:
            if 'test_case_id' in result :
                if result['result'] in 'pass':
                    passed_tests+=1
                elif result['test_case_id'] in 'wait_for_test_image_prompt':
                    # FIXME:This test is part of the boot action and fails
                    # randomly but doesn't affect the behaviour of the tests.
                    # No reply on the Lava IRC channel yet. We should update
                    # our Lava installation and try to reproduce it. This error
                    # was encountered ont the KVM trusty image only. Not seen
                    # on Xenial at this point.
                    pass
                else:
                    failed_tests+=1
    return (passed_tests, failed_tests)

# Parse the attachment of the testcase to fetch the stdout of the test suite
def print_test_output(server, job):
    content = get_job_bundle_content(server, job)
    found = False

    for run in content['test_runs']:
        if run['test_id'] in 'lttng-kernel-test':
            for attachment in run['attachments']:
                if attachment['pathname'] in 'stdout.log':

                    # Decode the base64 file and split on newlines to iterate
                    # on list
                    testoutput = base64.b64decode(attachment['content']).split('\n')

                    # Create a generator to iterate on the lines and keeping
                    # the state of the iterator across the two loops.
                    testoutput_iter = iter(testoutput)
                    for line in testoutput_iter:

                        # Find the header of the test case and start printing
                        # from there
                        if 'LAVA_SIGNAL_STARTTC run-tests' in line:
                            found = True
                            print('---- TEST SUITE OUTPUT BEGIN ----')
                            for line in testoutput_iter:
                                if 'LAVA_SIGNAL_ENDTC run-tests' not in line:
                                    print(line)
                                else:
                                    # Print until we reach the end of the
                                    # section
                                    break

                        if found is True:
                            print('----- TEST SUITE OUTPUT END -----')
                            break

def create_new_job(name, build_device):
    job = OrderedDict({
        'health_check': False,
        'job_name': name,
        'device_type':build_device,
        'tags': [ ],
        'timeout': 18000,
        'actions': []
    })
    if build_device in 'x86':
        job['tags'].append('dev-sda1')

    return job

def get_boot_cmd():
    command = OrderedDict({
        'command': 'boot_image'
        })
    return command

def get_config_cmd(build_device):
    packages=['bsdtar', 'psmisc', 'wget', 'python3', 'python3-pip', \
            'libglib2.0-dev', 'libffi-dev', 'elfutils', 'libdw-dev', \
            'libelf-dev', 'libmount-dev', 'libxml2']
    command = OrderedDict({
        'command': 'lava_command_run',
        'parameters': {
            'commands': [
                'cat /etc/resolv.conf',
                'echo nameserver 172.18.0.12 > /etc/resolv.conf',
                'groupadd tracing'
                ]
            }
        })
    if build_device in 'x86':
        command['parameters']['commands'].extend([
                    'mount /dev/sda1 /tmp',
                    'rm -rf /tmp/*'])

    command['parameters']['commands'].extend([
                    'depmod -a',
                    'locale-gen en_US.UTF-8',
                    'apt-get update',
                    'apt-get upgrade',
                    'apt-get install -y {}'.format(' '.join(packages))
                ])
    return command

def get_benchmarks_cmd():
    command = OrderedDict({
        'command': 'lava_test_shell',
        'parameters': {
            'testdef_repos': [
                {
                    'git-repo': 'https://github.com/lttng/lttng-ci.git',
                    'revision': 'master',
                    'testdef': 'lava/baremetal-tests/failing-close.yml'
                },
                {
                    'git-repo': 'https://github.com/lttng/lttng-ci.git',
                    'revision': 'master',
                    'testdef': 'lava/baremetal-tests/failing-open-efault.yml'
                },
                {
                    'git-repo': 'https://github.com/lttng/lttng-ci.git',
                    'revision': 'master',
                    'testdef': 'lava/baremetal-tests/failing-open-enoent.yml'
                },
                {
                    'git-repo': 'https://github.com/lttng/lttng-ci.git',
                    'revision': 'master',
                    'testdef': 'lava/baremetal-tests/perf-tests.yml'
                }
                ],
            'timeout': 18000
            }
        })
    return command

def get_tests_cmd():
    command = OrderedDict({
        'command': 'lava_test_shell',
        'parameters': {
            'testdef_repos': [
                {
                    'git-repo': 'https://github.com/lttng/lttng-ci.git',
                    'revision': 'master',
                    'testdef': 'lava/baremetal-tests/kernel-tests.yml'
                }
                ],
            'timeout': 18000
            }
        })
    return command

def get_results_cmd(stream_name):
    command = OrderedDict({
            'command': 'submit_results',
            'parameters': {
                'server': 'http://lava-master.internal.efficios.com/RPC2/'
            }
        })
    command['parameters']['stream']='/anonymous/'+stream_name+'/'
    return command

def get_deploy_cmd_kvm(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path):
    command = OrderedDict({
            'command': 'deploy_kernel',
            'metadata': {},
            'parameters': {
                'customize': {},
                'kernel': None,
                'target_type': 'ubuntu',
                'rootfs': 'file:///var/lib/lava-server/default/media/images/xenial.img.gz',
                'login_prompt': 'kvm02 login:',
                'username': 'root'
                }
            })

    command['parameters']['customize'][SCP_PATH+linux_modules_path]=['rootfs:/','archive']
    command['parameters']['customize'][SCP_PATH+lttng_modules_path]=['rootfs:/','archive']
    command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
    command['metadata']['jenkins_jobname'] = jenkins_job

    return command

def get_deploy_cmd_x86(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path, nb_iter=None):
    command = OrderedDict({
            'command': 'deploy_kernel',
            'metadata': {},
            'parameters': {
                'overlays': [],
                'kernel': None,
                'nfsrootfs': str(SCP_PATH+'/storage/jenkins-lava/rootfs/rootfs_amd64_trusty_2016-02-23-1134.tar.gz'),
                'target_type': 'ubuntu'
                }
            })

    command['parameters']['overlays'].append( str(SCP_PATH+linux_modules_path))
    command['parameters']['overlays'].append( str(SCP_PATH+lttng_modules_path))
    command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
    command['metadata']['jenkins_jobname'] = jenkins_job
    if nb_iter is not None:
        command['metadata']['nb_iterations'] = nb_iter

    return command


def get_env_setup_cmd(build_device, lttng_tools_commit, lttng_ust_commit=None):
    command = OrderedDict({
        'command': 'lava_command_run',
        'parameters': {
            'commands': [
                'pip3 install --upgrade pip',
                'git clone https://github.com/frdeso/syscall-bench-it.git bm',
                'pip3 install vlttng',
                        ],
            'timeout': 18000
            }
        })

    vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
                    ' --profile babeltrace-stable-1.4 ' \
                    ' --profile lttng-tools-master' \
                    ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
                    ' --profile lttng-tools-no-man-pages'

    if lttng_ust_commit is not None:
        vlttng_cmd += ' --profile lttng-ust-master ' \
                    ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
                    ' --profile lttng-ust-no-man-pages'

    virtenv_path = None
    if build_device in 'kvm':
        virtenv_path = '/root/virtenv'
    else:
        virtenv_path = '/tmp/virtenv'

    vlttng_cmd += ' '+virtenv_path

    command['parameters']['commands'].append(vlttng_cmd)
    command['parameters']['commands'].append('ln -s '+virtenv_path+' /root/lttngvenv')
    command['parameters']['commands'].append('sync')

    return command

def main():
    test_type = None
    parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
    parser.add_argument('-t', '--type', required=True)
    parser.add_argument('-j', '--jobname', required=True)
    parser.add_argument('-k', '--kernel', required=True)
    parser.add_argument('-km', '--kmodule', required=True)
    parser.add_argument('-lm', '--lmodule', required=True)
    parser.add_argument('-l', '--lava-key', required=True)
    parser.add_argument('-tc', '--tools-commit', required=True)
    parser.add_argument('-uc', '--ust-commit', required=False)
    args = parser.parse_args()

    if args.type in 'benchmarks':
        test_type = TestType.benchmarks
    elif args.type in 'tests':
        test_type = TestType.tests
    else:
        print('argument -t/--type {} unrecognized. Exiting...'.format(args.type))
        return -1

    if test_type is TestType.benchmarks:
        j = create_new_job(args.jobname, build_device='x86')
        j['actions'].append(get_deploy_cmd_x86(args.jobname, args.kernel, args.kmodule, args.lmodule))
    elif test_type  is TestType.tests:
        j = create_new_job(args.jobname, build_device='kvm')
        j['actions'].append(get_deploy_cmd_kvm(args.jobname, args.kernel, args.kmodule, args.lmodule))

    j['actions'].append(get_boot_cmd())

    if test_type is TestType.benchmarks:
        j['actions'].append(get_config_cmd('x86'))
        j['actions'].append(get_env_setup_cmd('x86', args.tools_commit))
        j['actions'].append(get_benchmarks_cmd())
        j['actions'].append(get_results_cmd(stream_name='benchmark-kernel'))
    elif test_type  is TestType.tests:
        if args.ust_commit is None:
            print('Tests runs need -uc/--ust-commit options. Exiting...')
            return -1
        j['actions'].append(get_config_cmd('kvm'))
        j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit))
        j['actions'].append(get_tests_cmd())
        j['actions'].append(get_results_cmd(stream_name='tests-kernel'))
    else:
        assert False, 'Unknown test type'

    server = xmlrpclib.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, args.lava_key, HOSTNAME))

    jobid = server.scheduler.submit_job(json.dumps(j))

    print('Lava jobid:{}'.format(jobid))

    #Check the status of the job every 30 seconds
    jobstatus = server.scheduler.job_status(jobid)['job_status']
    while jobstatus in 'Submitted' or jobstatus in 'Running':
        time.sleep(30)
        jobstatus = server.scheduler.job_status(jobid)['job_status']

    passed, failed=check_job_all_test_cases_state_count(server, jobid)

    if test_type is TestType.tests:
        print_test_output(server, jobid)

    print('Job ended with {} status.'.format(jobstatus))
    if jobstatus not in 'Complete':
        return -1
    else:
        print('With {} passed and {} failed Lava test cases.'.format(passed, failed))

    if failed == 0:
        return 0
    else:
        return -1

if __name__ == "__main__":
    sys.exit(main())
Commit	Line	Data
b3d73c46 FD	1	#!/usr/bin/python
	2	# Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
	3	#
	4	# This program is free software: you can redistribute it and/or modify
	5	# it under the terms of the GNU General Public License as published by
	6	# the Free Software Foundation, either version 3 of the License, or
	7	# (at your option) any later version.
	8	#
	9	# This program is distributed in the hope that it will be useful,
	10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	# GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License
	15	# along with this program. If not, see <http://www.gnu.org/licenses/>.
	16
	17	import argparse
	18	import base64
	19	import json
	20	import os
	21	import sys
	22	import time
	23	import xmlrpclib
	24	from collections import OrderedDict
	25	from enum import Enum
	26
	27	USERNAME = 'frdeso'
	28	HOSTNAME = 'lava-master.internal.efficios.com'
	29	SCP_PATH = 'scp://jenkins-lava@storage.internal.efficios.com'
	30
	31	class TestType(Enum):
	32	benchmarks=1
	33	tests=2
	34
	35	def get_job_bundle_content(server, job):
	36	bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
	37	bundle = server.dashboard.get(bundle_sha)
	38
	39	return json.loads(bundle['content'])
	40
	41	# Parse the results bundle to see the run-tests testcase
	42	# of the lttng-kernel-tests passed successfully
	43	def check_job_all_test_cases_state_count(server, job):
	44	content = get_job_bundle_content(server, job)
	45
	46	passed_tests=0
	47	failed_tests=0
	48	for run in content['test_runs']:
	49	for result in run['test_results']:
3e7245a4	50	if 'test_case_id' in result :
b3d73c46 FD	51	if result['result'] in 'pass':
b3d73c46 FD	52	passed_tests+=1
3e7245a4 FD	53	elif result['test_case_id'] in 'wait_for_test_image_prompt':
	54	# FIXME:This test is part of the boot action and fails
	55	# randomly but doesn't affect the behaviour of the tests.
	56	# No reply on the Lava IRC channel yet. We should update
	57	# our Lava installation and try to reproduce it. This error
	58	# was encountered ont the KVM trusty image only. Not seen
	59	# on Xenial at this point.
	60	pass
b3d73c46 FD	61	else:
	62	failed_tests+=1
	63	return (passed_tests, failed_tests)
	64
	65	# Parse the attachment of the testcase to fetch the stdout of the test suite
	66	def print_test_output(server, job):
	67	content = get_job_bundle_content(server, job)
	68	found = False
	69
	70	for run in content['test_runs']:
	71	if run['test_id'] in 'lttng-kernel-test':
	72	for attachment in run['attachments']:
	73	if attachment['pathname'] in 'stdout.log':
	74
	75	# Decode the base64 file and split on newlines to iterate
	76	# on list
	77	testoutput = base64.b64decode(attachment['content']).split('\n')
	78
	79	# Create a generator to iterate on the lines and keeping
	80	# the state of the iterator across the two loops.
	81	testoutput_iter = iter(testoutput)
	82	for line in testoutput_iter:
	83
	84	# Find the header of the test case and start printing
	85	# from there
	86	if 'LAVA_SIGNAL_STARTTC run-tests' in line:
	87	found = True
	88	print('---- TEST SUITE OUTPUT BEGIN ----')
	89	for line in testoutput_iter:
	90	if 'LAVA_SIGNAL_ENDTC run-tests' not in line:
	91	print(line)
	92	else:
	93	# Print until we reach the end of the
	94	# section
	95	break
	96
	97	if found is True:
	98	print('----- TEST SUITE OUTPUT END -----')
	99	break
	100
dc9700c9	101	def create_new_job(name, build_device):
b3d73c46 FD	102	job = OrderedDict({
	103	'health_check': False,
	104	'job_name': name,
dc9700c9 FD	105	'device_type':build_device,
dc9700c9 FD	106	'tags': [ ],
b3d73c46 FD	107	'timeout': 18000,
	108	'actions': []
	109	})
dc9700c9 FD	110	if build_device in 'x86':
	111	job['tags'].append('dev-sda1')
	112
b3d73c46 FD	113	return job
	114
	115	def get_boot_cmd():
	116	command = OrderedDict({
	117	'command': 'boot_image'
	118	})
	119	return command
	120
dc9700c9	121	def get_config_cmd(build_device):
b3d73c46 FD	122	packages=['bsdtar', 'psmisc', 'wget', 'python3', 'python3-pip', \
b3d73c46 FD	123	'libglib2.0-dev', 'libffi-dev', 'elfutils', 'libdw-dev', \
e194ed8e	124	'libelf-dev', 'libmount-dev', 'libxml2']
b3d73c46 FD	125	command = OrderedDict({
	126	'command': 'lava_command_run',
	127	'parameters': {
	128	'commands': [
b3d73c46 FD	129	'cat /etc/resolv.conf',
b3d73c46 FD	130	'echo nameserver 172.18.0.12 > /etc/resolv.conf',
dc9700c9	131	'groupadd tracing'
b3d73c46 FD	132	]
	133	}
	134	})
dc9700c9 FD	135	if build_device in 'x86':
	136	command['parameters']['commands'].extend([
	137	'mount /dev/sda1 /tmp',
	138	'rm -rf /tmp/*'])
	139
	140	command['parameters']['commands'].extend([
	141	'depmod -a',
	142	'locale-gen en_US.UTF-8',
	143	'apt-get update',
819f0b86	144	'apt-get upgrade',
dc9700c9 FD	145	'apt-get install -y {}'.format(' '.join(packages))
dc9700c9 FD	146	])
b3d73c46 FD	147	return command
	148
	149	def get_benchmarks_cmd():
	150	command = OrderedDict({
	151	'command': 'lava_test_shell',
	152	'parameters': {
	153	'testdef_repos': [
	154	{
	155	'git-repo': 'https://github.com/lttng/lttng-ci.git',
	156	'revision': 'master',
	157	'testdef': 'lava/baremetal-tests/failing-close.yml'
	158	},
	159	{
	160	'git-repo': 'https://github.com/lttng/lttng-ci.git',
	161	'revision': 'master',
	162	'testdef': 'lava/baremetal-tests/failing-open-efault.yml'
	163	},
	164	{
	165	'git-repo': 'https://github.com/lttng/lttng-ci.git',
	166	'revision': 'master',
	167	'testdef': 'lava/baremetal-tests/failing-open-enoent.yml'
f3d4ee9f FD	168	},
	169	{
	170	'git-repo': 'https://github.com/lttng/lttng-ci.git',
	171	'revision': 'master',
	172	'testdef': 'lava/baremetal-tests/perf-tests.yml'
b3d73c46 FD	173	}
	174	],
	175	'timeout': 18000
	176	}
	177	})
	178	return command
	179
	180	def get_tests_cmd():
	181	command = OrderedDict({
	182	'command': 'lava_test_shell',
	183	'parameters': {
	184	'testdef_repos': [
	185	{
	186	'git-repo': 'https://github.com/lttng/lttng-ci.git',
	187	'revision': 'master',
	188	'testdef': 'lava/baremetal-tests/kernel-tests.yml'
	189	}
	190	],
	191	'timeout': 18000
	192	}
	193	})
	194	return command
	195
	196	def get_results_cmd(stream_name):
	197	command = OrderedDict({
	198	'command': 'submit_results',
	199	'parameters': {
	200	'server': 'http://lava-master.internal.efficios.com/RPC2/'
	201	}
	202	})
	203	command['parameters']['stream']='/anonymous/'+stream_name+'/'
	204	return command
	205
dc9700c9 FD	206	def get_deploy_cmd_kvm(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path):
	207	command = OrderedDict({
	208	'command': 'deploy_kernel',
	209	'metadata': {},
	210	'parameters': {
	211	'customize': {},
	212	'kernel': None,
819f0b86 FD	213	'target_type': 'ubuntu',
	214	'rootfs': 'file:///var/lib/lava-server/default/media/images/xenial.img.gz',
	215	'login_prompt': 'kvm02 login:',
	216	'username': 'root'
dc9700c9 FD	217	}
	218	})
	219
	220	command['parameters']['customize'][SCP_PATH+linux_modules_path]=['rootfs:/','archive']
	221	command['parameters']['customize'][SCP_PATH+lttng_modules_path]=['rootfs:/','archive']
	222	command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
	223	command['metadata']['jenkins_jobname'] = jenkins_job
	224
	225	return command
	226
	227	def get_deploy_cmd_x86(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path, nb_iter=None):
b3d73c46 FD	228	command = OrderedDict({
	229	'command': 'deploy_kernel',
	230	'metadata': {},
	231	'parameters': {
	232	'overlays': [],
	233	'kernel': None,
	234	'nfsrootfs': str(SCP_PATH+'/storage/jenkins-lava/rootfs/rootfs_amd64_trusty_2016-02-23-1134.tar.gz'),
	235	'target_type': 'ubuntu'
	236	}
	237	})
	238
	239	command['parameters']['overlays'].append( str(SCP_PATH+linux_modules_path))
	240	command['parameters']['overlays'].append( str(SCP_PATH+lttng_modules_path))
	241	command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
dc9700c9	242	command['metadata']['jenkins_jobname'] = jenkins_job
b3d73c46 FD	243	if nb_iter is not None:
	244	command['metadata']['nb_iterations'] = nb_iter
	245
	246	return command
	247
	248
dc9700c9	249	def get_env_setup_cmd(build_device, lttng_tools_commit, lttng_ust_commit=None):
b3d73c46 FD	250	command = OrderedDict({
	251	'command': 'lava_command_run',
	252	'parameters': {
	253	'commands': [
819f0b86	254	'pip3 install --upgrade pip',
b3d73c46 FD	255	'git clone https://github.com/frdeso/syscall-bench-it.git bm',
	256	'pip3 install vlttng',
	257	],
	258	'timeout': 18000
	259	}
	260	})
	261
4418be37	262	vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
b3d73c46 FD	263	' --profile babeltrace-stable-1.4 ' \
	264	' --profile lttng-tools-master' \
	265	' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
	266	' --profile lttng-tools-no-man-pages'
	267
	268	if lttng_ust_commit is not None:
	269	vlttng_cmd += ' --profile lttng-ust-master ' \
	270	' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
	271	' --profile lttng-ust-no-man-pages'
	272
dc9700c9 FD	273	virtenv_path = None
	274	if build_device in 'kvm':
	275	virtenv_path = '/root/virtenv'
	276	else:
	277	virtenv_path = '/tmp/virtenv'
	278
	279	vlttng_cmd += ' '+virtenv_path
b3d73c46 FD	280
b3d73c46 FD	281	command['parameters']['commands'].append(vlttng_cmd)
dc9700c9 FD	282	command['parameters']['commands'].append('ln -s '+virtenv_path+' /root/lttngvenv')
	283	command['parameters']['commands'].append('sync')
	284
b3d73c46 FD	285	return command
	286
	287	def main():
	288	test_type = None
	289	parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
	290	parser.add_argument('-t', '--type', required=True)
	291	parser.add_argument('-j', '--jobname', required=True)
	292	parser.add_argument('-k', '--kernel', required=True)
	293	parser.add_argument('-km', '--kmodule', required=True)
	294	parser.add_argument('-lm', '--lmodule', required=True)
	295	parser.add_argument('-l', '--lava-key', required=True)
	296	parser.add_argument('-tc', '--tools-commit', required=True)
	297	parser.add_argument('-uc', '--ust-commit', required=False)
	298	args = parser.parse_args()
	299
b3d73c46 FD	300	if args.type in 'benchmarks':
	301	test_type = TestType.benchmarks
	302	elif args.type in 'tests':
	303	test_type = TestType.tests
	304	else:
	305	print('argument -t/--type {} unrecognized. Exiting...'.format(args.type))
	306	return -1
	307
	308	if test_type is TestType.benchmarks:
dc9700c9 FD	309	j = create_new_job(args.jobname, build_device='x86')
	310	j['actions'].append(get_deploy_cmd_x86(args.jobname, args.kernel, args.kmodule, args.lmodule))
	311	elif test_type is TestType.tests:
	312	j = create_new_job(args.jobname, build_device='kvm')
	313	j['actions'].append(get_deploy_cmd_kvm(args.jobname, args.kernel, args.kmodule, args.lmodule))
	314
	315	j['actions'].append(get_boot_cmd())
	316
	317	if test_type is TestType.benchmarks:
	318	j['actions'].append(get_config_cmd('x86'))
	319	j['actions'].append(get_env_setup_cmd('x86', args.tools_commit))
b3d73c46 FD	320	j['actions'].append(get_benchmarks_cmd())
	321	j['actions'].append(get_results_cmd(stream_name='benchmark-kernel'))
	322	elif test_type is TestType.tests:
	323	if args.ust_commit is None:
	324	print('Tests runs need -uc/--ust-commit options. Exiting...')
	325	return -1
dc9700c9 FD	326	j['actions'].append(get_config_cmd('kvm'))
dc9700c9 FD	327	j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit))
b3d73c46 FD	328	j['actions'].append(get_tests_cmd())
	329	j['actions'].append(get_results_cmd(stream_name='tests-kernel'))
	330	else:
	331	assert False, 'Unknown test type'
	332
	333	server = xmlrpclib.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, args.lava_key, HOSTNAME))
	334
	335	jobid = server.scheduler.submit_job(json.dumps(j))
	336
21e89f7e FD	337	print('Lava jobid:{}'.format(jobid))
21e89f7e FD	338
b3d73c46 FD	339	#Check the status of the job every 30 seconds
	340	jobstatus = server.scheduler.job_status(jobid)['job_status']
	341	while jobstatus in 'Submitted' or jobstatus in 'Running':
	342	time.sleep(30)
	343	jobstatus = server.scheduler.job_status(jobid)['job_status']
	344
b3d73c46 FD	345	passed, failed=check_job_all_test_cases_state_count(server, jobid)
b3d73c46 FD	346
b3d73c46 FD	347	if test_type is TestType.tests:
	348	print_test_output(server, jobid)
	349
d1b69f4d FD	350	print('Job ended with {} status.'.format(jobstatus))
	351	if jobstatus not in 'Complete':
	352	return -1
	353	else:
	354	print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
	355
21e89f7e	356	if failed == 0:
b3d73c46 FD	357	return 0
	358	else:
	359	return -1
	360
	361	if __name__ == "__main__":
	362	sys.exit(main())