jjb/lava: Parametrize URL for system-test's root FS

[lttng-ci.git] / jobs / system-tests.yaml

---
## Anchors
- _system_tests_parameters_defaults: &system_tests_parameters_defaults
    name: 'system_tests_parameters_defaults'
    parameters:
      - string:
          name: 'LTTNG_TOOLS_COMMIT_ID'
          default: ''
          description: 'The lttng-tools commit id to build.'
      - string:
          name: 'LTTNG_MODULES_COMMIT_ID'
          default: ''
          description: 'The lttng-modules commit id to build.'
      - string:
          name: 'LTTNG_UST_COMMIT_ID'
          default: ''
          description: 'The lttng-ust commit id to build.'
      - string:
          name: 'KERNEL_TAG_ID'
          default: ''
          description: 'The linux kernel git tag to build against.'
      - string:
          name: 'KERNEL_REPO'
          default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git'
          description: 'Linux kernel git repo to checkout the kernel id'
      - string:
          name: 'LTTNG_TOOLS_REPO'
          default: 'https://github.com/lttng/lttng-tools.git'
          description: 'LTTng-Tools git repo to checkout the tools id'
      - string:
          name: 'LTTNG_MODULES_REPO'
          default: 'https://github.com/lttng/lttng-modules.git'
          description: 'LTTng-Modules git repo to checkout the Modules id'
      - string:
          name: 'LTTNG_UST_REPO'
          default: 'https://github.com/lttng/lttng-ust.git'
          description: 'LTTng-UST git repo to checkout the UST id'
      - string:
          name: 'ROOTFS_URL'
          default: 'https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz'
          description: 'The URL at which the system root FS can be downloaded'

- publisher:
    name: 'system_tests_email_ext_default'
    publishers:
      - email-ext:
          recipients: 'ci-notification@lists.lttng.org, cc:francis.deslauriers@efficios.com'
          reply-to: ci-notification@lists.lttng.org
          always: false
          unstable: false
          first-failure: true
          first-unstable: true
          not-built: false
          aborted: false
          regression: false
          failure: false
          second-failure: false
          improvement: false
          still-failing: false
          success: false
          fixed: false
          fixed-unhealthy: true
          still-unstable: false
          pre-build: false
          matrix-trigger: only-parent
          send-to:
            - recipients

- job-template:
    name: vm_tests_k{kversion}_l{lttngversion}
    description: |
      Runs root_regression and root_destructive_tests on a virtual machine over different combinations of kernel and lttng configurations.
    project-type: freestyle
    node: 'amd64'
    <<: *system_tests_parameters_defaults

    properties:
      - build-discarder:
          num-to-keep: 10
      - throttle:
          max-total: 4
          option: 'category'
          categories:
            - 'kvm-tests'

    wrappers:
      - workspace-cleanup
      - timestamps
      - ansicolor
      - credentials-binding:
          - text:
              credential-id: jenkins_lava_key
              variable: LAVA_JENKINS_TOKEN
          - text:
              credential-id: jenkins_lava2_key
              variable: LAVA2_JENKINS_TOKEN
      - inject:
          properties-content: |
            BUILD_DEVICE=kvm
            LTTNG_VERSION={lttngversion}

    scm:
      - git:
          url: https://github.com/lttng/lttng-ci
          basedir: src/lttng-ci/

    builders:
      - shell: !include-raw-escape: scripts/system-tests/generate-properties-master.sh
      - trigger-builds:
        - project: "build_kernel_PARAM"
          property-file: 'properties.txt'
          block: true
      - inject:
          properties-file: properties.txt
      - shell: !include-raw-escape: scripts/system-tests/run-kvm-tests.sh

    publishers:
      - workspace-cleanup
      - email-ext:
          recipients: '{obj:email_to}'
          always: false
          unstable: false
          first-failure: true
          first-unstable: true
          not-built: false
          aborted: false
          regression: false
          failure: false
          second-failure: false
          improvement: false
          still-failing: true
          success: false
          fixed: false
          fixed-unhealthy: true
          still-unstable: false
          pre-build: false
          matrix-trigger: only-parent
          send-to:
            - recipients

- job-template:
    name: baremetal_tests_k{kversion}_l{lttngversion}
    description: |
      Runs perf_regression tests on a baremetal machine over different combinations of kernel and lttng configurations.
    project-type: freestyle
    node: 'amd64'

    <<: *system_tests_parameters_defaults

    properties:
      - build-discarder:
          num-to-keep: 10
          artifact-num-to-keep: 2
      - throttle:
          max-total: 2
          option: 'category'
          categories:
            - 'baremetal-tests'

    wrappers:
      - workspace-cleanup
      - timestamps
      - ansicolor
      - credentials-binding:
          - text:
              credential-id: jenkins_lava_key
              variable: LAVA_JENKINS_TOKEN
          - text:
              credential-id: jenkins_lava2_key
              variable: LAVA2_JENKINS_TOKEN
      - inject:
          properties-content: |
            BUILD_DEVICE=baremetal
            LTTNG_VERSION={lttngversion}

    scm:
      - git:
          url: https://github.com/lttng/lttng-ci
          basedir: src/lttng-ci/

    builders:
      - shell: !include-raw-escape: scripts/system-tests/generate-properties-master.sh
      - trigger-builds:
        - project: "build_kernel_PARAM"
          property-file: 'properties.txt'
          block: true
      - inject:
          properties-file: properties.txt
      - shell: !include-raw-escape: scripts/system-tests/run-baremetal-tests.sh

    publishers:
      - workspace-cleanup

- job-template:
    name: build_kernel_PARAM
    description: |
      Builds a Linux Kernel and LTTng Modules if necessary
    concurrent: true
    node: 'amd64'

    parameters:
      - string:
          name: 'LTTNG_MODULES_COMMIT_ID'
          description: 'The lttng-modules commmit to build.'
      - string:
          name: 'LTTNG_MODULES_REPO'
          description: 'The LTTng Modules git repo to fetch from'
          default: 'git://git-mirror.internal.efficios.com/lttng/lttng-modules.git'
      - string:
          name: 'KERNEL_COMMIT_ID'
          description: 'The kernel commit to build.'
      - string:
          name: 'KGITREPO'
          description: 'The kernel git repo to fetch from'
      - string:
          name: 'STORAGE_KERNEL_FOLDER'
          description: 'Path to store the Kernel image'
      - string:
          name: 'STORAGE_KERNEL_IMAGE'
          description: 'Path to store the Kernel IMAGE'
      - string:
          name: 'STORAGE_LINUX_MODULES'
          description: 'Path to store the Kernel Modules'
      - string:
          name: 'STORAGE_LTTNG_MODULES'
          description: 'Path to store the LTTng Modules'
      - string:
          name: 'BUILD_DEVICE'
          description: 'The target device. (kvm or baremetal)'
      - string:
          name: 'S3_STORAGE_KERNEL_FOLDER'
          description: 'Path to store the Kernel image'
      - string:
          name: 'S3_STORAGE_KERNEL_IMAGE'
          description: 'Path to store the Kernel IMAGE'
      - string:
          name: 'S3_STORAGE_LINUX_MODULES'
          description: 'Path to store the Kernel Modules'
      - string:
          name: 'S3_STORAGE_LTTNG_MODULES'
          description: 'Path to store the LTTng Modules'

    properties:
      - build-discarder:
          num-to-keep: 100

    wrappers:
      - workspace-cleanup
      - timestamps
      - ansicolor
      - credentials-binding:
          - file:
              credential-id: system_tests_storage_key
              variable: identity_file

    scm:
      - git:
          url: https://github.com/lttng/lttng-ci
          basedir: src/lttng-ci/

    builders:
      - shell: !include-raw-escape: scripts/system-tests/checkout-modules.sh
      - shell: !include-raw-escape: scripts/system-tests/generate-properties-slave.sh
      - inject:
          properties-file: properties.txt
      - shell: !include-raw-escape: scripts/system-tests/check-build-needs.sh
      - conditional-step:
          condition-kind: not
          condition-operand:
              condition-kind: file-exists
              condition-filename: kernel-built.txt
              condition-basedir: workspace
          steps:
              - shell: !include-raw-escape: scripts/system-tests/build-kernel.sh
      - conditional-step:
          condition-kind: not
          condition-operand:
              condition-kind: file-exists
              condition-filename: modules-built.txt
              condition-basedir: workspace
          steps:
              - shell: !include-raw-escape: scripts/system-tests/build-modules.sh

    publishers:
      - workspace-cleanup

- job-template:
    name: system_ALL_{test_type}_trigger
    description: |
      This job will trigger the build of jobs when a new tag is push specific
      tracked Linux branches and new commits on LTTng tracked branches

    project-type: freestyle
    node: 'master'
    parameters:
      - bool:
          name: 'FORCE_JOB_RUN'
          default: false
          description: 'Force the child jobs to run'
      - bool:
          name: 'FORCE_FAILED_JOB_RUN'
          default: false
          description: 'Force the failed child jobs to run'

    properties:
      - build-discarder:
          num-to-keep: 10

    triggers:
      - timed: "H 0 * * 1-5"

    wrappers:
      - timestamps
      - ansicolor

    builders:
      - system-groovy:
         command:
           !include-raw-escape: scripts/system-tests/system-trigger.groovy

    publishers:
      - system_tests_email_ext_default

## Project

# Canary jobs are made to confirm that the whole Lava pipeline is working.
# They are scheduled once a day always on the same (LTTng, Kernel) code. If any
# of those jobs fails, it means that there is an issue with the configuration

# Only build the "supported" version against the master kernel since
# "unsupported" version do not get backport for new kernel.
- project:
    name: system-tests-supported-kmaster
    email_to: 'ci-notification@lists.lttng.org, cc:jgalar@efficios.com'
    kversion:
      - master
    lttngversion:
      - master
      - stable-2.12
      - stable-2.13
    jobs:
      - 'vm_tests_k{kversion}_l{lttngversion}'
      - 'baremetal_tests_k{kversion}_l{lttngversion}'

# Test against particular linux version.
- project:
    name: system-tests
    email_to: 'ci-notification@lists.lttng.org, cc:jgalar@efficios.com'
    kversion:
      - canary
      - linux-4.9.y
      - linux-4.14.y
      - linux-4.19.y
      - linux-5.4.y
      - linux-5.10.y
      - linux-5.15.y
      - linux-5.16.y
    lttngversion:
      - canary
      - master
      - stable-2.12
      - stable-2.13
    jobs:
      - 'vm_tests_k{kversion}_l{lttngversion}'
      - 'baremetal_tests_k{kversion}_l{lttngversion}'
- project:
    name: system-tests-vm-only
    email_to: 'ci-notification@lists.lttng.org, cc:jgalar@efficios.com'
    kversion:
      - linux-3.18.y
    lttngversion:
      - master
      - stable-2.12
      - stable-2.13
    jobs:
      - 'vm_tests_k{kversion}_l{lttngversion}'

- project:
    name: system-general
    test_type:
      - vm_tests
      - baremetal_tests
    jobs:
      - 'build_kernel_PARAM'
      - 'system_ALL_{test_type}_trigger'