jobs/system-tests.yaml

   1 ---
   2 ## Anchors
   3 - _system_tests_parameters_defaults: &system_tests_parameters_defaults
   4     name: 'system_tests_parameters_defaults'
   5     parameters:
   6       - string:
   7           name: 'LTTNG_TOOLS_COMMIT_ID'
   8           default: ''
   9           description: 'The lttng-tools commit id to build.'
  10       - string:
  11           name: 'LTTNG_MODULES_COMMIT_ID'
  12           default: ''
  13           description: 'The lttng-modules commit id to build.'
  14       - string:
  15           name: 'LTTNG_UST_COMMIT_ID'
  16           default: ''
  17           description: 'The lttng-ust commit id to build.'
  18       - string:
  19           name: 'KERNEL_TAG_ID'
  20           default: ''
  21           description: 'The linux kernel git tag to build against.'
  22       - string:
  23           name: 'KERNEL_REPO'
  24           default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git'
  25           description: 'Linux kernel git repo to checkout the kernel id'
  26       - string:
  27           name: 'LTTNG_TOOLS_REPO'
  28           default: 'https://github.com/lttng/lttng-tools.git'
  29           description: 'LTTng-Tools git repo to checkout the tools id'
  30       - string:
  31           name: 'LTTNG_MODULES_REPO'
  32           default: 'https://github.com/lttng/lttng-modules.git'
  33           description: 'LTTng-Modules git repo to checkout the Modules id'
  34       - string:
  35           name: 'LTTNG_UST_REPO'
  36           default: 'https://github.com/lttng/lttng-ust.git'
  37           description: 'LTTng-UST git repo to checkout the UST id'
  38       - string:
  39           name: 'ROOTFS_URL'
  40           default: 'https://obj.internal.efficios.com/lava/rootfs_amd64_bookworm_2024-01-15.tar.gz'
  41           description: 'The URL at which the system root FS can be downloaded'
  42       - string:
  43           name: 'LTTNG_CI_REPO'
  44           default: 'https://github.com/lttng/lttng-ci.git'
  45           description: 'LTTng-ci git repo to checkout the CI scripts'
  46       - string:
  47           name: 'LTTNG_CI_BRANCH'
  48           default: 'master'
  49           description: 'The branch of the CI repository to clone for job scripts'
  50       - string:
  51           name: 'LAVA_HOST'
  52           default: 'lava-master-03.internal.efficios.com'
  53           description: 'The hostname of the LAVA instance'
  54       - choice:
  55           name: 'LAVA_PROTO'
  56           choices:
  57             - 'http'
  58             - 'https'
  59           description: 'The protocol to use with the LAVA host'
  60
  61 - publisher:
  62     name: 'system_tests_email_ext_default'
  63     publishers:
  64       - email-ext:
  65           recipients: 'ci-notification@lists.lttng.org'
  66           reply-to: ci-notification@lists.lttng.org
  67           always: false
  68           unstable: false
  69           first-failure: true
  70           first-unstable: true
  71           not-built: false
  72           aborted: false
  73           regression: false
  74           failure: false
  75           second-failure: false
  76           improvement: false
  77           still-failing: false
  78           success: false
  79           fixed: false
  80           fixed-unhealthy: true
  81           still-unstable: false
  82           pre-build: false
  83           matrix-trigger: only-parent
  84           send-to:
  85             - recipients
  86
  87 - job-template:
  88     name: vm_tests_k{kversion}_l{lttngversion}
  89     description: |
  90       Runs root_regression and root_destructive_tests on a virtual machine over different combinations of kernel and lttng configurations.
  91     project-type: freestyle
  92     node: 'deb12-amd64'
  93     <<: *system_tests_parameters_defaults
  94
  95     properties:
  96       - build-discarder:
  97           num-to-keep: 10
  98       - throttle:
  99           max-total: 4
 100           option: 'category'
 101           categories:
 102             - 'kvm-tests'
 103
 104     wrappers:
 105       - workspace-cleanup
 106       - timestamps
 107       - ansicolor
 108       - credentials-binding:
 109           - text:
 110               credential-id: jenkins_lava_key
 111               variable: LAVA_JENKINS_TOKEN
 112           - text:
 113               credential-id: jenkins_lava2_key
 114               variable: LAVA2_JENKINS_TOKEN
 115       - inject:
 116           properties-content: |
 117             BUILD_DEVICE=kvm
 118             LTTNG_VERSION={lttngversion}
 119             LTTNG_CI_REPO=${{LTTNG_CI_REPO}}
 120             LTTNG_CI_BRANCH=${{LTTNG_CI_BRANCH}}
 121             LAVA_HOST=${{LAVA_HOST}}
 122             LAVA_PROTO=${{LAVA_PROTO}}
 123
 124     scm:
 125       - git:
 126           url: "${{LTTNG_CI_REPO}}"
 127           basedir: src/lttng-ci/
 128           branches:
 129             - "${{LTTNG_CI_BRANCH}}"
 130
 131     builders:
 132       - shell: !include-raw-escape: scripts/system-tests/generate-properties-master.sh
 133       - trigger-builds:
 134         - project: "build_kernel_PARAM"
 135           property-file: 'properties.txt'
 136           block: true
 137       - inject:
 138           properties-file: properties.txt
 139       - shell: !include-raw-escape: scripts/system-tests/run-kvm-tests.sh
 140
 141     publishers:
 142       - workspace-cleanup
 143       - email-ext:
 144           recipients: '{obj:email_to}'
 145           always: false
 146           unstable: false
 147           first-failure: true
 148           first-unstable: true
 149           not-built: false
 150           aborted: false
 151           regression: false
 152           failure: false
 153           second-failure: false
 154           improvement: false
 155           still-failing: true
 156           success: false
 157           fixed: false
 158           fixed-unhealthy: true
 159           still-unstable: false
 160           pre-build: false
 161           matrix-trigger: only-parent
 162           send-to:
 163             - recipients
 164
 165 - job-template:
 166     name: baremetal_tests_k{kversion}_l{lttngversion}
 167     description: |
 168       Runs perf_regression tests on a baremetal machine over different combinations of kernel and lttng configurations.
 169     project-type: freestyle
 170     node: 'deb12-amd64'
 171
 172     <<: *system_tests_parameters_defaults
 173
 174     properties:
 175       - build-discarder:
 176           num-to-keep: 10
 177           artifact-num-to-keep: 2
 178       - throttle:
 179           max-total: 2
 180           option: 'category'
 181           categories:
 182             - 'baremetal-tests'
 183
 184     wrappers:
 185       - workspace-cleanup
 186       - timestamps
 187       - ansicolor
 188       - credentials-binding:
 189           - text:
 190               credential-id: jenkins_lava_key
 191               variable: LAVA_JENKINS_TOKEN
 192           - text:
 193               credential-id: jenkins_lava2_key
 194               variable: LAVA2_JENKINS_TOKEN
 195       - inject:
 196           properties-content: |
 197             BUILD_DEVICE=baremetal
 198             LTTNG_VERSION={lttngversion}
 199             LTTNG_CI_REPO=${{LTTNG_CI_REPO}}
 200             LTTNG_CI_BRANCH=${{LTTNG_CI_BRANCH}}
 201             LAVA_HOST=${{LAVA_HOST}}
 202             LAVA_PROTO=${{LAVA_PROTO}}
 203     scm:
 204       - git:
 205           url: "${{LTTNG_CI_REPO}}"
 206           basedir: src/lttng-ci/
 207           branches:
 208             - "${{LTTNG_CI_BRANCH}}"
 209
 210     builders:
 211       - shell: !include-raw-escape: scripts/system-tests/generate-properties-master.sh
 212       - trigger-builds:
 213         - project: "build_kernel_PARAM"
 214           property-file: 'properties.txt'
 215           block: true
 216       - inject:
 217           properties-file: properties.txt
 218       - shell: !include-raw-escape: scripts/system-tests/run-baremetal-tests.sh
 219
 220     publishers:
 221       - workspace-cleanup
 222
 223 - job-template:
 224     name: build_kernel_PARAM
 225     description: |
 226       Builds a Linux Kernel and LTTng Modules if necessary
 227     concurrent: true
 228     node: 'deb12-amd64'
 229
 230     parameters:
 231       - string:
 232           name: 'LTTNG_MODULES_COMMIT_ID'
 233           description: 'The lttng-modules commmit to build.'
 234       - string:
 235           name: 'LTTNG_MODULES_REPO'
 236           description: 'The LTTng Modules git repo to fetch from'
 237           default: 'git://git-mirror.internal.efficios.com/lttng/lttng-modules.git'
 238       - string:
 239           name: 'KERNEL_COMMIT_ID'
 240           description: 'The kernel commit to build.'
 241       - string:
 242           name: 'KGITREPO'
 243           description: 'The kernel git repo to fetch from'
 244       - string:
 245           name: 'STORAGE_KERNEL_FOLDER'
 246           description: 'Path to store the Kernel image'
 247       - string:
 248           name: 'STORAGE_KERNEL_IMAGE'
 249           description: 'Path to store the Kernel IMAGE'
 250       - string:
 251           name: 'STORAGE_LINUX_MODULES'
 252           description: 'Path to store the Kernel Modules'
 253       - string:
 254           name: 'STORAGE_LTTNG_MODULES'
 255           description: 'Path to store the LTTng Modules'
 256       - string:
 257           name: 'BUILD_DEVICE'
 258           description: 'The target device. (kvm or baremetal)'
 259       - string:
 260           name: 'S3_STORAGE_KERNEL_FOLDER'
 261           description: 'Path to store the Kernel image'
 262       - string:
 263           name: 'S3_STORAGE_KERNEL_IMAGE'
 264           description: 'Path to store the Kernel IMAGE'
 265       - string:
 266           name: 'S3_STORAGE_LINUX_MODULES'
 267           description: 'Path to store the Kernel Modules'
 268       - string:
 269           name: 'S3_STORAGE_LTTNG_MODULES'
 270           description: 'Path to store the LTTng Modules'
 271       - string:
 272           name: 'LTTNG_CI_REPO'
 273           default: 'https://github.com/lttng/lttng-ci.git'
 274           description: 'LTTng-ci git repo to checkout the CI scripts'
 275       - string:
 276           name: 'LTTNG_CI_BRANCH'
 277           default: 'master'
 278           description: 'The branch of the CI repository to clone for job scripts'
 279
 280     properties:
 281       - build-discarder:
 282           num-to-keep: 100
 283
 284     wrappers:
 285       - workspace-cleanup
 286       - timestamps
 287       - ansicolor
 288       - credentials-binding:
 289           - file:
 290               credential-id: system_tests_storage_key
 291               variable: identity_file
 292
 293     scm:
 294       - git:
 295           url: "${{LTTNG_CI_REPO}}"
 296           basedir: src/lttng-ci/
 297           branches:
 298             - "${{LTTNG_CI_BRANCH}}"
 299
 300     builders:
 301       - shell: !include-raw-escape: scripts/system-tests/checkout-modules.sh
 302       - shell: !include-raw-escape: scripts/system-tests/generate-properties-slave.sh
 303       - inject:
 304           properties-file: properties.txt
 305       - shell: !include-raw-escape: scripts/system-tests/check-build-needs.sh
 306       - conditional-step:
 307           condition-kind: not
 308           condition-operand:
 309               condition-kind: file-exists
 310               condition-filename: kernel-built.txt
 311               condition-basedir: workspace
 312           steps:
 313               - shell: !include-raw-escape: scripts/system-tests/build-kernel.sh
 314       - conditional-step:
 315           condition-kind: not
 316           condition-operand:
 317               condition-kind: file-exists
 318               condition-filename: modules-built.txt
 319               condition-basedir: workspace
 320           steps:
 321               - shell: !include-raw-escape: scripts/system-tests/build-modules.sh
 322
 323     publishers:
 324       - workspace-cleanup
 325
 326 - job-template:
 327     name: system_ALL_{test_type}_trigger
 328     description: |
 329       This job will trigger the build of jobs when a new tag is push specific
 330       tracked Linux branches and new commits on LTTng tracked branches
 331
 332     project-type: freestyle
 333     node: 'master'
 334     parameters:
 335       - bool:
 336           name: 'FORCE_JOB_RUN'
 337           default: false
 338           description: 'Force the child jobs to run'
 339       - bool:
 340           name: 'FORCE_FAILED_JOB_RUN'
 341           default: true
 342           description: 'Force the failed child jobs to run'
 343
 344     properties:
 345       - build-discarder:
 346           num-to-keep: 10
 347
 348     triggers:
 349       - timed: "H 0 * * 1-5"
 350
 351     wrappers:
 352       - timestamps
 353       - ansicolor
 354
 355     builders:
 356       - system-groovy:
 357          command:
 358            !include-raw-escape: scripts/system-tests/system-trigger.groovy
 359
 360     publishers:
 361       - system_tests_email_ext_default
 362
 363 ## Project
 364
 365 # Canary jobs are made to confirm that the whole Lava pipeline is working.
 366 # They are scheduled once a day always on the same (LTTng, Kernel) code. If any
 367 # of those jobs fails, it means that there is an issue with the configuration
 368
 369 # Only build the "supported" version against the master kernel since
 370 # "unsupported" version do not get backport for new kernel.
 371 - project:
 372     name: system-tests-supported-kmaster
 373     email_to: 'ci-notification@lists.lttng.org, cc:jgalar@efficios.com'
 374     kversion:
 375       - master
 376     lttngversion:
 377       - master
 378       - stable-2.13
 379     jobs:
 380       - 'vm_tests_k{kversion}_l{lttngversion}'
 381       - 'baremetal_tests_k{kversion}_l{lttngversion}'
 382
 383 ## Views
 384 - view-template:
 385     name: 'System tests'
 386     view-type: list
 387     regex: 'vm_tests_.*|baremetal_tests_.*|baremetal_benchmarks_.*|system_.*'
 388
 389 # Test against particular linux version.
 390 - project:
 391     name: system-tests
 392     email_to: 'ci-notification@lists.lttng.org, cc:jgalar@efficios.com'
 393     kversion:
 394       - canary
 395       - linux-4.9.y
 396       - linux-4.14.y
 397       - linux-4.19.y
 398       - linux-5.4.y
 399       - linux-5.10.y
 400       - linux-5.15.y
 401       - linux-5.16.y
 402       - linux-6.1.y
 403       - linux-6.6.y
 404     lttngversion:
 405       - canary
 406       - master
 407       - stable-2.13
 408     jobs:
 409       - 'vm_tests_k{kversion}_l{lttngversion}'
 410       - 'baremetal_tests_k{kversion}_l{lttngversion}'
 411 - project:
 412     name: system-tests-2.12
 413     email_to: 'ci-notification@lists.lttng.org, cc:jgalar@efficios.com'
 414     kversion:
 415       - linux-4.9.y
 416       - linux-4.14.y
 417       - linux-4.19.y
 418       - linux-5.4.y
 419       - linux-5.10.y
 420       - linux-5.15.y
 421       - linux-5.16.y
 422     lttngversion:
 423       - stable-2.12
 424     jobs:
 425       - 'vm_tests_k{kversion}_l{lttngversion}'
 426       - 'baremetal_tests_k{kversion}_l{lttngversion}'
 427 - project:
 428     name: system-tests-vm-only
 429     email_to: 'ci-notification@lists.lttng.org, cc:jgalar@efficios.com'
 430     kversion:
 431       - linux-3.18.y
 432     lttngversion:
 433       - stable-2.12
 434       - stable-2.13
 435     jobs:
 436       - 'vm_tests_k{kversion}_l{lttngversion}'
 437
 438 - project:
 439     name: system-general
 440     test_type:
 441       - vm_tests
 442       - baremetal_tests
 443     jobs:
 444       - 'build_kernel_PARAM'
 445       - 'system_ALL_{test_type}_trigger'
 446
 447 - project:
 448     name: system-tests-views
 449     views:
 450       - 'System tests'