jjb: Add lttng-modules builds across certain EL kernels
authorKienan Stewart <kstewart@efficios.com>
Thu, 14 Mar 2024 12:49:30 +0000 (08:49 -0400)
committerKienan Stewart <kstewart@efficios.com>
Fri, 22 Mar 2024 18:58:37 +0000 (14:58 -0400)
Change-Id: I99675d49a910ffbbbf841901200524a8f7839603
Signed-off-by: Kienan Stewart <kstewart@efficios.com>
automation/ansible/roles/gitmirror/files/update.sh
jobs/lttng-modules.yaml
scripts/lttng-modules/master.groovy
scripts/lttng-modules/param-build.sh

index a804c03795f056f2006cb538e6720cfd062b5d95..d7fa11c08e8aa6be4131cf6dc6f39c4d5e025ea7 100644 (file)
@@ -39,6 +39,11 @@ git tag -d v2.6.11 || true
 git tag -d v2.6.11-tree || true
 popd
 
+##
+# EL kernel RPMs
+##
+update_git rocky.git/ https://git.rockylinux.org/staging/rpms/kernel.git
+
 ##
 # SLES kernels
 ##
index b3d5646bbe41eccb636befa93ed59fca2abe9a0e..ed98b73c96fda7571ceed65523615f24d35b72b3 100644 (file)
@@ -45,7 +45,7 @@
           strategy: statechange-only
           message-type: summary
           matrix-notifier: only-parent
-      - email-ext:
+      - email-ext: &lttng-modules_publisher_email-ext_defaults
           recipients: '{obj:email_to}'
           reply-to: ci-notification@lists.lttng.org
           always: false
           default: 'lttng-modules_PARAM_build'
           description: 'The parametrized job to use for child builds.'
 
+- _lttng-modules_build_parameters_ubuntu: &lttng-modules_build_parameters_el
+    name: 'lttng-modules_build_parameters_el'
+    parameters:
+      - string:
+          name: 'maxConcurrentBuild'
+          default: '20'
+          description: 'The maximum number of concurrent child build to run.'
+      - string:
+          name: 'kverfloor'
+          default: '{kverfloor}'
+          description: 'The lowest kernel version to build.'
+      - string:
+          name: 'kverceil'
+          default: '{kverceil}'
+          description: 'The highest kernel version to build. (excluded)'
+      - string:
+          name: 'kverfilter'
+          default: '{kverfilter}'
+          description: 'Kernel versions list filtering strategy.'
+      - string:
+          name: 'kverrc'
+          default: 'false'
+          description: 'Add latest RC to kernel versions list.'
+      - string:
+          name: 'kgitrepo'
+          default: 'git://git-mirror.internal.efficios.com/git/rocky.git'
+          description: 'The linux kernel git repository url.'
+      - string:
+          name: 'kbuildjob'
+          default: 'lttng-modules_PARAM_{parambuildtype}'
+          description: 'The parametrized job to use for child builds.'
+      - string:
+          name: 'elversion'
+          default: '{elversion}'
+          description: 'The EL release (eg. el8)'
+      - string:
+          name: 'getsrc_repo'
+          default: 'https://github.com/rocky-linux/rocky-tools.git'
+          description: 'The git repository that contains the getsrc tool'
+      - string:
+          name: 'getsrc_version'
+          default: 'main'
+          description: 'The tag or commit to use when cloning the getsrc tool'
+
 - _lttng-modules_build_parameters_ubuntu: &lttng-modules_build_parameters_sles
     name: 'lttng-modules_build_parameters_sles'
     parameters:
     publishers:
       - workspace-cleanup
 
+- job-template:
+    name: 'dev_review_lttng-modules_{mversion}_build-{elversion}'
+    defaults: lttng-modules
+    concurrent: true
+
+    node: 'master'
+
+    scm:
+      - git:
+          url: https://review.lttng.org/lttng-modules
+          refspec: 'refs/changes/*:refs/changes/*'
+          branches:
+            - '$GERRIT_REFSPEC'
+          basedir: src/lttng-modules
+          skip-tag: true
+
+    triggers:
+      - gerrit:
+          trigger-on:
+            - comment-added-event:
+                approval-category: 'CI-Build'
+                approval-value: 1
+          projects:
+            - project-compare-type: 'PLAIN'
+              project-pattern: 'lttng-modules'
+              branches:
+                - branch-compare-type: 'PLAIN'
+                  branch-pattern: '{mversion}'
+
+    properties:
+      - build-discarder:
+          days-to-keep: 1
+      - throttle:
+          option: 'category'
+          categories:
+            - 'gerrit-build'
+
+    <<: *lttng-modules_build_parameters_el
+    <<: *lttng-modules_build_builders_defaults
+
+    publishers:
+      - workspace-cleanup
+
 - job-template:
     name: 'dev_review_lttng-modules_{mversion}_build-{slesversion}'
     defaults: lttng-modules
       - pollscm:
           cron: "0 0 * * 5"
 
+- job-template:
+    name: 'lttng-modules_{mversion}_{buildtype}-{elversion}'
+    defaults: lttng-modules
+    description: |
+      The LTTng modules provide Linux kernel tracing capability to the LTTng
+      2.0 tracer toolset.
+
+      This job will the {mversion} branch against EL {elversion} released
+      kernels.
+
+      <p>Job is managed by Jenkins Job Builder.</p>
+
+    node: 'master'
+    triggers:
+      - pollscm:
+          cron: "@hourly"
+      - timed: "H 0 * * 6"
+    <<: *lttng-modules_build_parameters_el
+    <<: *lttng-modules_build_builders_defaults
+
 - job-template:
     name: lttng-modules_{mversion}_{buildtype}-{slesversion}
     defaults: lttng-modules
           name: 'kgitrepo'
           default: 'git://git-mirror.internal.efficios.com/git/linux-all.git'
           description: 'The linux kernel git repository url.'
+      - string:
+          name: 'distroversion'
+          default: ''
+          description: 'Supplementary distribution version, eg. sles15sp4 or el8'
+      - string:
+          name: 'getsrc_repo'
+          default: 'https://github.com/rocky-linux/rocky-tools.git'
+          description: 'The git repository that contains the getsrc tool'
+      - string:
+          name: 'getsrc_version'
+          default: 'main'
+          description: 'The tag or commit to use when cloning the getsrc tool'
 
     concurrent: true
 
     triggers: []
 
     builders:
+      - conditional-step:
+          condition-kind: regex-match
+          label: '$distroversion'
+          regex: '^el.*$'
+          steps:
+            - shell: |
+                #!/bin/bash
+                set -exu
+                git clone -b "$getsrc_version" "$getsrc_repo" src/getsrc
       - shell:
           !include-raw-escape: scripts/lttng-modules/param-build.sh
 
           kverceil: v5.18-rc1
           kverfilter: none
 
+# EL normal builds
+      - 'lttng-modules_{mversion}_{buildtype}-{elversion}':
+          mversion:
+            - master
+            - stable-2.13
+            - stable-2.12
+          elversion:
+            - el8
+          buildtype: build
+          parambuildtype: build
+          kverfloor: ''
+          kverceil: ''
+          kverfilter: stable-head
+
 # RT normal builds
       - 'lttng-modules_{mversion}_build-rt':
           mversion: master
           kverfloor: v3.0
           kverceil: v5.18
           kverfilter: lts-head
+      - 'dev_review_lttng-modules_{mversion}_build-{elversion}':
+          mversion:
+            - master
+            - stable-2.13
+            - stable-2.12
+          elversion:
+            - el8
+          buildtype: build
+          parambuildtype: build
+          kverfloor: ''
+          kverceil: ''
+          kverfilter: stable-head
       - 'dev_review_lttng-modules_{mversion}_build-{slesversion}':
           mversion:
             - master
index 92fb0c355755e384e4ba7609aaf908008f050116..5c515b7e8497fc83673b9a4b2be296e3808c84e2 100644 (file)
@@ -163,6 +163,127 @@ class VanillaKVersion implements Comparable<VanillaKVersion> {
   }
 }
 
+class ElKVersion implements Comparable<ElKVersion> {
+  Integer major = 0
+  Integer minor = 0
+  Integer patch = 0
+  Integer elmajor = 0
+  Integer elminor = 0
+  Integer elpatch = 0
+  Integer el_release_major = 0
+  Integer el_release_minor = 0
+  Boolean append_zero_el_release_minor = false
+  String version_prefix = ""
+
+  ElKVersion() {}
+
+  ElKVersion(version) {
+    this.parse(version)
+  }
+
+  static ElKVersion minKVersion() {
+    return new ElKVersion("kernel-0.0.0-0.0.0.el0_0")
+  }
+
+  static ElKVersion maxKVersion() {
+    return new ElKVersion("kernel-" + Integer.MAX_VALUE + ".0.0-0.0.0.el0_0")
+  }
+
+  static ElKVersion factory(version) {
+    return new ElKVersion(version)
+  }
+
+  def parse(version) {
+    this.major = 0
+    this.minor = 0
+    this.patch = 0
+    this.elmajor = 0
+    this.elminor = 0
+    this.elpatch = 0
+    this.el_release_major = 0
+    this.el_release_minor = 0
+    this.append_zero_el_release_minor = false
+    this.version_prefix = ""
+
+    if (!version) {
+      throw new EmptyKVersionException("Empty kernel version")
+    }
+
+    // Eg. imports/r8/kernel-4.18.0-513.18.1.el8_9
+    // Eg. imports/r9/kernel-5.14.0-362.13.1.el9_3
+    // Eg. imports/r8s/kernel-4.18.0-528.el8
+    // Eg. imports/r8/kernel-4.18.0-80.1.2.el8_0
+    // Eg. imports/r8/kernel-4.18.0-348.el8
+    def match = version =~ /^([\w\d\/]*kernel-)??(\d+)\.(\d+)\.(\d+)-(\d+)(\.(\d+))??(\.(\d+))??\.el(\d+)(_(\d+))??$/
+    if (!match) {
+      throw new InvalidKVersionException("Invalid kernel version: ${version}")
+    }
+
+    if (match.group(1)) {
+      this.version_prefix = match.group(1)
+    }
+    this.major = Integer.parseInt(match.group(2))
+    this.minor = Integer.parseInt(match.group(3))
+    this.patch = Integer.parseInt(match.group(4))
+    this.elmajor = Integer.parseInt(match.group(5))
+    if (match.group(7)) {
+      this.elminor = Integer.parseInt(match.group(7))
+    }
+    if (match.group(9)) {
+      this.elpatch = Integer.parseInt(match.group(9))
+    }
+    if (match.group(10)) {
+      this.el_release_major = Integer.parseInt(match.group(10))
+    }
+    if (match.group(12)) {
+      this.el_release_minor = Integer.parseInt(match.group(12))
+      this.append_zero_el_release_minor = true
+    }
+  }
+
+  Boolean isRC() {
+    return false
+  }
+
+  Boolean isSameStable(ElKVersion o) {
+    def properties = ['major', 'minor', 'patch', 'elmajor', 'elminor', 'elpatch', 'el_release_major', 'el_release_minor']
+    for (property in properties) {
+      if (this."$property" != o."$property") {
+        return false
+      }
+    }
+    return true
+  }
+
+  @Override int compareTo(ElKVersion o) {
+    def properties = ['major', 'minor', 'patch', 'elmajor', 'elminor', 'elpatch', 'el_release_major', 'el_release_minor']
+    for (property in properties) {
+      if (this."$property" != o."$property") {
+        return Integer.compare(this."$property", o."$property")
+      }
+    }
+    return 0
+  }
+
+  String toString() {
+    String vString = "${this.version_prefix}${this.major}.${this.minor}.${this.patch}-${this.elmajor}"
+    // Sometimes a tag can have no elminor, eg. imports/r8/kernel-4.18.0-80.el8
+    if (this.elminor != 0) {
+      vString = vString.concat(".${this.elminor}")
+    }
+    if (this.elpatch != 0) {
+      vString = vString.concat(".${this.elpatch}")
+    }
+    vString = vString.concat(".el${this.el_release_major}")
+    // Some tags have a trailing el_release_minor that is 0, eg.
+    // imports/r8/kernel-4.18.0-80.1.2.el8
+    if (this.el_release_minor != 0 || (this.el_release_minor == 0 && this.append_zero_el_release_minor)) {
+      vString = vString.concat("_${this.el_release_minor}")
+    }
+    return vString
+  }
+}
+
 class SlesKVersion implements Comparable<SlesKVersion> {
   Integer major = 0
   Integer minor = 0
@@ -430,6 +551,7 @@ def kverfloor_raw = build.buildVariableResolver.resolve('kverfloor')
 def kverceil_raw = build.buildVariableResolver.resolve('kverceil')
 def kverfilter = build.buildVariableResolver.resolve('kverfilter')
 def kverrc = build.buildVariableResolver.resolve('kverrc')
+def elversion = build.buildVariableResolver.resolve('elversion')
 def slesversion = build.buildVariableResolver.resolve('slesversion')
 def uversion = build.buildVariableResolver.resolve('uversion')
 def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob'))
@@ -456,8 +578,30 @@ def kversionsRC = []
 def matchStrs = []
 def blacklist = []
 def kversionFactory = ""
+def distroversion = ""
 
-if (slesversion != null) {
+if (elversion != null) {
+  distroversion = elversion
+  kversionFactory = new ElKVersion()
+  switch (elversion) {
+    case 'el8':
+      matchStrs = [
+        // EL 8 (all)
+        // ~/^refs\/tags\/(imports\/r8\/kernel-[\d\.-]+.el8.*)$/,
+        // EL 8.4
+        ~/^refs\/tags\/(imports\/r8\/kernel-4\.18\.0-305\.[\d\.-]+.el8.*)$/,
+        // EL 8.6
+        ~/^refs\/tags\/(imports\/r8\/kernel-4\.18\.0-372\.[\d\.-]+.el8.*)$/,
+      ]
+      blacklist = []
+      break
+    default:
+      println ("Unsupport EL version: ${elversion}")
+      throw new InterruptedException()
+      break
+  }
+} else if (slesversion != null) {
+  distroversion = slesversion
   kversionFactory = new SlesKVersion()
   switch (slesversion) {
     case 'sles15sp4':
@@ -478,6 +622,7 @@ if (slesversion != null) {
       break
   }
 } else if (uversion != null) {
+  distroversion = uversion
   kversionFactory = new UbuntuKVersion()
   switch (uversion) {
     case 'jammy':
@@ -676,6 +821,9 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) {
       new StringParameterValue('mgitrepo', mgitrepo),
       new StringParameterValue('ktag', kversion.toString()),
       new StringParameterValue('kgitrepo', kgitrepo),
+      new StringParameterValue('distroversion', distroversion),
+      new StringParameterValue('getsrc_repo', build.buildVariableResolver.resolve('getsrc_repo')),
+      new StringParameterValue('getsrc_version', build.buildVariableResolver.resolve('getsrc_version'))
     ]
 
     // Launch the parametrized build
index 42d076739ed5e411b9b5e91e36b486f4c1313513..cf3689d5b615bb013ec532f718c608467d2885b5 100644 (file)
@@ -96,7 +96,32 @@ git_clone_modules_sources() {
 # Checkout a shallow kernel tree of the specified tag
 git_clone_linux_sources() {
     mkdir -p "$LINUX_GIT_DIR"
-    git clone --depth=1 -b "${ktag}" --reference-if-able "$LINUX_GIT_REF_REPO_DIR" "${kgitrepo}" "$LINUX_GIT_DIR"
+    case "${distroversion:-}" in
+        el*)
+            git clone -b "${ktag}" "${kgitrepo}" src/linux-rpm
+
+            # Get the source files
+            pushd src/linux-rpm
+            "${WORKSPACE}/src/getsrc/getsrc/getsrc.sh"
+            tar -x -C "${LINUX_GIT_DIR}" --strip-components=1 -f SOURCES/linux-*.tar.xz
+            popd
+
+            # Pretend we're a repo like the default expects
+            pushd "${LINUX_GIT_DIR}"
+            git init .
+            git config user.name 'Jenkins'
+            git config user.email 'jenkins@efficios.com'
+            git add .
+            git commit -a -m 'Initial commit'
+            git tag "${ktag}"
+            echo "${LINUX_GIT_REF_REPO_DIR}" > .git/objects/info/alternates
+            popd
+            ;;
+
+        *)
+            git clone --depth=1 -b "${ktag}" --reference-if-able "$LINUX_GIT_REF_REPO_DIR" "${kgitrepo}" "$LINUX_GIT_DIR"
+            ;;
+    esac
 }
 
 
@@ -311,6 +336,35 @@ build_linux_kernel() {
 
     # Generate kernel configuration
     case "$ktag" in
+      *el*)
+        # Copy the EL kernel configuration
+        el_arch="${cross_arch:-$arch}"
+        case "${el_arch}" in
+          amd64)
+            el_arch=x86_64
+            ;;
+          arm64)
+            el_arch=aarch64
+            ;;
+          ppc64el)
+            el_arch=ppc64le
+            ;;
+          *)
+            ;;
+        esac
+        ls "${WORKSPACE}/src/linux-rpm/SOURCES/"
+        if [ -f "${WORKSPACE}/src/linux-rpm/SOURCES/kernel-${el_arch}.config" ] ; then
+          cp "${WORKSPACE}/src/linux-rpm/SOURCES/kernel-${el_arch}.config" .config
+        elif [ -f "${WORKSPACE}/src/linux-rpm/SOURCES/kernel-${el_arch}-rhel.config" ] ; then
+          cp "${WORKSPACE}/src/linux-rpm/SOURCES/kernel-${el_arch}-rhel.config" .config
+        fi
+
+        # Eg.
+        # mm/mempolicy.c: In function ‘mpol_parse_str’:
+        # mm/mempolicy.c:2980:26: error: writing 1 byte into a region of size 0 [-Werror=stringop-overflow=]
+        export KCFLAGS="${KCFLAGS} -Wno-error -Wno-all -Wno-error=stringop-overflow -Wno-error=address-of-packed-member"
+        ;;
+
       Ubuntu*)
         if [ "${cross_arch}" = "powerpc" ] && vergte "${kversion}" "4.10"; then
           echo "Ubuntu removed big endian powerpc configuration from kernel >= 4.10. Don't try to build it."
@@ -677,7 +731,9 @@ build_linux_kernel() {
 
     if [ "$(scripts/config --state CONFIG_DEBUG_INFO_BTF)" == "y" ] &&
            { vergte "${pahole_version}" "1.24"; } &&
-           { vergte "${kversion}" "5.10"; } && { verlt "${kversion}" "6.0"; } ;then
+           (
+               ( { vergte "${kversion}" "5.10"; } && { verlt "${kversion}" "6.0"; } ) || [[ "${ktag}" =~ .el8 ]]
+           ) ; then
         # Some kernels Eg. Ubuntu-hwe-5.13-5.13.0-52.59_20.04.1
         # fail with the following error:
         #   BTFIDS  vmlinux
@@ -696,6 +752,8 @@ build_linux_kernel() {
         else
             # shellcheck disable=SC2016
             sed -i 's/ -J ${extra_paholeopt} / -J ${extra_paholeopt} --skip_encoding_btf_enum64 /' scripts/link-vmlinux.sh
+            # Some older versions of RHEL don't have '${extra_paholeopt}'
+            sed -i 's/${PAHOLE} -J ${1}/${PAHOLE} -J --skip_encoding_btf_enum64 ${1}/' scripts/link-vmlinux.sh
         fi
     fi
 
This page took 0.030115 seconds and 4 git commands to generate.