scripts/system-tests/system-trigger.groovy

   1 /**
   2  * Copyright (C) 2017 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3  *
   4  * This program is free software: you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 import hudson.console.HyperlinkNote
  19 import hudson.model.*
  20 import java.io.File
  21 import org.eclipse.jgit.api.Git
  22 import org.eclipse.jgit.lib.Ref
  23
  24 class InvalidKVersionException extends Exception {
  25   public InvalidKVersionException(String message) {
  26     super(message)
  27   }
  28 }
  29
  30 class EmptyKVersionException extends Exception {
  31   public EmptyKVersionException(String message) {
  32     super(message)
  33   }
  34 }
  35
  36 class VanillaKVersion implements Comparable<VanillaKVersion> {
  37
  38   Integer major = 0
  39   Integer majorB = 0
  40   Integer minor = 0
  41   Integer patch = 0
  42   Integer rc = Integer.MAX_VALUE
  43   Boolean inStable = false;
  44
  45   VanillaKVersion() {}
  46
  47   VanillaKVersion(version) {
  48     this.parse(version)
  49   }
  50
  51   static VanillaKVersion minKVersion() {
  52     return new VanillaKVersion("v0.0.0")
  53   }
  54
  55   static VanillaKVersion maxKVersion() {
  56     return new VanillaKVersion("v" + Integer.MAX_VALUE + ".0.0")
  57   }
  58
  59   static VanillaKVersion factory(version) {
  60     return new VanillaKVersion(version)
  61   }
  62
  63   def parse(version) {
  64     this.major = 0
  65     this.majorB = 0
  66     this.minor = 0
  67     this.patch = 0
  68     this.rc = Integer.MAX_VALUE
  69
  70     if (!version) {
  71       throw new EmptyKVersionException("Empty kernel version")
  72     }
  73
  74     def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rc(\d+))?$/
  75     if (!match) {
  76       throw new InvalidKVersionException("Invalid kernel version: ${version}")
  77     }
  78
  79     Integer offset = 0;
  80
  81     // Major
  82     this.major = Integer.parseInt(match.group(1))
  83     if (this.major <= 2) {
  84       offset = 2
  85       this.majorB = Integer.parseInt(match.group(2))
  86     }
  87
  88     // Minor
  89     if (match.group(2 + offset) != null) {
  90       this.minor = Integer.parseInt(match.group(2 + offset))
  91     }
  92
  93     // Patch level
  94     if (match.group(4 + offset) != null) {
  95       this.patch = Integer.parseInt(match.group(4 + offset))
  96       this.inStable = true
  97     }
  98
  99     // RC
 100     if (match.group(8) != null) {
 101       this.rc = Integer.parseInt(match.group(8))
 102     }
 103   }
 104
 105   Boolean isInStableBranch() {
 106     return this.inStable
 107   }
 108
 109   // Return true if both version are of the same stable branch
 110   Boolean isSameStable(VanillaKVersion o) {
 111     if (this.major != o.major) {
 112       return false
 113     }
 114     if (this.majorB != o.majorB) {
 115       return false
 116     }
 117     if (this.minor != o.minor) {
 118       return false
 119     }
 120
 121     return true
 122   }
 123
 124   @Override int compareTo(VanillaKVersion o) {
 125     if (this.major != o.major) {
 126       return Integer.compare(this.major, o.major)
 127     }
 128     if (this.majorB != o.majorB) {
 129       return Integer.compare(this.majorB, o.majorB)
 130     }
 131     if (this.minor != o.minor) {
 132       return Integer.compare(this.minor, o.minor)
 133     }
 134     if (this.patch != o.patch) {
 135       return Integer.compare(this.patch, o.patch)
 136     }
 137     if (this.rc != o.rc) {
 138       return Integer.compare(this.rc, o.rc)
 139     }
 140
 141     // Same version
 142     return 0;
 143   }
 144
 145   String toString() {
 146     String vString = "v${this.major}"
 147
 148     if (this.majorB > 0) {
 149       vString = vString.concat(".${this.majorB}")
 150     }
 151
 152     vString = vString.concat(".${this.minor}")
 153
 154     if (this.patch > 0) {
 155       vString = vString.concat(".${this.patch}")
 156     }
 157
 158     if (this.rc > 0 && this.rc < Integer.MAX_VALUE) {
 159       vString = vString.concat("-rc${this.rc}")
 160     }
 161     return vString
 162   }
 163 }
 164
 165 // Save the hashmap containing all the jobs and their status to disk. We can do
 166 // that because this job is configured to always run on the master node on
 167 // Jenkins.
 168 def SaveCurrentJobsToWorkspace = { currentJobs, ondiskpath->
 169   try {
 170     File myFile = new File(ondiskpath);
 171     myFile.createNewFile();
 172     def out = new ObjectOutputStream(new FileOutputStream(ondiskpath))
 173     out.writeObject(currentJobs)
 174     out.close()
 175   } catch (e) {
 176     println("Failed to save previous Git object IDs to disk." + e);
 177   }
 178 }
 179
 180 // Load the hashmap containing all the jobs and their last status from disk.
 181 // It's possible because this job is configured to always run on the master
 182 // node on Jenkins
 183 def LoadPreviousJobsFromWorkspace = { ondiskpath ->
 184   def previousJobs = [:]
 185   try {
 186     File myFile = new File(ondiskpath);
 187     def input = new ObjectInputStream(new FileInputStream(ondiskpath))
 188     previousJobs = input.readObject()
 189     input.close()
 190   } catch (e) {
 191     println("Failed to load previous runs from disk." + e);
 192   }
 193   return previousJobs
 194 }
 195
 196
 197 def GetHeadCommits = { remoteRepo, branchesOfInterest ->
 198   def remoteHeads = [:]
 199   def remoteHeadRefs = Git.lsRemoteRepository()
 200                           .setTags(false)
 201                           .setHeads(true)
 202                           .setRemote(remoteRepo).call()
 203
 204   remoteHeadRefs.each {
 205     def branch = it.getName().replaceAll('refs/heads/', '')
 206     if (branchesOfInterest.contains(branch))
 207       remoteHeads[branch] = it.getObjectId().name()
 208   }
 209
 210   return remoteHeads
 211 }
 212
 213 def GetTagIds = { remoteRepo ->
 214   def remoteTags = [:]
 215   def remoteTagRefs = Git.lsRemoteRepository()
 216                          .setTags(true)
 217                          .setHeads(false)
 218                          .setRemote(remoteRepo).call()
 219
 220   remoteTagRefs.each {
 221     // Exclude release candidate tags
 222     if (!it.getName().contains('-rc')) {
 223       remoteTags[it.getName().replaceAll('refs/tags/', '')] = it.getObjectId().name()
 224     }
 225   }
 226
 227   return remoteTags
 228 }
 229
 230 def GetLastTagOfBranch = { tagRefs, branch ->
 231   def tagVersions = tagRefs.collect {new VanillaKVersion(it.key)}
 232   def currMax = new VanillaKVersion('v0.0.0');
 233   if (!branch.contains('master')){
 234     def targetVersion = new VanillaKVersion(branch.replaceAll('linux-', 'v').replaceAll('.y', ''))
 235     tagVersions.each {
 236       if (it.isSameStable(targetVersion)) {
 237         if (currMax < it) {
 238           currMax = it;
 239         }
 240       }
 241     }
 242   } else {
 243     tagVersions.each {
 244       if (!it.isInStableBranch() && currMax < it) {
 245         currMax = it;
 246       }
 247     }
 248   }
 249   return currMax.toString()
 250 }
 251
 252 // Returns the latest tags of each of the branches passed in the argument
 253 def GetLastTagIds = { remoteRepo, branchesOfInterest ->
 254   def remoteHeads = GetHeadCommits(remoteRepo, branchesOfInterest)
 255   def remoteTagRefs = GetTagIds(remoteRepo)
 256   def remoteLastTagCommit = [:]
 257
 258   remoteTagRefs = remoteTagRefs.findAll { !it.key.contains("v2.") }
 259   branchesOfInterest.each {
 260     remoteLastTagCommit[it] = remoteTagRefs[GetLastTagOfBranch(remoteTagRefs, it)]
 261   }
 262
 263   return remoteLastTagCommit
 264 }
 265
 266 def CraftJobName = { jobType, linuxBranch, lttngBranch ->
 267   return "${jobType}_k${linuxBranch}_l${lttngBranch}"
 268 }
 269
 270 def LaunchJob = { jobName, jobInfo ->
 271   def job = Hudson.instance.getJob(jobName)
 272   def params = []
 273   for (paramdef in job.getProperty(ParametersDefinitionProperty.class).getParameterDefinitions()) {
 274     // If there is a default value for this parameter, use it. Don't use empty
 275     // default value parameters.
 276     if (paramdef.getDefaultValue()) {
 277       params += paramdef.getDefaultParameterValue();
 278     }
 279   }
 280
 281   params.add(new StringParameterValue('LTTNG_TOOLS_COMMIT_ID', jobInfo['config']['toolsCommit']))
 282   params.add(new StringParameterValue('LTTNG_MODULES_COMMIT_ID', jobInfo['config']['modulesCommit']))
 283   params.add(new StringParameterValue('LTTNG_UST_COMMIT_ID', jobInfo['config']['ustCommit']))
 284   params.add(new StringParameterValue('KERNEL_TAG_ID', jobInfo['config']['linuxTagID']))
 285   def currBuild = job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(params))
 286
 287   if (currBuild != null ) {
 288     println("Launching job: ${HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)}");
 289   } else {
 290     println("Job ${jobName} not found or deactivated.");
 291   }
 292
 293   return currBuild
 294 }
 295
 296 final String toolsRepo = "https://github.com/lttng/lttng-tools.git"
 297 final String modulesRepo = "https://github.com/lttng/lttng-modules.git"
 298 final String ustRepo = "https://github.com/lttng/lttng-ust.git"
 299 final String linuxRepo = "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git"
 300
 301 final String pastJobsPath = build.getEnvironment(listener).get('WORKSPACE') + "/pastjobs";
 302
 303 def recentLttngBranchesOfInterest = ['master',
 304   'stable-2.12',
 305   'stable-2.11',
 306   'stable-2.10',
 307   'stable-2.9']
 308 def recentLinuxBranchesOfInterest = ['master',
 309   'linux-5.1.y',
 310   'linux-5.0.y',
 311   'linux-4.19.y',
 312   'linux-4.14.y',
 313   'linux-4.9.y',
 314   'linux-4.4.y']
 315
 316 def legacyLttngBranchesOfInterest = []
 317 def legacyLinuxBranchesOfInterest = []
 318
 319 def vmLinuxBranchesOfInterest = ['linux-3.18.y']
 320
 321 // Generate configurations of interest.
 322 def configurationOfInterest = [] as Set
 323
 324 recentLttngBranchesOfInterest.each { lttngBranch ->
 325   recentLinuxBranchesOfInterest.each { linuxBranch ->
 326     configurationOfInterest.add([lttngBranch, linuxBranch])
 327   }
 328 }
 329
 330 legacyLttngBranchesOfInterest.each { lttngBranch ->
 331   legacyLinuxBranchesOfInterest.each { linuxBranch ->
 332     configurationOfInterest.add([lttngBranch, linuxBranch])
 333   }
 334 }
 335
 336 def lttngBranchesOfInterest = recentLttngBranchesOfInterest + legacyLttngBranchesOfInterest
 337 def linuxBranchesOfInterest = recentLinuxBranchesOfInterest + legacyLinuxBranchesOfInterest + vmLinuxBranchesOfInterest
 338
 339 // For LTTng branches, we look for new commits.
 340 def toolsHeadCommits = GetHeadCommits(toolsRepo, lttngBranchesOfInterest)
 341 def modulesHeadCommits = GetHeadCommits(modulesRepo, lttngBranchesOfInterest)
 342 def ustHeadCommits = GetHeadCommits(ustRepo, lttngBranchesOfInterest)
 343
 344 // For Linux branches, we look for new non-RC tags.
 345 def linuxLastTagIds = GetLastTagIds(linuxRepo, linuxBranchesOfInterest)
 346
 347 def CraftConfig = { linuxBr, lttngBr ->
 348   def job = [:];
 349   job['config'] = [:];
 350   job['config']['linuxBranch'] = linuxBr;
 351   job['config']['lttngBranch'] = lttngBr;
 352   job['config']['linuxTagID'] = linuxLastTagIds[linuxBr];
 353   job['config']['toolsCommit'] = toolsHeadCommits[lttngBr];
 354   job['config']['modulesCommit'] = modulesHeadCommits[lttngBr];
 355   job['config']['ustCommit'] = ustHeadCommits[lttngBr];
 356   job['status'] = 'NOT_SET';
 357   job['build'] = null;
 358   return job;
 359 }
 360
 361 // Check what type of jobs should be triggered.
 362 triggerJobName = build.project.getFullDisplayName();
 363 if (triggerJobName.contains("vm_tests")) {
 364   jobType = 'vm_tests';
 365   recentLttngBranchesOfInterest.each { lttngBranch ->
 366     vmLinuxBranchesOfInterest.each { linuxBranch ->
 367       configurationOfInterest.add([lttngBranch, linuxBranch])
 368     }
 369   }
 370 } else if (triggerJobName.contains("baremetal_tests")) {
 371   jobType = 'baremetal_tests';
 372 } else if (triggerJobName.contains("baremetal_benchmarks")) {
 373   jobType = 'baremetal_benchmarks';
 374 }
 375
 376 // Hashmap containing all the jobs, their configuration (commit id, etc. )and
 377 // their status (SUCCEEDED, FAILED, etc.). This Hashmap is made of basic strings
 378 // rather than objects and enums because strings are easily serializable.
 379 def currentJobs = [:];
 380
 381 // Get an up to date view of all the branches of interest.
 382 configurationOfInterest.each { lttngBr, linuxBr  ->
 383   def jobName = CraftJobName(jobType, linuxBr, lttngBr);
 384   currentJobs[jobName] = CraftConfig(linuxBr, lttngBr);
 385 }
 386
 387 //Add canary job
 388 def jobNameCanary = jobType + "_canary";
 389 currentJobs[jobNameCanary] = [:];
 390 currentJobs[jobNameCanary]['config'] = [:];
 391 currentJobs[jobNameCanary]['config']['linuxBranch'] = 'v4.4.194';
 392 currentJobs[jobNameCanary]['config']['lttngBranch'] = 'v2.10.7';
 393 currentJobs[jobNameCanary]['config']['linuxTagID'] ='a227f8436f2b21146fc024d84e6875907475ace2';
 394 currentJobs[jobNameCanary]['config']['toolsCommit'] = '93fa2c9ff6b52c30173bee80445501ce8677fecc'
 395 currentJobs[jobNameCanary]['config']['modulesCommit'] = 'fe3ca7a9045221ffbedeac40ba7e09b1fc500e21'
 396 currentJobs[jobNameCanary]['config']['ustCommit'] = '0172ce8ece2102d46c7785e6bd96163225c59e49'
 397 currentJobs[jobNameCanary]['status'] = 'NOT_SET';
 398 currentJobs[jobNameCanary]['build'] = null;
 399
 400 def pastJobs = LoadPreviousJobsFromWorkspace(pastJobsPath);
 401
 402 def failedRuns = []
 403 def abortedRuns = []
 404 def isFailed = false
 405 def isAborted = false
 406 def ongoingJobs = 0;
 407
 408 currentJobs.each { jobName, jobInfo ->
 409   // If the job ran in the past, we check if the IDs changed since.
 410   // Fetch past results only if the job is not of type canary.
 411   if (!jobName.contains('_canary') && pastJobs.containsKey(jobName) &&
 412          build.getBuildVariables().get('FORCE_JOB_RUN') == 'false') {
 413     pastJob = pastJobs[jobName];
 414
 415     // If the code has not changed report previous status.
 416     if (pastJob['config'] == jobInfo['config']) {
 417       // if the config has not changed, we keep it.
 418       // if it's failed, we don't launch a new job and keep it failed.
 419       jobInfo['status'] = pastJob['status'];
 420       if (pastJob['status'] == 'FAILED' &&
 421             build.getBuildVariables().get('FORCE_FAILED_JOB_RUN') == 'false') {
 422         println("${jobName} as not changed since the last failed run. Don't run it again.");
 423         // Marked the umbrella job for failure but still run the jobs that since the
 424         // last run.
 425         isFailed = true;
 426         return;
 427       } else if (pastJob['status'] == 'ABORTED') {
 428         println("${jobName} as not changed since last aborted run. Run it again.");
 429       } else if (pastJob['status'] == 'SUCCEEDED') {
 430         println("${jobName} as not changed since the last successful run. Don't run it again.");
 431         return;
 432       }
 433     }
 434   }
 435
 436   jobInfo['status'] = 'PENDING';
 437   jobInfo['build'] = LaunchJob(jobName, jobInfo);
 438   ongoingJobs += 1;
 439 }
 440
 441 while (ongoingJobs > 0) {
 442   currentJobs.each { jobName, jobInfo ->
 443
 444     if (jobInfo['status'] != 'PENDING') {
 445       return;
 446     }
 447
 448     jobBuild = jobInfo['build']
 449
 450     // The isCancelled() method checks if the run was cancelled before
 451     // execution. We consider such run as being aborted.
 452     if (jobBuild.isCancelled()) {
 453       println("${jobName} was cancelled before launch.")
 454       isAborted = true;
 455       abortedRuns.add(jobName);
 456       ongoingJobs -= 1;
 457       jobInfo['status'] = 'ABORTED'
 458       // Invalidate the build field, as it's not serializable and we don't need
 459       // it anymore.
 460       jobInfo['build'] = null;
 461     } else if (jobBuild.isDone()) {
 462
 463       jobExitStatus = jobBuild.get();
 464
 465       // Invalidate the build field, as it's not serializable and we don't need
 466       // it anymore.
 467       jobInfo['build'] = null;
 468       println("${jobExitStatus.fullDisplayName} completed with status ${jobExitStatus.result}.");
 469
 470       // If the job didn't succeed, add its name to the right list so it can
 471       // be printed at the end of the execution.
 472       ongoingJobs -= 1;
 473       switch (jobExitStatus.result) {
 474       case Result.ABORTED:
 475         isAborted = true;
 476         abortedRuns.add(jobName);
 477         jobInfo['status'] = 'ABORTED'
 478         break;
 479       case Result.FAILURE:
 480         isFailed = true;
 481         failedRuns.add(jobName);
 482         jobInfo['status'] = 'FAILED'
 483         break;
 484       case Result.SUCCESS:
 485         jobInfo['status'] = 'SUCCEEDED'
 486         break;
 487       default:
 488         break;
 489       }
 490     }
 491   }
 492
 493   // Sleep before the next iteration.
 494   try {
 495     Thread.sleep(30000)
 496   } catch(e) {
 497     if (e in InterruptedException) {
 498       build.setResult(hudson.model.Result.ABORTED)
 499       throw new InterruptedException()
 500     } else {
 501       throw(e)
 502     }
 503   }
 504 }
 505
 506 //All jobs are done running. Save their exit status to disk.
 507 SaveCurrentJobsToWorkspace(currentJobs, pastJobsPath);
 508
 509 // Get log of failed runs.
 510 if (failedRuns.size() > 0) {
 511   println("Failed job(s):");
 512   for (failedRun in failedRuns) {
 513     println("\t" + failedRun)
 514   }
 515 }
 516
 517 // Get log of aborted runs.
 518 if (abortedRuns.size() > 0) {
 519   println("Cancelled job(s):");
 520   for (cancelledRun in abortedRuns) {
 521     println("\t" + cancelledRun)
 522   }
 523 }
 524
 525 // Mark this build as Failed if atleast one child build has failed and mark as
 526 // aborted if there was no failure but atleast one job aborted.
 527 if (isFailed) {
 528   build.setResult(hudson.model.Result.FAILURE)
 529 } else if (isAborted) {
 530   build.setResult(hudson.model.Result.ABORTED)
 531 }