dsl/kernel-lttng-modules.seed.groovy

   1 enum KernelVersioning {
   2     MAJOR,MINOR,REVISION,BUILD
   3 }
   4
   5 class BasicVersion implements Comparable<BasicVersion> {
   6     int major = -1
   7     int minor = -1
   8     int revision = -1
   9     int build = -1
  10     int rc = -1
  11     String gitRefs
  12
  13     // Default Constructor
  14     BasicVersion() {}
  15
  16     // Parse a version string of format X.Y.Z.W-A
  17     BasicVersion(String version, String ref) {
  18         gitRefs = ref
  19         def tokenVersion
  20         def token
  21         if (version.contains('-')) {
  22             // Release canditate
  23             token = version.tokenize('-')
  24             tokenVersion = token[0]
  25             if (token[1]?.isInteger()) {
  26                 rc = token[1].toInteger()
  27             }
  28         } else {
  29             tokenVersion = version
  30         }
  31
  32         tokenVersion = tokenVersion.tokenize('.')
  33
  34         def tagEnum = KernelVersioning.MAJOR
  35         tokenVersion.each {
  36             if (it?.isInteger()) {
  37                 switch (tagEnum) {
  38                     case KernelVersioning.MAJOR:
  39                         major = it.toInteger()
  40                         tagEnum = KernelVersioning.MINOR
  41                         break
  42                     case KernelVersioning.MINOR:
  43                         minor = it.toInteger()
  44                         tagEnum = KernelVersioning.REVISION
  45                         break
  46                     case KernelVersioning.REVISION:
  47                         revision = it.toInteger()
  48                         tagEnum = KernelVersioning.BUILD
  49                         break
  50                     case KernelVersioning.BUILD:
  51                         build = it.toInteger()
  52                         tagEnum = -1
  53                         break
  54                     default:
  55                         println("Unsupported version extension")
  56                         println("Trying to parse: ${version}")
  57                         println("Invalid sub version value: ${it}")
  58                 //TODO: throw exception for jenkins
  59                 }
  60             }
  61         }
  62     }
  63
  64     String print() {
  65         String ret = ""
  66         if (major != -1) {
  67             ret += major
  68             if (minor != -1) {
  69                 ret += "." + minor
  70                 if (revision != -1) {
  71                     ret += "." + revision
  72                     if (build != -1) {
  73                         ret += "." + build
  74                     }
  75                 }
  76             }
  77             if (rc != -1) {
  78                 ret += "-rc" + rc
  79             }
  80         }
  81         return ret
  82     }
  83
  84     @Override
  85     int compareTo(BasicVersion kernelVersion) {
  86         return major <=> kernelVersion.major ?: minor <=> kernelVersion.minor ?: revision <=> kernelVersion.revision ?: build <=> kernelVersion.build ?: rc <=> kernelVersion.rc
  87     }
  88 }
  89
  90 def kernelTagCutOff = new BasicVersion("2.6.36", "")
  91 def modulesBranches = ["master", "stable-2.5", "stable-2.6"]
  92
  93 //def modulesBranches = ["master","stable-2.5","stable-2.6", "stable-2.4"]
  94
  95 def linuxURL = "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git"
  96 def modulesURL = "https://github.com/lttng/lttng-modules.git"
  97
  98 // Linux specific variable
  99 String linuxCheckoutTo = "linux-source"
 100 String recipeCheckoutTo = "recipe"
 101 String modulesCheckoutTo = "lttng-modules"
 102
 103 def linuxGitReference = "/home/jenkins/gitcache/linux-stable.git"
 104
 105 // Check if we are on jenkins
 106 // Useful for outside jenkins devellopment related to groovy only scripting
 107 def isJenkinsInstance = binding.variables.containsKey('JENKINS_HOME')
 108
 109 // Fetch tags and format
 110 // Split the string into sections based on |
 111 // And pipe the results together
 112 String process = "git ls-remote -t $linuxURL | cut -c42- | sort -V"
 113 def out = new StringBuilder()
 114 def err = new StringBuilder()
 115 Process result = process.tokenize( '|' ).inject( null ) { p, c ->
 116     if( p )
 117         p | c.execute()
 118     else
 119         c.execute()
 120 }
 121
 122 result.waitForProcessOutput(out,err)
 123
 124 if ( result.exitValue() == 0 ) {
 125     def branches = out.readLines().collect {
 126         // Scrap special string tag
 127         it.replaceAll("\\^\\{\\}", '')
 128     }
 129
 130     branches = branches.unique()
 131
 132     List versions = []
 133     branches.each { branch ->
 134         def stripBranch = branch.replaceAll("rc", '').replaceAll(/refs\/tags\/v/,'')
 135         BasicVersion kVersion = new BasicVersion(stripBranch, branch)
 136         versions.add(kVersion)
 137     }
 138
 139     // Sort the version via Comparable implementation of KernelVersion
 140     versions = versions.sort()
 141
 142     // Find the version cutoff
 143     def cutoffPos = versions.findIndexOf{(it.major >= kernelTagCutOff.major) && (it.minor >= kernelTagCutOff.minor) && (it.revision >= kernelTagCutOff.revision) && (it.build >= kernelTagCutOff.build) && (it.rc >= kernelTagCutOff.rc)}
 144
 145         // If error set cutoff on last so no job are created
 146         if (cutoffPos == -1) {
 147                 cutoffPos = versions.size()
 148         }
 149     // Get last version and include only last rc
 150     def last
 151     def lastNoRcPos
 152     last = versions.last()
 153     if (last.rc != -1) {
 154         int i = versions.size()-1
 155         while (i > -1 && versions[i].rc != -1 ) {
 156             i--
 157         }
 158         lastNoRcPos = i + 1
 159     } else {
 160         lastNoRcPos = versions.size()
 161     }
 162
 163     String modulesPrefix = "lttng-modules"
 164     String kernelPrefix = "dsl-kernel"
 165     String separator = "-"
 166
 167
 168         println("CutOff index")
 169         println(cutoffPos)
 170
 171
 172     // Actual job creation
 173     for (int i = cutoffPos; i < versions.size() ; i++) {
 174
 175         // Only create for valid build
 176         if ( (i < lastNoRcPos && versions[i].rc == -1) || (i >= lastNoRcPos)) {
 177             println ("Preparing job for")
 178
 179             String jobName = kernelPrefix + separator + versions[i].print()
 180
 181             // Generate modules job based on supported modules jobs
 182             def modulesJob = [:]
 183             modulesBranches.each { branch ->
 184                 modulesJob[branch] = modulesPrefix + separator + branch + separator + jobName
 185             }
 186
 187             // Jenkins only dsl
 188             println(jobName)
 189             if (isJenkinsInstance) {
 190                 matrixJob("${jobName}") {
 191                     using("linux-master")
 192                     scm {
 193                         git {
 194                             remote {
 195                                 url("${linuxURL}")
 196                             }
 197                             branch(versions[i].gitRefs)
 198                             shallowClone(true)
 199                             relativeTargetDir(linuxCheckoutTo)
 200                             reference(linuxGitReference)
 201                         }
 202                     }
 203                     publishers {
 204                         modulesJob.each {
 205                             downstream(it.value, 'SUCCESS')
 206                         }
 207                     }
 208                 }
 209             }
 210             // Corresponding Module job
 211             modulesJob.each { job ->
 212                 println("\t" + job.key + " " + job.value)
 213                 if (isJenkinsInstance) {
 214                     matrixJob(job.value) {
 215                         using("modules")
 216                         multiscm {
 217                             git {
 218                                 remote {
 219                                     name(kernelPrefix)
 220                                     url("${linuxURL}")
 221                                 }
 222                                 branch(versions[i].gitRefs)
 223                                 shallowClone(true)
 224                                 relativeTargetDir(linuxCheckoutTo)
 225                                 reference(linuxGitReference)
 226                             }
 227                             git {
 228                                 remote {
 229                                     name(modulesPrefix)
 230                                     url(modulesURL)
 231                                 }
 232                                 branch(job.key)
 233                                 relativeTargetDir(modulesCheckoutTo)
 234                             }
 235                         }
 236                         steps {
 237                             copyArtifacts("${jobName}/arch=\$arch,label=kernel", "linux-artifact/**", '', false, false) {
 238                                 latestSuccessful(true) // Latest successful build
 239                             }
 240                             shell(readFileFromWorkspace('lttng-modules/lttng-modules-dsl-master.sh'))
 241                         }
 242                     }
 243                 }
 244             }
 245         }
 246     }
 247
 248     // Trigger generations
 249     def dslTriggerKernel = """\
 250 import hudson.model.*
 251 import jenkins.model.*
 252 import hudson.AbortException
 253 import hudson.console.HyperlinkNote
 254 import java.util.concurrent.CancellationException
 255 import java.util.Random
 256
 257
 258 Random random = new Random()
 259 def jobs = hudson.model.Hudson.instance.items
 260 def fail = false
 261 def jobStartWithKernel = "KERNELPREFIX"
 262 def jobStartWithModule = "MODULEPREFIX"
 263 def toBuild = []
 264 def counter = 0
 265 def limitQueue = 4
 266
 267 def anotherBuild
 268 jobs.each { job ->
 269   def jobName = job.getName()
 270   if (jobName.startsWith(jobStartWithKernel)) {
 271     counter = counter + 1
 272     def lastBuild = job.getLastBuild()
 273     if (lastBuild == null || lastBuild.result != Result.SUCCESS) {
 274       toBuild.push(job)
 275     } else {
 276       println("\tAlready built")
 277     }
 278   }
 279 }
 280
 281 println "Kernel total "+ counter
 282 println "Kernel to build "+ toBuild.size()
 283
 284
 285 def kernelEnabledNode = 0
 286 hudson.model.Hudson.instance.nodes.each { node ->
 287   if (node.getLabelString().contains("kernel")){
 288     kernelEnabledNode++
 289   }
 290 }
 291 println "Nb of live kernel enabled build node "+ kernelEnabledNode
 292
 293 def ongoingBuild = []
 294 def q = jenkins.model.Jenkins.getInstance().getQueue()
 295
 296 def queuedTaskKernel = 0
 297 def queuedTaskModule = 0
 298
 299 while (toBuild.size() != 0) {
 300   // Throttle the build with both the number of current parent task and queued
 301   // task.Look for both kernel and downstream module from previous kernel.
 302   queuedTaskKernel = q.getItems().findAll {
 303     it.task.getParent().name.startsWith(jobStartWithKernel)
 304   }.size()
 305
 306   queuedTaskModule = q.getItems().findAll {
 307           it.task.getParent().name.startsWith(jobStartWithModule)
 308   }.size()
 309
 310   it.task.getParent().name.startsWith(jobStartWithModule)
 311   if ((ongoingBuild.size() <= kernelEnabledNode.intdiv(2)) && (queuedTaskKernel + queuedTaskModule < limitQueue)) {
 312                 def job = toBuild.pop()
 313                 ongoingBuild.push(job.scheduleBuild2(0))
 314                 println "\t trigering " + HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)
 315   } else {
 316     println "Currently " + ongoingBuild.size() + " build currently on execution. Limit: " + kernelEnabledNode.intdiv(2)
 317     println "Currently " + queuedTask.findAll{it.task.getParent().name.startsWith(jobStartWithModule)}.size() + " module jobs are queued. Limit: " + limitQueue
 318     println "Currently " + queuedTask.findAll{it.task.getParent().name.startsWith(jobStartWithKernel)}.size() + " kernel jobs are queued. Limit: " + limitQueue
 319     Thread.sleep(random.nextInt(60000))
 320     ongoingBuild.removeAll{ it.isCancelled() || it.isDone() }
 321   }
 322 }
 323
 324 if (fail){
 325   throw new AbortException("Some job failed")
 326 }
 327 """
 328         def dslTriggerModule = """\
 329 import hudson.model.*
 330 import hudson.AbortException
 331 import hudson.console.HyperlinkNote
 332 import java.util.concurrent.CancellationException
 333 import java.util.Random
 334
 335
 336 Random random = new Random()
 337 def jobs = hudson.model.Hudson.instance.items
 338 def fail = false
 339 def modulePrefix = "MODULEPREFIX"
 340 def branchName = "BRANCHNAME"
 341 def kernelPrefix = "KERNELPREFIX"
 342 def nodeLabels=["kernel"]
 343 def validNodeDivider = 2
 344
 345 def fullModulePrefix = modulesPrefix + branchName
 346
 347 def toBuild = []
 348 def counter = 0
 349 def limitQueue = 4
 350
 351 jobs.each { job ->
 352         def jobName = job.getName()
 353         if (jobName.startsWith(fullModulePrefix)) {
 354                 counter = counter + 1
 355                 toBuild.push(job)
 356         }
 357 }
 358
 359 // Get valid labeled node node
 360 def validNodeCount = 0
 361 hudson.model.Hudson.instance.nodes.each { node ->
 362         def valid = true
 363         nodeLabels.each { label ->
 364                 if (!node.getLabelString().contains(nodeLabel)){
 365                         valid = false
 366                         break;
 367                 }
 368         }
 369         if (valid){
 370                 validNodeCount++
 371         }
 372 }
 373
 374 // Divide the valid node by validNodeDivider based on user defined label slave descriminant ex arck type
 375 def finalValidNodeCount = validNodeCount.intdiv(validNodeDivider
 376
 377 // Scheduling
 378
 379 def ongoingBuild = []
 380 def q = jenkins.model.Jenkins.getInstance().getQueue()
 381 def queuedTaskKernel = 0
 382 def queuedTaskModule = 0
 383 def sleep = 0
 384
 385 while (toBuild.size() != 0) {
 386         // Throttle the build with both the number of current parent task and queued
 387         // task.Look for both kernel and downstream module from previous kernel.
 388         queuedTaskKernel = q.getItems().findAll {it.task.getParent().getDisplayName().startsWith(jobStartWithKernel)}.size()
 389         queuedTaskModule = q.getItems().findAll {it.task.getParent().getDisplayName().startsWith(jobStartWithModule)}.size()
 390         if ((ongoingBuild.size() <= finalValidNodeCount) && (queuedTaskKernel + queuedTaskModule < limitQueue)) {
 391                 def job = toBuild.pop()
 392                 ongoingBuild.push(job.scheduleBuild2(0))
 393                 println "\t trigering " + HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)
 394         } else {
 395         println "Holding trigger"
 396         println "Currently " + ongoingBuild.size() + "  build ongoing. Max = " + validNodeCount
 397         println "Currently " + queuedTaskKernel + " Kernel build ongoing."
 398         println "Currently " + queuedTaskModule + " LTTng-modules build ongoing."
 399         println "Limit for combination of both:" + limitQueue
 400
 401         sleep = random.nextInt(60000)
 402         println "Sleeping for " + sleep.intdiv(1000) + " seconds"
 403                 Thread.sleep(sleep)
 404                 ongoingBuild.removeAll{ it.isCancelled() || it.isDone() }
 405         }
 406 }
 407 if (fail){
 408         throw new AbortException("Some job failed")
 409 }
 410 """
 411
 412         dslTriggerKernel = dslTriggerKernel.replaceAll("KERNELPREFIX", kernelPrefix)
 413         dslTriggerKernel = dslTriggerKernel.replaceAll("MODULEPREFIX", modulesPrefix)
 414     if (isJenkinsInstance) {
 415         freeStyleJob("dsl-trigger-kernel") {
 416             steps {
 417                 systemGroovyCommand(dslTriggerKernel)
 418             }
 419                         triggers {
 420                                 cron("H 0 * * *")
 421                         }
 422                 }
 423
 424                 modulesBranches.each { branch ->
 425                         dslTriggerModule = dslTriggerModule.replaceAll("MODULEPREFIX",modulesPrefix + separator + branch + separator)
 426                         dslTriggerModule = dslTriggerModule.replaceAll("BRANCHNAME",separator + branch + separator)
 427                         freeStyleJob("dsl-trigger-module-${branch}") {
 428                                 steps {
 429                                         systemGroovyCommand(dslTriggerModule)
 430                                 }
 431                                 triggers {
 432                                         scm('@daily')
 433                                 }
 434                         }
 435                 }
 436     }
 437 }