From 6d4385f14106751d2dc0263b68f137b6b4196823 Mon Sep 17 00:00:00 2001 From: estebanporcelli <32097088+estebanporcelli@users.noreply.github.com> Date: Fri, 22 Jul 2022 09:49:10 -0400 Subject: [PATCH 1/4] example to execute parallel jobs with limits parallel limits are controlled by the lockable resources plugin --- .../parallelWithLimits.groovy | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 parallel-with-lockable-resources-limits/parallelWithLimits.groovy diff --git a/parallel-with-lockable-resources-limits/parallelWithLimits.groovy b/parallel-with-lockable-resources-limits/parallelWithLimits.groovy new file mode 100644 index 0000000..a788f87 --- /dev/null +++ b/parallel-with-lockable-resources-limits/parallelWithLimits.groovy @@ -0,0 +1,66 @@ +/* +This pipeline will execute all jobs in the folder that contains it. +This pipeline uses the LockableResources plugin to control how many parallel jobs run at the same time. +The maximum number of jobs that can run at the same time is specified by the optional RESOURCES parameter. +The optional LABEL parameter can be used to customize how resources are grouped. +If this job is stopped, all child jobs started by this job will be stopped by Jenkins. + */ + +import org.jenkins.plugins.lockableresources.LockableResourcesManager + +node { + // the label of all resources used by this pipeline + def label = params.getOrDefault('LABEL', currentBuild.rawBuild.project.parent.name) + println "LABEL: $label" + + // number of resources to be used by this pipeline + def resources = Integer.valueOf(params.getOrDefault('EXECUTORS', 5)) + println "RESOURCES: $resources" + + // the list of stages that will be executed in parallel + def parallelStages = [:] + + stage('Prepare') { + // setup lockable resources needed for this pipeline + setupLockableResources(label, resources) + + // list of jobs to be executed + def jobs = currentBuild.rawBuild.project.parent.items.findAll({ + project -> project.name != currentBuild.rawBuild.project.name && !project.isDisabled() + }).collect { job -> return job.name } + + println "Total jobs to run: ${jobs.size()}" + + jobs.each() { job -> + parallelStages[job] = { + stage(job) { + // Lock one of the resources assigned the specified label + lock(label: label, quantity: 1) { + build(job: job, wait: true, propagate: false, quietPeriod: 10) + } + } + } + } + } + + stage('Run') { + // run all stages in parallel + parallel parallelStages + } +} + +// Setup lockable ephemeral resources that will be deleted after the pipeline completes. +// Ephemeral resources are not configurable via the Global Configuration screen, +// this is an advantage when working with large number of resources. +@NonCPS +def setupLockableResources(label, executors) { + println "Setting up lockable resources for this build" + def lrm = LockableResourcesManager.get() + for (i = 0; i < executors; i++) { + lrm.createResourceWithLabel("${label}-$i", label) + // Get lockable resource by name + def lockableResource = lrm.fromName("${label}-$i") + // Make resource ephemeral, resources will be automatically deleted when this pipeline ends + lockableResource.setEphemeral(true) + } +} From 1a589bb5c42ed290ab85a41850d302e1b58465a1 Mon Sep 17 00:00:00 2001 From: estebanporcelli <32097088+estebanporcelli@users.noreply.github.com> Date: Fri, 22 Jul 2022 10:42:56 -0400 Subject: [PATCH 2/4] add README.md --- .../parallel-with-limits/README.md | 19 +++++++++++++++++++ .../parallelWithLimits.groovy | 0 2 files changed, 19 insertions(+) create mode 100644 jenkinsfile-examples/parallel-with-limits/README.md rename {parallel-with-lockable-resources-limits => jenkinsfile-examples/parallel-with-limits}/parallelWithLimits.groovy (100%) diff --git a/jenkinsfile-examples/parallel-with-limits/README.md b/jenkinsfile-examples/parallel-with-limits/README.md new file mode 100644 index 0000000..7ff129b --- /dev/null +++ b/jenkinsfile-examples/parallel-with-limits/README.md @@ -0,0 +1,19 @@ +# Synopsis +Demonstrate an example of a Jenkinsfile to execute a large number of jobs in parallel, while +limiting the total number of jobs that can run concurrently + +# Background + +The requirement was to execute over 200 long running free style jobs with a limited number of resources. +- Multiples of this pipeline needed to run during the day on different branches +- With 40 threads, it took 13 hours to complete the execution of all jobs +- It was necessary to prevent a single pipeline from consuming all available threads at one time +- It was necessary to allow exection of jobs from the same pipeline on other branches at the same time +- It was required that users be allowed stop the pipeline and all child jobs easily +- It was required to dynamically build the list of jobs to execute as jobs are added and removed frequently +- It was desired to keep to a minumum the number jobs added to the Jenkins job queue +- The Throttle plugin was used for many years, but the performance of the plugin became an issue as the number of nodes grew past 50 and the number of jobs exceded several 100 + + +Note: other locking/limiting mechanisms were explored, but the Lockable Resources plugin ended up being the most elegant solution. +A missing feature of the Lockable Resources plugin, to allow the creation of multiple ephemeral resources on the fly, would simplify the pipeline code even more. This does not work: lock(label: "mylabel", quantity: 5){} \ No newline at end of file diff --git a/parallel-with-lockable-resources-limits/parallelWithLimits.groovy b/jenkinsfile-examples/parallel-with-limits/parallelWithLimits.groovy similarity index 100% rename from parallel-with-lockable-resources-limits/parallelWithLimits.groovy rename to jenkinsfile-examples/parallel-with-limits/parallelWithLimits.groovy From 62f1f2b916e86a623b93bb7fb98128dd0a71f59a Mon Sep 17 00:00:00 2001 From: estebanporcelli <32097088+estebanporcelli@users.noreply.github.com> Date: Fri, 22 Jul 2022 10:46:56 -0400 Subject: [PATCH 3/4] parallel pipeline with limits --- .../parallel-with-limits/README.md | 0 .../parallel-with-limits/parallelWithLimits.groovy | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {jenkinsfile-examples => pipeline-examples}/parallel-with-limits/README.md (100%) rename {jenkinsfile-examples => pipeline-examples}/parallel-with-limits/parallelWithLimits.groovy (100%) diff --git a/jenkinsfile-examples/parallel-with-limits/README.md b/pipeline-examples/parallel-with-limits/README.md similarity index 100% rename from jenkinsfile-examples/parallel-with-limits/README.md rename to pipeline-examples/parallel-with-limits/README.md diff --git a/jenkinsfile-examples/parallel-with-limits/parallelWithLimits.groovy b/pipeline-examples/parallel-with-limits/parallelWithLimits.groovy similarity index 100% rename from jenkinsfile-examples/parallel-with-limits/parallelWithLimits.groovy rename to pipeline-examples/parallel-with-limits/parallelWithLimits.groovy From 8ce18b5042eae267af0839daed1266a7a2f38b4f Mon Sep 17 00:00:00 2001 From: estebanporcelli <32097088+estebanporcelli@users.noreply.github.com> Date: Fri, 22 Jul 2022 10:58:34 -0400 Subject: [PATCH 4/4] parallel pipeline with limits --- pipeline-examples/parallel-with-limits/README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pipeline-examples/parallel-with-limits/README.md b/pipeline-examples/parallel-with-limits/README.md index 7ff129b..110207f 100644 --- a/pipeline-examples/parallel-with-limits/README.md +++ b/pipeline-examples/parallel-with-limits/README.md @@ -4,16 +4,15 @@ limiting the total number of jobs that can run concurrently # Background -The requirement was to execute over 200 long running free style jobs with a limited number of resources. +The requirement was to execute over 200 long running free style jobs with a limited number of resources. With 40 threads, it took 13 hours to complete the execution of all jobs - Multiples of this pipeline needed to run during the day on different branches -- With 40 threads, it took 13 hours to complete the execution of all jobs - It was necessary to prevent a single pipeline from consuming all available threads at one time -- It was necessary to allow exection of jobs from the same pipeline on other branches at the same time - It was required that users be allowed stop the pipeline and all child jobs easily -- It was required to dynamically build the list of jobs to execute as jobs are added and removed frequently -- It was desired to keep to a minumum the number jobs added to the Jenkins job queue -- The Throttle plugin was used for many years, but the performance of the plugin became an issue as the number of nodes grew past 50 and the number of jobs exceded several 100 +- It was required to dynamically build the list of jobs to execute. Jobs are added and removed frequently +- It was desired to keep to a minumum the number jobs added to the Jenkins job queue at one time +- All jobs run in a shared pool of VMs used by different teams. It is not possible to manually assign and maintain labels for each team. VMs must be able to be added and removed from the shared pool as needed. +- The Throttle plugin was used for many years, but the performance of the plugin became an issue as the number of nodes grew past 50 and the number of jobs exceded several hundreds Note: other locking/limiting mechanisms were explored, but the Lockable Resources plugin ended up being the most elegant solution. -A missing feature of the Lockable Resources plugin, to allow the creation of multiple ephemeral resources on the fly, would simplify the pipeline code even more. This does not work: lock(label: "mylabel", quantity: 5){} \ No newline at end of file +If the Lockable Resources plugin allowed the creation of multiple ephemeral resources on the fly, the sample pipeline would be even simpler. At the moment this does not work: lock(label: "mylabel", quantity: 5){} \ No newline at end of file