This is an automated email from the ASF dual-hosted git repository.
mcimbalek pushed a commit to branch main
in repository
https://gitbox.apache.org/repos/asf/incubator-kie-kogito-pipelines.git
The following commit(s) were added to refs/heads/main by this push:
new 3ca94f06 [kie-issues#777] Allow restricting jenkins agent labels...
(#1165)
3ca94f06 is described below
commit 3ca94f06bb8d5526d7385c90ffecd778d4b3f7b6
Author: Martin Cimbalek <[email protected]>
AuthorDate: Wed Jan 31 15:04:37 2024 +0100
[kie-issues#777] Allow restricting jenkins agent labels... (#1165)
This change should allow restricting on which nodes pipeline can run. If
environment variable 'FAULTY_NODES' is set or inherited from parent folders,
nodes listed here will be added to the label expression to avoid them. This way
we can have a single source (KIE folder in jenkins) when we can exclude faulty
nodes before Apache admins remove the node, which is sometimes lengthy
procedure.
---
.ci/jenkins/Jenkinsfile.build-kogito-ci-image | 8 ++++--
.ci/jenkins/Jenkinsfile.build-operator-node | 2 +-
.ci/jenkins/Jenkinsfile.nightly | 2 +-
.ci/jenkins/Jenkinsfile.nightly.cloud | 2 +-
.ci/jenkins/Jenkinsfile.release | 2 +-
.ci/jenkins/Jenkinsfile.release.cloud | 2 +-
.ci/jenkins/Jenkinsfile.setup-branch | 2 +-
.ci/jenkins/Jenkinsfile.setup-branch.cloud | 2 +-
.../Jenkinsfile.tools.update-jenkins-dependencies | 2 +-
.../org/kie/jenkins/JenkinsfileNightly.groovy | 2 ++
README.md | 32 +++++++++++++++++++++-
dsl/seed/jenkinsfiles/Jenkinsfile.buildchain | 1 +
.../Jenkinsfile.environment.integration-branch | 2 +-
dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare | 2 +-
dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches | 2 +-
dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch | 2 +-
dsl/seed/jenkinsfiles/Jenkinsfile.seed.main | 2 +-
dsl/seed/jenkinsfiles/Jenkinsfile.seed.trigger | 2 +-
.../jenkinsfiles/Jenkinsfile.tools.toggle-triggers | 2 +-
.../Jenkinsfile.update-quarkus-version | 2 +-
jenkins-pipeline-shared-libraries/vars/util.groovy | 16 +++++++++++
21 files changed, 71 insertions(+), 20 deletions(-)
diff --git a/.ci/jenkins/Jenkinsfile.build-kogito-ci-image
b/.ci/jenkins/Jenkinsfile.build-kogito-ci-image
index a3c536d3..f8cfca5b 100644
--- a/.ci/jenkins/Jenkinsfile.build-kogito-ci-image
+++ b/.ci/jenkins/Jenkinsfile.build-kogito-ci-image
@@ -1,12 +1,14 @@
import org.jenkinsci.plugins.workflow.libs.Library
pipeline {
- agent {
- label 'ubuntu'
- }
libraries {
lib("jenkins-pipeline-shared-libraries@${BRANCH_NAME}")
}
+
+ agent {
+ label util.getLabel('ubuntu')
+ }
+
environment {
DOCKER_CONFIG = "${WORKSPACE}/.docker"
diff --git a/.ci/jenkins/Jenkinsfile.build-operator-node
b/.ci/jenkins/Jenkinsfile.build-operator-node
index fba8ec3e..fb7c9a66 100644
--- a/.ci/jenkins/Jenkinsfile.build-operator-node
+++ b/.ci/jenkins/Jenkinsfile.build-operator-node
@@ -4,7 +4,7 @@ import org.jenkinsci.plugins.workflow.libs.Library
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
options {
diff --git a/.ci/jenkins/Jenkinsfile.nightly b/.ci/jenkins/Jenkinsfile.nightly
index e8940188..a0c606c8 100644
--- a/.ci/jenkins/Jenkinsfile.nightly
+++ b/.ci/jenkins/Jenkinsfile.nightly
@@ -19,7 +19,7 @@ UNSTABLE_STAGES = [:]
// Should be multibranch pipeline
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
options {
diff --git a/.ci/jenkins/Jenkinsfile.nightly.cloud
b/.ci/jenkins/Jenkinsfile.nightly.cloud
index c8d9037c..9defc643 100644
--- a/.ci/jenkins/Jenkinsfile.nightly.cloud
+++ b/.ci/jenkins/Jenkinsfile.nightly.cloud
@@ -25,7 +25,7 @@ promoteImageParamsPrefix = 'PROMOTE_IMAGE'
// Should be multibranch pipeline
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
options {
diff --git a/.ci/jenkins/Jenkinsfile.release b/.ci/jenkins/Jenkinsfile.release
index 64f9f551..29e56ee3 100644
--- a/.ci/jenkins/Jenkinsfile.release
+++ b/.ci/jenkins/Jenkinsfile.release
@@ -19,7 +19,7 @@ releaseProperties = [:]
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
// parameters {
diff --git a/.ci/jenkins/Jenkinsfile.release.cloud
b/.ci/jenkins/Jenkinsfile.release.cloud
index 26883e2b..c0093673 100644
--- a/.ci/jenkins/Jenkinsfile.release.cloud
+++ b/.ci/jenkins/Jenkinsfile.release.cloud
@@ -26,7 +26,7 @@ promoteImageParamsPrefix = 'PROMOTE_IMAGE'
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
// parameters {
diff --git a/.ci/jenkins/Jenkinsfile.setup-branch
b/.ci/jenkins/Jenkinsfile.setup-branch
index 9fa593ff..a74917e7 100644
--- a/.ci/jenkins/Jenkinsfile.setup-branch
+++ b/.ci/jenkins/Jenkinsfile.setup-branch
@@ -12,7 +12,7 @@ UNSTABLE_STAGES = [:]
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
options {
diff --git a/.ci/jenkins/Jenkinsfile.setup-branch.cloud
b/.ci/jenkins/Jenkinsfile.setup-branch.cloud
index bdbb330e..6222b3a6 100644
--- a/.ci/jenkins/Jenkinsfile.setup-branch.cloud
+++ b/.ci/jenkins/Jenkinsfile.setup-branch.cloud
@@ -16,7 +16,7 @@ examplesImagesParamsPrefix = 'EXAMPLES_IMAGES'
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
options {
diff --git a/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies
b/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies
index 45d98d7d..6192a9ad 100644
--- a/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies
+++ b/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies
@@ -10,7 +10,7 @@ branchCreated = false
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
options {
diff --git
a/.ci/jenkins/tests/src/test/groovy/org/kie/jenkins/JenkinsfileNightly.groovy
b/.ci/jenkins/tests/src/test/groovy/org/kie/jenkins/JenkinsfileNightly.groovy
index c83ca0da..52ba0e10 100644
---
a/.ci/jenkins/tests/src/test/groovy/org/kie/jenkins/JenkinsfileNightly.groovy
+++
b/.ci/jenkins/tests/src/test/groovy/org/kie/jenkins/JenkinsfileNightly.groovy
@@ -48,6 +48,7 @@ class TestJenkinsfileNightly extends
SingleFileDeclarativePipelineTest {
addEnvVar('STAGE_NAME', 'STAGE_NAME')
addEnvVar('BUILD_NUMBER', 'BUILD_NUMBER')
addEnvVar('BUILD_URL', 'BUILD_URL')
+ addEnvVar('FAULTY_NODES', 'faultyNode99')
helper.addShMock('date -u "+%Y-%m-%d"', 'date', 0)
@@ -58,6 +59,7 @@ class TestJenkinsfileNightly extends
SingleFileDeclarativePipelineTest {
mockSharedLibVarsCall('githubscm', 'resolveRepository', { repo,
author, branch, ignoreErrors, credsId ->
registerTestCallstack('githubscm.resolveRepository', "${repo}, ${author},
${branch}, ${ignoreErrors}, ${credsId}") })
mockSharedLibVarsCall('githubscm', 'createBranch', { branch ->
registerTestCallstack('githubscm.createBranch', "${branch}") })
mockSharedLibVarsCall('githubscm', 'pushObject', { remote, branch,
credsId -> registerTestCallstack('githubscm.pushObject', "${remote}, ${branch},
${credsId}") })
+ mockSharedLibVarsCall('util', 'getLabel', { label -> return label})
}
@Test
diff --git a/README.md b/README.md
index d11695de..1c637e06 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,11 @@ This repository contains some of the pipelines of Kogito
project.
- [GitHub Action checks](#github-action-checks)
- [Sonar cloud](#sonar-cloud)
- [Configuration of pipelines](#configuration-of-pipelines)
+ - [Source Code](#source-code)
+ - [Pipelines](#pipelines)
+ - [Shared Pipeline libraries](#shared-pipeline-libraries)
- [Jenkins](#jenkins)
+ - [Jenkins KIE folder](#jenkins-kie-folder)
- [Jenkins jobs generation](#jenkins-jobs-generation)
- [Zulip notifications](#zulip-notifications)
- [Format](#format)
@@ -132,9 +136,35 @@ NOTE: test coverage analysis is executed only by **Jenkins
PR simple build&test*
# Configuration of pipelines
+## Source Code
+*Note: Creating separate readme.md documenting how-tos and best practices for
implementing pipelines might be useful*
+
+### Pipelines
+In this repository two types of pipelines can be found:
+- **Kogito pipelines** (obviously) - located in the
[.ci/jenkins](./.ci/jenkins) folder
+- **Seed jobs library** - see [Jenkins documentation](../docs/jenkins.md)
+
+### Shared pipeline libraries
+Apart from these pipelines, the `jenkins-pipeline-shared-libraries` are also
stored in this repository. Functions and classes contained in these libraries
can be freely used in all pipelines located under [KIE Jenkins
folder](https://ci-builds.apache.org/job/KIE). Just include correct import and
annotation in your Jenkinsfile:
+```
+import org.jenkinsci.plugins.workflow.libs.Library
+
+@Library('jenkins-pipeline-shared-libraries')_
+```
+For more details please see our [Jenkins pipelines shared libraries
documentation](./jenkins-pipeline-shared-libraries/README.md) and [Jenkins.io
documentation](https://www.jenkins.io/doc/book/pipeline/shared-libraries/)
+
## Jenkins
-All pipelines can be found in [kogito Jenkins
folder](https://eng-jenkins-csb-business-automation.apps.ocp-c1.prod.psi.redhat.com/job/KIE/job/kogito).
+### Jenkins KIE folder
+
+All KIE jobs (pipelines) can be found in [KIE Jenkins
folder](https://ci-builds.apache.org/job/KIE)<br />
+For this folder and all its descendants there is several useful things set at
this folder level:<br />
+- **Pipeline library** - accessible in pipelines under name
`jenkins-pipeline-shared-libraries` it gives access to some useful functions
used throughout various KIE pipelines. More details can be found in our
[Jenkins pipeline shared libraries
documentation](./jenkins-pipeline-shared-libraries/README.md) and in the
[previous chapter](#source-code)
+- **Environment Variables** - Environment variables set here are inherited by
all the folders and jobs located in the [KIE
folder](https://ci-builds.apache.org/job/KIE) tree in Jenkins. However, they
can be overridden or extended. You can modify the variables by clicking
`Configure` in the left menu (if you have necessary permissions). Currently
present Environment Variables are:
+ - **FAULTY_NODES** - Comma separated list of Jenkins execution nodes that
are faulty in some way and cause KIE jobs to fail. This variable is expected by
the *pipeline-library* function `getLabel(String label)`, which extends desired
`label` by expression that ensures avoiding these faulty nodes. This way we can
increase durability of KIE automation by the time the Apache CI team fixes the
issue with faulty node.
+
+
+All pipelines from this repository can be found in [kogito Jenkins
folder](https://ci-builds.apache.org/job/KIE/job/kogito/).
### Jenkins jobs generation
diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain
b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain
index 912a3e68..0dfc3b23 100644
--- a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain
+++ b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain
@@ -49,6 +49,7 @@ pipeline {
agent {
docker {
image env.AGENT_DOCKER_BUILDER_IMAGE
+ label util.getLabel(agentLabel)
args env.AGENT_DOCKER_BUILDER_ARGS
}
}
diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch
b/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch
index d37af021..83bd5e52 100644
--- a/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch
+++ b/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch
@@ -4,7 +4,7 @@ import org.jenkinsci.plugins.workflow.libs.Library
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
environment {
diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare
b/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare
index 49f3e87c..92e80ff6 100644
--- a/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare
+++ b/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare
@@ -9,7 +9,7 @@ repositoriesToBranch = []
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
environment {
diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches
b/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches
index 6ef444b2..417996ad 100644
--- a/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches
+++ b/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches
@@ -6,7 +6,7 @@ import org.kie.jenkins.MavenCommand
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
options{
diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch
b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch
index 4a06492b..35ff15c2 100644
--- a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch
+++ b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch
@@ -28,7 +28,7 @@ SeedJobUtils.createSeedJobTrigger(
pipeline {
agent {
- label "${env.AGENT_LABEL}"
+ label util.getLabel(env.AGENT_LABEL)
}
tools {
diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main
b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main
index 67a470b4..8f84b7ae 100644
--- a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main
+++ b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main
@@ -11,7 +11,7 @@ seedConfig = [:]
pipeline {
agent {
- label "${env.AGENT_LABEL}"
+ label util.getLabel(env.AGENT_LABEL)
}
options {
diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.trigger
b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.trigger
index ffd3229b..68d45551 100644
--- a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.trigger
+++ b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.trigger
@@ -4,7 +4,7 @@ import org.jenkinsci.plugins.workflow.libs.Library
pipeline {
agent {
- label "${env.AGENT_LABEL}"
+ label util.getLabel(env.AGENT_LABEL)
}
options {
diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers
b/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers
index ea3456fe..3de6a0b8 100644
--- a/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers
+++ b/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers
@@ -5,7 +5,7 @@ import org.kie.jenkins.MavenCommand
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
options {
diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-version
b/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-version
index 3074c3bf..73479d08 100644
--- a/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-version
+++ b/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-version
@@ -7,7 +7,7 @@ failingRepos = []
pipeline {
agent {
- label 'ubuntu'
+ label util.getLabel('ubuntu')
}
environment {
diff --git a/jenkins-pipeline-shared-libraries/vars/util.groovy
b/jenkins-pipeline-shared-libraries/vars/util.groovy
index 091fe3fd..cd3d45ce 100644
--- a/jenkins-pipeline-shared-libraries/vars/util.groovy
+++ b/jenkins-pipeline-shared-libraries/vars/util.groovy
@@ -555,3 +555,19 @@ void waitForDocker() {
sleep(10) // give it some ahead time not to invoke docker exec immediately
after container start
sh 'wait-for-docker.sh' // script in kogito-ci-build image itself put in
/usr/local/bin
}
+
+/**
+ * Method to wrap original label and exclude nodes that are marked as faulty
in some of the parent folders.
+ * This environment variable is inherited down the folder tree and available
in jobs.
+ * @param label Node label to be used
+ * @return Node label extended with an expression ensuring to exclude nodes
marked as faulty.
+ */
+String getLabel(String label) {
+ String faultyNodesString = env.FAULTY_NODES
+ if((faultyNodesString == null) || faultyNodesString.isEmpty()) {
+ return label
+ }
+ String[] faultyNodes = faultyNodesString.split(',')
+ String result = "(${label}) && !(${String.join(' || ', faultyNodes)})"
+ return result.toString()
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
