DAOS-19247 test: Support scripted Test stages

vars/getFunctionalTestStage.groovy

@@ -1,4 +1,4 @@
-/* groovylint-disable VariableName */
+/* groovylint-disable NestedBlockDepth, VariableName */
 // vars/getFunctionalTestStage.groovy
 
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
@@ -33,8 +33,9 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
  *      job_status      Map of status for each stage in the job/build
  * @return a scripted stage to run in a pipeline
  */
+/* groovylint-disable-next-line MethodSize */
 Map call(Map kwargs = [:]) {
-    String name = kwargs.get('name', 'Unknown Functional Test Stage')
+    String name = kwargs.get('name', '')
     String pragma_suffix = kwargs.get('pragma_suffix')
     String label = kwargs.get('label')
     String next_version = kwargs.get('next_version', null)
@@ -57,24 +58,32 @@ Map call(Map kwargs = [:]) {
     // to be skipped by the existing skipFunctionalTestStage logic, while new stages can use
     // runStage directly. Once all stages have been converted to use runStage, this should be
     // changed to a Boolean.
-    String runStage = kwargs.get('runStage', 'undefined').toString()
+    String runStage = kwargs.get('runStage', 'undefined')
 
     Map job_status = kwargs.get('job_status', [:])
 
+    if (!name) {
+        error("getFunctionalTestStage() requires a stage 'name' argument")
+    }
+
     return {
         stage("${name}") {
+            /* groovylint-disable-next-line DuplicateStringLiteral */
+            if (runStage == 'false') {
+                println("[${name}] Stage skipped by runStage=false")
+                Utils.markStageSkippedForConditional("${name}")
+                return
+            }
+
             // Get the tags for the stage. Use either the build parameter, commit pragma, or
             // default tags. All tags are combined with the stage tags to ensure only tests that
             // 'fit' the cluster will be run.
             String tags = getFunctionalTags(
                 pragma_suffix: pragma_suffix, stage_tags: stage_tags, default_tags: default_tags)
 
-            if (runStage == 'false') {
-                println("[${name}] Stage skipped by runStage=false")
-                Utils.markStageSkippedForConditional("${name}")
-                return
-            } else if (runStage == 'undefined') {
-                // To be removed once all stages have been converted to use runStage.
+            // To be removed once all stages have been converted to use runStage.
+            /* groovylint-disable-next-line DuplicateStringLiteral */
+            if (runStage == 'undefined') {
                 Map skip_kwargs = [
                 'tags': tags,
                 'pragma_suffix': pragma_suffix,
@@ -86,7 +95,10 @@ Map call(Map kwargs = [:]) {
                     Utils.markStageSkippedForConditional("${name}")
                     return
                 }
-            } else if (!testsInStage(tags)) {
+            }
+
+            // Verify tags are valid for the stage and that there are tests to run.
+            if (runStage == 'true' && !testsInStage(tags)) {
                 println("[${name}] Stage skipped by no tests matching the '${tags}' tags")
                 Utils.markStageSkippedForConditional("${name}")
                 return
@@ -105,6 +117,7 @@ Map call(Map kwargs = [:]) {
                     checkoutScm(pruneStaleBranch: true)
                 }
 
+                Throwable tryError = null
                 try {
                     println("[${name}] Running functionalTest() on ${label} with tags=${tags}")
                     Map ftestConfig = [
@@ -123,19 +136,37 @@ Map call(Map kwargs = [:]) {
                             provider: provider)['ftest_arg'],
                         test_function: 'runTestFunctionalV2']
                     if (node_count != null) {
+                        /* groovylint-disable-next-line DuplicateStringLiteral */
                         ftestConfig['node_count'] = node_count
                     }
                     jobStatusUpdate(
                         job_status,
                         name,
                         functionalTest(ftestConfig))
+                /* groovylint-disable-next-line CatchException */
+                } catch (Exception e) {
+                    tryError = e
+                    println("[${name}] Caught exception in try: ${tryError}")
+                    jobStatusUpdate(job_status, name, 'FAILURE')
+                    throw tryError
                 } finally {
-                    println("[${name}] Running functionalTestPostV2()")
-                    functionalTestPostV2()
-                    jobStatusUpdate(job_status, name)
+                    try {
+                        println("[${name}] Running functionalTestPostV2()")
+                        functionalTestPostV2()
+                        jobStatusUpdate(job_status, name)
+                    /* groovylint-disable-next-line CatchException */
+                    } catch (Exception finallyError) {
+                        println("[${name}] Caught exception in finally: ${finallyError}")
+                        /* groovylint-disable-next-line DuplicateStringLiteral */
+                        jobStatusUpdate(job_status, name, 'FAILURE')
+                        if (tryError == null) {
+                            /* groovylint-disable-next-line ThrowExceptionFromFinallyBlock */
+                            throw finallyError
+                        }
+                    }
                 }
             }
+            println("[${name}] Finished with ${job_status}")
         }
-        println("[${name}] Finished with ${job_status}")
     }
 }

vars/scriptedTestRpmStage.groovy

@@ -0,0 +1,113 @@
+/* groovylint-disable NestedBlockDepth */
+// vars/scriptedTestRpmStage.groovy
+
+import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
+
+/**
+ * scriptedTestRpmStage.groovy
+ *
+ * Get a test stage in scripted syntax.
+ *
+ * @param kwargs Map containing the following optional arguments (empty strings yield defaults):
+ *      name                  test stage name
+ *      runStage              whether or not to run the test stage; defaults to true
+ *      label                 test stage default cluster label
+ *      testBranch            if specified, checkout sources from this branch before running tests;
+ *                              defaults to ''
+ *      jobStatus             Map of status for each stage in the job/build
+ *      testRpmArgs           Map of arguments to pass to testRpm() for the stage
+ *      alwaysScript          script to always run after the test stage, e.g.
+ *                              'ci/rpm/test_daos_post.sh'; defaults to ''
+ *      archiveArtifactsArgs  Map of arguments to pass to archiveArtifacts() for the stage
+ *      distro                the distro to pass to daosRepos() if testRpmArgs does not specify a
+ *                              inst_repos, e.g. 'el9'; defaults to null
+ *      next_version          next daos package version to pass to daosPackagesVersion() if
+ *                              testRpmArgs does not specify a daos_pkg_version; defaults to null
+ * @return a scripted stage to run in a pipeline
+ */
+Map call(Map kwargs = [:]) {
+    // General parameters
+    String name = kwargs.get('name', '')
+    Boolean runStage = kwargs.get('runStage', true) as Boolean
+    String label = kwargs.get('label')
+    String testBranch = kwargs.get('testBranch', '')
+    Map jobStatus = kwargs.get('jobStatus', null) ?: [:]
+    Map testRpmArgs = kwargs.get('testRpmArgs', null) ?: [:]
+    String alwaysScript = kwargs.get('alwaysScript', '')
+    Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', null) ?: [:]
+
+    if (!name) {
+        error("scriptedTestRpmStage() requires a stage 'name' argument")
+    }
+
+    return {
+        stage("${name}") {
+            if (!runStage) {
+                println("[${name}] Stage skipped by runStage=false")
+                Utils.markStageSkippedForConditional("${name}")
+                return
+            }
+
+            // Add defaults for any missing testRpm() arguments
+            if (!testRpmArgs.containsKey('inst_repos')) {
+                /* groovylint-disable-next-line DuplicateStringLiteral */
+                testRpmArgs['inst_repos'] = daosRepos(kwargs.get('distro', null))
+            }
+            if (!testRpmArgs.containsKey('daos_pkg_version')) {
+                /* groovylint-disable-next-line DuplicateStringLiteral */
+                testRpmArgs['daos_pkg_version'] = daosPackagesVersion(
+                    kwargs.get('next_version', null))
+            }
+
+            node(label) {
+                // Ensure access to any branch provisioning scripts exist
+                if (testBranch) {
+                    println("[${name}] Check out '${testBranch}' from version control")
+                    checkoutScm(
+                        url: 'https://github.com/daos-stack/daos.git',
+                        branch: testBranch,
+                        withSubmodules: false,
+                        pruneStaleBranch: true)
+                } else {
+                    println("[${name}] Check out branch from version control")
+                    checkoutScm(pruneStaleBranch: true)
+                }
+
+                Throwable tryError = null
+                try {
+                    println("[${name}] Running testRpm() on ${label}")
+                    jobStatusUpdate(jobStatus, name, testRpm(testRpmArgs))
+                /* groovylint-disable-next-line CatchException */
+                } catch (Exception e) {
+                    tryError = e
+                    println("[${name}] Caught exception in try: ${tryError}")
+                    jobStatusUpdate(jobStatus, name, 'FAILURE')
+                    throw tryError
+                } finally {
+                    try {
+                        if (alwaysScript) {
+                            sh(script: alwaysScript,
+                               label: "Running alwaysScript: ${alwaysScript}",
+                               returnStatus: true)
+                        }
+                        if (archiveArtifactsArgs) {
+                            println("[${name}] Running archiveArtifacts()")
+                            archiveArtifacts(archiveArtifactsArgs)
+                        }
+                        jobStatusUpdate(jobStatus, name)
+                    /* groovylint-disable-next-line CatchException */
+                    } catch (Exception finallyError) {
+                        println("[${name}] Caught exception in finally: ${finallyError}")
+                        /* groovylint-disable-next-line DuplicateStringLiteral */
+                        jobStatusUpdate(jobStatus, name, 'FAILURE')
+                        if (tryError == null) {
+                            /* groovylint-disable-next-line ThrowExceptionFromFinallyBlock */
+                            throw finallyError
+                        }
+                    }
+                }
+            }
+            println("[${name}] Finished with ${jobStatus}")
+        }
+    }
+}

vars/scriptedUnitTestStage.groovy

@@ -0,0 +1,112 @@
+/* groovylint-disable NestedBlockDepth */
+// vars/scriptedUnitTestStage.groovy
+
+import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
+
+/**
+ * scriptedUnitTestStage.groovy
+ *
+ * Get a unit test stage in scripted syntax.
+ *
+ * @param kwargs Map containing the following optional arguments (empty strings yield defaults):
+ *      name                  test stage name
+ *      runStage              whether or not to run the test stage
+ *      label                 test stage default cluster label
+ *      testBranch            if specified, checkout sources from this branch before running tests
+ *      jobStatus             Map of status for each stage in the job/build
+ *      distro                the distro to use for daosRepos() and unitPackages() when providing
+ *                              default arguments in unitTestArgs, e.g. 'el9'; defaults to ''
+ *      unitTestArgs          Map of arguments to pass to unitTest; defaults to an empty Map.
+ *      unitTestPostArgs      Map of arguments to pass to unitTestPost() for the stage; defaults to
+ *                              an empty Map.
+ *      archiveArtifactsArgs  Map of arguments to pass to archiveArtifacts() for the stage; defaults
+ *                              to an empty Map.
+ * @return a scripted stage to run in a pipeline
+ */
+Map call(Map kwargs = [:]) {
+    // General parameters
+    String name = kwargs.get('name', '')
+    Boolean runStage = kwargs.get('runStage', true) as Boolean
+    String label = kwargs.get('label')
+    String testBranch = kwargs.get('testBranch', '')
+    Map jobStatus = kwargs.get('jobStatus', null) ?: [:]
+    String distro = kwargs.get('distro', '')
+
+    // Unit Test stage parameters
+    Map unitTestArgs = kwargs.get('unitTestArgs', null) ?: [:]
+    Map unitTestPostArgs = kwargs.get('unitTestPostArgs', null) ?: [:]
+    Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', null) ?: [:]
+
+    if (!name) {
+        error("scriptedUnitTestStage() requires a stage 'name' argument")
+    }
+
+    return {
+        stage("${name}") {
+            if (!runStage) {
+                println("[${name}] Stage skipped by runStage=false")
+                Utils.markStageSkippedForConditional("${name}")
+                return
+            }
+
+            // Add defaults for any missing unitTest() arguments
+            if (!unitTestArgs.containsKey('inst_repos')) {
+                /* groovylint-disable-next-line DuplicateStringLiteral */
+                unitTestArgs['inst_repos'] = daosRepos(distro)
+            }
+            if (!unitTestArgs.containsKey('inst_rpms')) {
+                /* groovylint-disable-next-line DuplicateStringLiteral */
+                unitTestArgs['inst_rpms'] = unitPackages(target: distro) + ' daos-client-tests'
+            }
+
+            node(label) {
+                // Ensure access to any branch provisioning scripts exist
+                if (testBranch) {
+                    println("[${name}] Check out '${testBranch}' from version control")
+                    checkoutScm(
+                        url: 'https://github.com/daos-stack/daos.git',
+                        branch: testBranch,
+                        withSubmodules: false,
+                        pruneStaleBranch: true)
+                } else {
+                    println("[${name}] Check out branch from version control")
+                    checkoutScm(pruneStaleBranch: true)
+                }
+
+                Throwable tryError = null
+                try {
+                    println("[${name}] Running unitTest() on ${label}")
+                    jobStatusUpdate(jobStatus, name, unitTest(unitTestArgs))
+                /* groovylint-disable-next-line CatchException */
+                } catch (Exception e) {
+                    tryError = e
+                    println("[${name}] Caught exception in try: ${tryError}")
+                    jobStatusUpdate(jobStatus, name, 'FAILURE')
+                    throw tryError
+                } finally {
+                    try {
+                        if (unitTestPostArgs) {
+                            println("[${name}] Running unitTestPost()")
+                            unitTestPost(unitTestPostArgs)
+                        }
+                        if (archiveArtifactsArgs) {
+                            println("[${name}] Running archiveArtifacts()")
+                            archiveArtifacts(archiveArtifactsArgs)
+                        }
+                        jobStatusUpdate(jobStatus, name)
+                    /* groovylint-disable-next-line CatchException */
+                    } catch (Exception finallyError) {
+                        println("[${name}] Caught exception in finally: ${finallyError}")
+                        /* groovylint-disable-next-line DuplicateStringLiteral */
+                        jobStatusUpdate(jobStatus, name, 'FAILURE')
+                        if (tryError == null) {
+                            /* groovylint-disable-next-line ThrowExceptionFromFinallyBlock */
+                            throw finallyError
+                        }
+                    }
+                }
+            }
+            println("[${name}] Finished with ${jobStatus}")
+        }
+    }
+}

vars/testsInStage.groovy

@@ -10,6 +10,7 @@
  * @return boolean true if there are tests that match the tags
  */
 boolean call(String tags) {
+    println("[${env.STAGE_NAME}] Determining if tests w/ '${tags}' tags exist for this stage")
     if (env.UNIT_TEST && env.UNIT_TEST == 'true') {
         return true
     }