DAOS-19247 build: Support running Bullseye

vars/buildRpmPost.groovy

@@ -49,13 +49,19 @@
    *
    * config['rpmlint']             Whether to run rpmlint on resulting RPMs.
    *                               Default false.
+   *
    * config['new_rpm']             Whether we are using new RPM or not
-   *                               Default false
+   *                               Default false.
+   *                               Deprecated, use config['productArtifacts'] instead.
+   *
+   * config['productArtifacts']    List of product names and artifact directories to publish.
+   *                               Default is to publish the whole target directory.
    *
    * config['unsuccessful_script'] Script to run if build is not successful.
    *                               Default 'ci/rpm/build_unsuccessful.sh'
    */
 
+/* groovylint-disable-next-line MethodSize */
 void call(Map config = [:]) {
     Map stage_info = parseStageInfo(config)
 
@@ -89,24 +95,37 @@ void call(Map config = [:]) {
                   includes: rpm_version_file
         }
 
+        List<String> productNames = config.get('productArtifacts', [])
+        // Backwards compatibility for new_rpm parameter.
+        if (!productNames && config.get('new_rpm', false)) {
+            productNames = ['daos', 'deps']
+        }
+
         String product = config.get('product', 'daos-stack')
-        String artdir = 'artifacts/' + target
-        if (config.get('new_rpm', false)) {
-            String deps_dir = 'artifacts/' + target + '/deps'
-            if (fileExists(deps_dir)) {
-                publishToRepository product: 'deps',
+        Map<String, String> productArtifacts = [:]
+        for (String name : productNames) {
+            if (name == 'daos') {
+                // Use the product name for daos
+                productArtifacts[product] = "artifacts/${target}/${name}"
+            } else {
+                // Use the specified name or other products, e.g. deps
+                productArtifacts[name] = "artifacts/${target}/${name}"
+            }
+        }
+        if (!productArtifacts) {
+            // Publish the whole target directory if no productArtifacts are specified
+            productArtifacts[product] = "artifacts/${target}"
+        }
+        for (String key in productArtifacts.keySet()) {
+            String artifactDir = productArtifacts[key]
+            if (fileExists(artifactDir)) {
+                publishToRepository product: key,
                                     format: repo_format,
                                     maturity: 'stable',
                                     tech: target,
-                                    repo_dir: deps_dir
+                                    repo_dir: artifactDir
             }
-            artdir = 'artifacts/' + target + '/daos'
         }
-        publishToRepository product: product,
-                            format: repo_format,
-                            maturity: 'stable',
-                            tech: target,
-                            repo_dir: artdir
 
         if (config.get('rpmlint', false)) {
             rpmlintMockResults(sh(label: 'Get chroot name',

vars/functionalTest.groovy

@@ -59,13 +59,21 @@
    *
    * config['ftest_arg']         Functional test launch.py arguments.
    *                             Default determined by parseStageInfo().
+   *
+   * config['coverage_stash']    Name to stash coverage artifacts.
+   *                             Default is empty string, which will result in no stashing.
+   *
+   * config['bullseye']          Set to true to use bullseye-sepecific repo.
+   *                             Default false.
    */
 
 Map call(Map config = [:]) {
     long startDate = System.currentTimeMillis()
     String nodelist = config.get('NODELIST', env.NODELIST)
     String context = config.get('context', 'test/' + env.STAGE_NAME)
     String description = config.get('description', env.STAGE_NAME)
+    String coverage_stash = config.get('coverage_stash', '')
+    Boolean bullseye = config.get('bullseye', false)
 
     Map stage_info = parseStageInfo(config)
 
@@ -93,7 +101,8 @@ Map call(Map config = [:]) {
                  node_count: stage_info['node_count'],
                  distro: image_version,
                  inst_repos: config.get('inst_repos', ''),
-                 inst_rpms: stage_inst_rpms)
+                 inst_rpms: stage_inst_rpms,
+                 bullseye: bullseye)
 
     List stashes = []
     if (config['stashes']) {
@@ -113,6 +122,7 @@ Map call(Map config = [:]) {
     run_test_config['ftest_arg'] = config.get('ftest_arg', stage_info['ftest_arg'])
     run_test_config['context'] = context
     run_test_config['description'] = description
+    run_test_config['coverage_stash'] = coverage_stash
 
     Map runtestData = [:]
     if (config.get('test_function', 'runTestFunctional') ==

vars/getFunctionalTestStage.groovy

@@ -24,15 +24,20 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
  *                      '.el9', '.suse.lp156', etc.  If not specified, it will be determined by
  *                      rpmDistValue(distro)
  *      base_branch     if specified, checkout sources from this branch before running tests
+ *      inst_rpms       space-separated string of RPM packages to install on the test nodes;
+ *                      exclusive of next_version, rpm_distro, and other_packages.
  *      other_packages  space-separated string of additional RPM packages to install
+ *      bullseye        whether or not to use the bullseye-sepecific repo for provisioning
  *      node_count      number of nodes to provision and use for the stage; overrides the count
  *                      that would otherwise be inferred from the stage name by parseStageInfo()
  *      runStage        whether or not to run the stage; overrides skipFunctionalTestStage()
  *      run_if_pr       whether or not the stage should run for PR builds
  *      run_if_landing  whether or not the stage should run for landing builds
  *      job_status      Map of status for each stage in the job/build
+ *      coverage_stash  name of stash to include code coverage results from the tests
  * @return a scripted stage to run in a pipeline
  */
+/* groovylint-disable-next-line MethodSize */
 Map call(Map kwargs = [:]) {
     String name = kwargs.get('name', 'Unknown Functional Test Stage')
     String pragma_suffix = kwargs.get('pragma_suffix')
@@ -47,7 +52,9 @@ Map call(Map kwargs = [:]) {
     String image_version = kwargs.get('image_version', null)
     String rpm_distro = kwargs.get('rpm_distro', null)
     String base_branch = kwargs.get('base_branch')
+    String inst_rpms = kwargs.get('inst_rpms', null)
     String other_packages = kwargs.get('other_packages', '')
+    Boolean bullseye = kwargs.get('bullseye', false)
     Integer node_count = kwargs.get('node_count') as Integer
     Boolean run_if_pr = kwargs.get('run_if_pr', false)
     Boolean run_if_landing = kwargs.get('run_if_landing', false)
@@ -57,39 +64,49 @@ Map call(Map kwargs = [:]) {
     // to be skipped by the existing skipFunctionalTestStage logic, while new stages can use
     // runStage directly. Once all stages have been converted to use runStage, this should be
     // changed to a Boolean.
-    String runStage = kwargs.get('runStage', 'undefined').toString()
+    final String UNDEFINED = 'undefined'
+    String runStage = kwargs.get('runStage', UNDEFINED)
 
     Map job_status = kwargs.get('job_status', [:])
+    String coverage_stash = kwargs.get('coverage_stash', '')
 
     return {
         stage("${name}") {
+            /* groovylint-disable-next-line DuplicateStringLiteral */
+            if (runStage == 'false') {
+                println("[${name}] Stage skipped by runStage=false")
+                Utils.markStageSkippedForConditional("${name}")
+                return
+            }
+
             // Get the tags for the stage. Use either the build parameter, commit pragma, or
             // default tags. All tags are combined with the stage tags to ensure only tests that
             // 'fit' the cluster will be run.
             String tags = getFunctionalTags(
                 pragma_suffix: pragma_suffix, stage_tags: stage_tags, default_tags: default_tags)
+            println("[${name}] Functional test stage tags: ${tags}")
+            if (runStage == 'true') {
+                println("[${name}] Checking if the tags match any tests to run in the stage")
+                if (!testsInStage(tags)) {
+                    println("[${name}] Stage skipped by no tests matching the '${tags}' tags")
+                    Utils.markStageSkippedForConditional("${name}")
+                    return
+                }
+            }
 
-            if (runStage == 'false') {
-                println("[${name}] Stage skipped by runStage=false")
-                Utils.markStageSkippedForConditional("${name}")
-                return
-            } else if (runStage == 'undefined') {
-                // To be removed once all stages have been converted to use runStage.
+            // To be removed once all stages have been converted to use runStage.
+            if (runStage == UNDEFINED) {
                 Map skip_kwargs = [
-                'tags': tags,
-                'pragma_suffix': pragma_suffix,
-                'distro': distro,
-                'run_if_pr': run_if_pr,
-                'run_if_landing': run_if_landing]
+                    'tags': tags,
+                    'pragma_suffix': pragma_suffix,
+                    'distro': distro,
+                    'run_if_pr': run_if_pr,
+                    'run_if_landing': run_if_landing]
                 if (skipFunctionalTestStage(skip_kwargs)) {
                     println("[${name}] Stage skipped by skipFunctionalTestStage()")
                     Utils.markStageSkippedForConditional("${name}")
                     return
                 }
-            } else if (!testsInStage(tags)) {
-                println("[${name}] Stage skipped by no tests matching the '${tags}' tags")
-                Utils.markStageSkippedForConditional("${name}")
-                return
             }
 
             node(cachedCommitPragma("Test-label${pragma_suffix}", label)) {
@@ -110,7 +127,7 @@ Map call(Map kwargs = [:]) {
                     Map ftestConfig = [
                         image_version: image_version,
                         inst_repos: daosRepos(distro),
-                        inst_rpms: functionalPackages(
+                        inst_rpms: inst_rpms ?: functionalPackages(
                             clientVersion: 1,
                             nextVersion: next_version,
                             addDaosPkgs: 'tests-internal',
@@ -121,8 +138,11 @@ Map call(Map kwargs = [:]) {
                             nvme: nvme,
                             default_nvme: default_nvme,
                             provider: provider)['ftest_arg'],
-                        test_function: 'runTestFunctionalV2']
+                        test_function: 'runTestFunctionalV2',
+                        bullseye: bullseye,
+                        coverage_stash: coverage_stash]
                     if (node_count != null) {
+                        /* groovylint-disable-next-line DuplicateStringLiteral */
                         ftestConfig['node_count'] = node_count
                     }
                     jobStatusUpdate(

vars/provisionNodes.groovy

@@ -26,6 +26,8 @@
  * config['timeout']    Timeout in minutes.  Default 30.
  * config['inst_repos'] DAOS stack repos that should be configured.
  * config['inst_rpms']  DAOS stack RPMs that should be installed.
+ * config['bullseye']   Set to true to use bullseye-specific repo.  Default false.
+ *
  *  if timeout is <= 0, then will not wait for provisioning.
  *  if power_only is specified, the nodes will be rebooted and the
  *  provisioning information ignored.
@@ -175,6 +177,7 @@ Map call(Map config = [:]) {
                       // https://issues.jenkins.io/browse/JENKINS-55819
                       'CI_RPM_TEST_VERSION="' + (params.CI_RPM_TEST_VERSION ?: '') + '" ' +
                       'CI_PR_REPOS="' + (params.CI_PR_REPOS ?: '') + '" ' +
+                      'CI_BULLSEYE="' + (config.get('bullseye', false) ? 'true' : 'false') + '" ' +
                       ((env.DAOS_HTTPS_PROXY ?: env.HTTPS_PROXY) ?
                           'HTTPS_PROXY="' + (env.DAOS_HTTPS_PROXY ?: env.HTTPS_PROXY) + '" ' : '') +
                       'ci/provisioning/post_provision_config.sh'

vars/runTestFunctionalV2.groovy

@@ -90,14 +90,18 @@ Map call(Map config = [:]) {
     // Restore the ignore failure setting
     config['ignore_failure'] = ignore_failure
 
-    String coverageFile = 'test.cov'
-    if (!fileExists('test.cov')) {
-        coverageFile += '_not_done'
-        fileOperations([fileCreateOperation(fileName: coverageFile,
-                                            fileContent: '')])
+    // Stash code coverage file if it exists
+    String coverage_stash = config.get('coverage_stash', '')
+    if (coverage_stash) {
+        try {
+            stash name: coverage_stash, includes: '**/test.cov'
+            println("[${env.STAGE_NAME}] Stashed code coverage file in ${coverage_stash}")
+        } catch (hudson.AbortException e) {
+            println(
+                "[${env.STAGE_NAME}] Failed to stash code coverage file in ${coverage_stash}: " +
+                "${e.message}")
+        }
     }
-    String name = 'func' + stage_info['pragma_suffix'] + '-cov'
-    stash name: config.get('coverage_stash', name),
-          includes: coverageFile
+
     return runData
 }