From 2e157fab922ea3af00bf4b5c90fcb54d0086083c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 1 Jul 2026 14:02:51 -0400 Subject: [PATCH 01/19] DAOS-19248 test: Fix no tag match for Functional VM stages Ensure Functional VM stages are skipped if no test tags match. Test-tag: test_always_passes_hw Skip-functional-on-el-8: false Skip-functional-on-el-9: false Skip-functional-on-leap-15: false Skip-functional-on-sles-15: false Signed-off-by: Phil Henderson --- Jenkinsfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dd086e88a0e..c918985e726 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1179,7 +1179,8 @@ pipeline { stage('Functional on EL 8.8 with Valgrind') { when { beforeAgent true - expression { shouldStageRun('Functional on EL 8.8 with Valgrind') } + expression { shouldStageRun('Functional on EL 8.8 with Valgrind') && + !testsInStage(getFunctionalTags('-vm', 'vm', 'pr')) } } agent { label vm9_label('EL8') @@ -1202,7 +1203,8 @@ pipeline { stage('Functional on EL 8') { when { beforeAgent true - expression { shouldStageRun('Functional on EL 8') } + expression { shouldStageRun('Functional on EL 8') && + !testsInStage(getFunctionalTags('-vm', 'vm', 'pr')) } } agent { label vm9_label('EL8') @@ -1226,7 +1228,8 @@ pipeline { stage('Functional on EL 9') { when { beforeAgent true - expression { shouldStageRun('Functional on EL 9') } + expression { shouldStageRun('Functional on EL 9') && + !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} } agent { label vm9_label('EL9') @@ -1250,7 +1253,8 @@ pipeline { stage('Functional on Leap 15') { when { beforeAgent true - expression { shouldStageRun('Functional on Leap 15') } + expression { shouldStageRun('Functional on Leap 15') && + !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} } agent { label vm9_label('Leap15') @@ -1274,7 +1278,8 @@ pipeline { stage('Functional on SLES 15') { when { beforeAgent true - expression { shouldStageRun('Functional on SLES 15') } + expression { shouldStageRun('Functional on SLES 15') && + !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} } agent { label vm9_label('Leap15') @@ -1298,7 +1303,8 @@ pipeline { stage('Functional on Ubuntu 20.04') { when { beforeAgent true - expression { shouldStageRun('Functional on Ubuntu 20.04') } + expression { shouldStageRun('Functional on Ubuntu 20.04') && + !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} } agent { label vm9_label('Ubuntu') From ab22926fb0aa4380cc622c989a4f3a6d71047b27 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 1 Jul 2026 18:54:26 -0400 Subject: [PATCH 02/19] Use scripted pipeline methods for the Test stages. Skip-test-hardware: true Test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 388 ++++++++++++++++++---------------------------------- 1 file changed, 130 insertions(+), 258 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c918985e726..59f3542fd75 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,6 +21,7 @@ import groovy.transform.Field // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value='pipeline-lib@your_branch') _ +@Library(value='pipeline-lib@hendersp/DAOS-19247-fix') _ /* groovylint-disable-next-line CompileStatic */ job_status_internal = [:] @@ -1175,268 +1176,139 @@ pipeline { beforeAgent true expression { shouldStageRun('Test') } } - parallel { - stage('Functional on EL 8.8 with Valgrind') { - when { - beforeAgent true - expression { shouldStageRun('Functional on EL 8.8 with Valgrind') && - !testsInStage(getFunctionalTags('-vm', 'vm', 'pr')) } - } - agent { - label vm9_label('EL8') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } - } // stage('Functional on EL 8.8 with Valgrind') - stage('Functional on EL 8') { - when { - beforeAgent true - expression { shouldStageRun('Functional on EL 8') && - !testsInStage(getFunctionalTags('-vm', 'vm', 'pr')) } - } - agent { - label vm9_label('EL8') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2', - image_version: 'el8.10')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } - } // stage('Functional on EL 8') - stage('Functional on EL 9') { - when { - beforeAgent true - expression { shouldStageRun('Functional on EL 9') && - !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} - } - agent { - label vm9_label('EL9') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2', - image_version: 'el9.7')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } - } // stage('Functional on EL 9') - stage('Functional on Leap 15') { - when { - beforeAgent true - expression { shouldStageRun('Functional on Leap 15') && - !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} - } - agent { - label vm9_label('Leap15') - } - steps { - job_step_update( - functionalTest( - inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2', - image_version: 'leap15.6')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } // post - } // stage('Functional on Leap 15') - stage('Functional on SLES 15') { - when { - beforeAgent true - expression { shouldStageRun('Functional on SLES 15') && - !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} - } - agent { - label vm9_label('Leap15') - } - steps { - job_step_update( - functionalTest( + steps { + script { + parallel( + 'Functional on EL 8.8 with Valgrind': getFunctionalTestStage( + name: 'Functional on EL 8.8 with Valgrind', + runStage: shouldStageRun('Functional on EL 8.8 with Valgrind'), + pragma_suffix: '-vm', + label: vm9_label('EL8'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: 'memcheck', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'el8.8' + ), + 'Functional on EL 8': getFunctionalTestStage( + name: 'Functional on EL 8', + runStage: shouldStageRun('Functional on EL 8'), + pragma_suffix: '-vm', + label: vm9_label('EL8'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'el8.10' + ), + 'Functional on EL 9': getFunctionalTestStage( + name: 'Functional on EL 9', + runStage: shouldStageRun('Functional on EL 9'), + pragma_suffix: '-vm', + label: vm9_label('EL9'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'el9.7' + ), + 'Functional on Leap 15': getFunctionalTestStage( + name: 'Functional on Leap 15', + runStage: shouldStageRun('Functional on Leap 15'), + pragma_suffix: '-vm', + label: vm9_label('Leap15'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'leap15.6' + ), + 'Functional on SLES 15': getFunctionalTestStage( + name: 'Functional on SLES 15', + runStage: shouldStageRun('Functional on SLES 15'), + pragma_suffix: '-vm', + label: vm9_label('Leap15'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal, + image_version: 'sles15.7' + ), + 'Functional on Ubuntu 20.04': getFunctionalTestStage( + name: 'Functional on Ubuntu 20.04', + runStage: shouldStageRun('Functional on Ubuntu 20.04'), + pragma_suffix: '-vm', + label: vm9_label('Ubuntu'), + next_version: next_version(), + other_packages: 'mercury-libfabric', + stage_tags: 'vm', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + job_status: job_status_internal + ), + 'Fault injection testing': scriptedTestStage( + name: 'Fault injection testing', + runStage: shouldStageRun('Fault injection testing'), + label: params.CI_FI_1_LABEL, + job_status: job_status_internal, + unitTestArgs: [ + timeout_time: 240, inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2', - image_version: 'sles15.7')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } // post - } // stage('Functional on SLES 15') - stage('Functional on Ubuntu 20.04') { - when { - beforeAgent true - expression { shouldStageRun('Functional on Ubuntu 20.04') && - !testsInStage(getFunctionalTags('-vm', 'vm', 'pr'))} - } - agent { - label vm9_label('Ubuntu') - } - steps { - job_step_update( - functionalTest( + test_script: 'ci/unit/test_nlt.sh --memcheck no' + + ' --system-ram-reserved 4 --server-debug WARN' + + ' --log-usage-import nltr.json' + + ' --log-usage-save nltr.xml' + + ' --class-name fault-injection fi', + with_valgrind: '', + always_script: 'ci/unit/test_nlt_post.sh', + testResults: 'nlt-junit.xml', + unstash_opt: true, + unstash_tests: false, + inst_rpms: unitPackages(target: 'el9') + ' daos-client-tests', + image_version: 'el9.7', + prov_env_vars: 'VM_CPUS=14'], + unitTestPostArgs: [ + artifacts: ['nlt_logs/'], + testResults: 'nlt-junit.xml', + with_valgrind: '', + FI: true], + archiveArtifactsArgs: [ + artifacts: 'nlt_logs/fault-injection/', + allowEmptyArchive: true] + ), + 'Test RPMs on EL 9.6': scriptedTestStage( + name: 'Test RPMs on EL 9.6', + runStage: shouldStageRun('Test RPMs on EL 9.6'), + label: params.CI_UNIT_VM1_LABEL, + job_status: job_status_internal, + testRpmArgs: [ inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', - test_function: 'runTestFunctionalV2')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } // post - } // stage('Functional on Ubuntu 20.04') - stage('Fault injection testing') { - when { - beforeAgent true - expression { shouldStageRun('Fault injection testing') } - } - agent { - label params.CI_FI_1_LABEL - } - steps { - job_step_update( - unitTest(timeout_time: 240, - inst_repos: daosRepos(), - test_script: 'ci/unit/test_nlt.sh --memcheck no' + - ' --system-ram-reserved 4 --server-debug WARN' + - ' --log-usage-import nltr.json' + - ' --log-usage-save nltr.xml' + - ' --class-name fault-injection fi', - with_valgrind: '', - always_script: 'ci/unit/test_nlt_post.sh', - testResults: 'nlt-junit.xml', - unstash_opt: true, - unstash_tests: false, - inst_rpms: unitPackages(target: 'el9') + ' daos-client-tests', - image_version: 'el9.7', - prov_env_vars: 'VM_CPUS=14')) - } - post { - always { - unitTestPost artifacts: ['nlt_logs/'], - testResults: 'nlt-junit.xml', - with_valgrind: '', - FI: true - archiveArtifacts artifacts: 'nlt_logs/fault-injection/', - allowEmptyArchive: true - job_status_update() - } - } - } // stage('Fault injection testing') - stage('Test RPMs on EL 9.6') { - when { - beforeAgent true - expression { shouldStageRun('Test RPMs on EL 9.6') } - } - agent { - label params.CI_UNIT_VM1_LABEL - } - steps { - job_step_update( - testRpm(inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version()), - inst_rpms: 'mercury-libfabric') - ) - } - post { - always { - rpm_test_post(env.STAGE_NAME, env.NODELIST) - } - } - } // stage('Test RPMs on EL 9.6') - stage('Test RPMs on Leap 15.5') { - when { - beforeAgent true - expression { shouldStageRun('Test RPMs on Leap 15.5') } - } - agent { - label params.CI_UNIT_VM1_LABEL - } - steps { - /* neither of these work as FTest strips the first node - out of the pool requiring 2 node clusters at minimum - * additionally for this use-case, can't override - ftest_arg with this :-( - script { - 'Test RPMs on Leap 15.5': getFunctionalTestStage( - name: 'Test RPMs on Leap 15.5', - pragma_suffix: '', - label: params.CI_UNIT_VM1_LABEL, - next_version: next_version(), - stage_tags: '', - default_tags: 'test_daos_management', - nvme: 'auto', - run_if_pr: true, - run_if_landing: true, - job_status: job_status_internal - ) - } - job_step_update( - functionalTest( - test_tag: 'test_daos_management', - ftest_arg: '--yaml_extension single_host', + daos_pkg_version: daosPackagesVersion(next_version()), + inst_rpms: 'mercury-libfabric'] + ), + 'Test RPMs on Leap 15.5': scriptedTestStage( + name: 'Test RPMs on Leap 15.5', + runStage: shouldStageRun('Test RPMs on Leap 15.5'), + label: params.CI_UNIT_VM1_LABEL, + job_status: job_status_internal, + testRpmArgs: [ inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal'), - test_function: 'runTestFunctionalV2')) - } - post { - always { - functionalTestPostV2() - job_status_update() - } - } */ - job_step_update( - testRpm(inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version()), - inst_rpms: 'mercury-libfabric') + daos_pkg_version: daosPackagesVersion(next_version()), + inst_rpms: 'mercury-libfabric'] ) - } - post { - always { - rpm_test_post(env.STAGE_NAME, env.NODELIST) - } - } - } // stage('Test RPMs on Leap 15.5') - } // parallel + ) + } + } } // stage('Test') stage('Test Storage Prep on EL 8.8') { when { From e2c7edaf89cf6c9872cdca2f72461d55ab6fccbf Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 1 Jul 2026 22:58:52 -0400 Subject: [PATCH 03/19] Updates. skip-unit-tests: true skip-test-hardware: true test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 59f3542fd75..322583259d4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1260,15 +1260,15 @@ pipeline { name: 'Fault injection testing', runStage: shouldStageRun('Fault injection testing'), label: params.CI_FI_1_LABEL, - job_status: job_status_internal, + jobStatus: job_status_internal, unitTestArgs: [ timeout_time: 240, inst_repos: daosRepos(), test_script: 'ci/unit/test_nlt.sh --memcheck no' + - ' --system-ram-reserved 4 --server-debug WARN' + - ' --log-usage-import nltr.json' + - ' --log-usage-save nltr.xml' + - ' --class-name fault-injection fi', + ' --system-ram-reserved 4 --server-debug WARN' + + ' --log-usage-import nltr.json' + + ' --log-usage-save nltr.xml' + + ' --class-name fault-injection fi', with_valgrind: '', always_script: 'ci/unit/test_nlt_post.sh', testResults: 'nlt-junit.xml', @@ -1290,7 +1290,7 @@ pipeline { name: 'Test RPMs on EL 9.6', runStage: shouldStageRun('Test RPMs on EL 9.6'), label: params.CI_UNIT_VM1_LABEL, - job_status: job_status_internal, + jobStatus: job_status_internal, testRpmArgs: [ inst_repos: daosRepos(), daos_pkg_version: daosPackagesVersion(next_version()), @@ -1300,7 +1300,7 @@ pipeline { name: 'Test RPMs on Leap 15.5', runStage: shouldStageRun('Test RPMs on Leap 15.5'), label: params.CI_UNIT_VM1_LABEL, - job_status: job_status_internal, + jobStatus: job_status_internal, testRpmArgs: [ inst_repos: daosRepos(), daos_pkg_version: daosPackagesVersion(next_version()), From 231152d34d652cdc5bd6f9ffbb0b598f5c1f031c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 01:24:57 -0400 Subject: [PATCH 04/19] Updates. skip-unit-tests: true skip-test-hardware: true test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 63 +++++++++++++++++++---------------------------------- 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 322583259d4..279943a63c5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -470,19 +470,6 @@ String vm9_label(String distro) { def_val: params.FUNCTIONAL_VM_LABEL)) } -void rpm_test_post(String stageName, String node) { - // Extract first node from comma-delimited list - String firstNode = node.split(',')[0].trim() - sh label: 'Fetch and stage artifacts', - script: 'hostname; ssh -i ci_key jenkins@' + firstNode + - ' ls -ltar /tmp; mkdir -p "' + env.STAGE_NAME + '/" && ' + - 'scp -i ci_key jenkins@' + firstNode + - ':/tmp/{{suite_dmg,daos_{server_helper,{control,agent}}}.log,daos_server.log.*} "' + - stageName + '/"' - archiveArtifacts artifacts: env.STAGE_NAME + '/**' - job_status_update() -} - String sconsArgs() { if (!params.CI_SCONS_ARGS) { return sconsFaultsArgs() @@ -1256,27 +1243,25 @@ pipeline { nvme: 'auto', job_status: job_status_internal ), - 'Fault injection testing': scriptedTestStage( + 'Fault injection testing': scriptedUnitTestStage( name: 'Fault injection testing', runStage: shouldStageRun('Fault injection testing'), label: params.CI_FI_1_LABEL, jobStatus: job_status_internal, - unitTestArgs: [ - timeout_time: 240, - inst_repos: daosRepos(), - test_script: 'ci/unit/test_nlt.sh --memcheck no' + - ' --system-ram-reserved 4 --server-debug WARN' + - ' --log-usage-import nltr.json' + - ' --log-usage-save nltr.xml' + - ' --class-name fault-injection fi', - with_valgrind: '', - always_script: 'ci/unit/test_nlt_post.sh', - testResults: 'nlt-junit.xml', - unstash_opt: true, - unstash_tests: false, - inst_rpms: unitPackages(target: 'el9') + ' daos-client-tests', - image_version: 'el9.7', - prov_env_vars: 'VM_CPUS=14'], + distro: 'el9', + timeoutTime: 240, + testScript: 'ci/unit/test_nlt.sh --memcheck no' + + ' --system-ram-reserved 4 --server-debug WARN' + + ' --log-usage-import nltr.json' + + ' --log-usage-save nltr.xml' + + ' --class-name fault-injection fi', + withValgrind: '', + alwaysScript: 'ci/unit/test_nlt_post.sh', + testResults: 'nlt-junit.xml', + unstashOpt: true, + unstashTests: false, + imageVersion: 'el9.7', + provEnvVars: 'VM_CPUS=14', unitTestPostArgs: [ artifacts: ['nlt_logs/'], testResults: 'nlt-junit.xml', @@ -1286,25 +1271,23 @@ pipeline { artifacts: 'nlt_logs/fault-injection/', allowEmptyArchive: true] ), - 'Test RPMs on EL 9.6': scriptedTestStage( + 'Test RPMs on EL 9.6': scriptedTestRpmStage( name: 'Test RPMs on EL 9.6', runStage: shouldStageRun('Test RPMs on EL 9.6'), label: params.CI_UNIT_VM1_LABEL, jobStatus: job_status_internal, - testRpmArgs: [ - inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version()), - inst_rpms: 'mercury-libfabric'] + nextVersion: next_version(), + instRpms: 'mercury-libfabric', + nodeList: env.NODELIST, ), - 'Test RPMs on Leap 15.5': scriptedTestStage( + 'Test RPMs on Leap 15.5': scriptedTestRpmStage( name: 'Test RPMs on Leap 15.5', runStage: shouldStageRun('Test RPMs on Leap 15.5'), label: params.CI_UNIT_VM1_LABEL, jobStatus: job_status_internal, - testRpmArgs: [ - inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version()), - inst_rpms: 'mercury-libfabric'] + nextVersion: next_version(), + instRpms: 'mercury-libfabric', + nodeList: env.NODELIST, ) ) } From 9658ef1faf144febf0429d7796f3bc91ef81c55c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 11:57:06 -0400 Subject: [PATCH 05/19] Updates skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 6 ++++++ ci/rpm/test_daos_post.sh | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100755 ci/rpm/test_daos_post.sh diff --git a/Jenkinsfile b/Jenkinsfile index 279943a63c5..de7b257e7a4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1279,6 +1279,9 @@ pipeline { nextVersion: next_version(), instRpms: 'mercury-libfabric', nodeList: env.NODELIST, + alwaysScript: 'ci/rpm/test_daos_post.sh test_rpms_el_logs', + archiveArtifactsArgs: [ + artifacts: 'test_rpms_el_logs/'] ), 'Test RPMs on Leap 15.5': scriptedTestRpmStage( name: 'Test RPMs on Leap 15.5', @@ -1288,6 +1291,9 @@ pipeline { nextVersion: next_version(), instRpms: 'mercury-libfabric', nodeList: env.NODELIST, + alwaysScript: 'ci/rpm/test_daos_post.sh test_rpms_leap_logs', + archiveArtifactsArgs: [ + artifacts: 'test_rpms_leap_logs/'] ) ) } diff --git a/ci/rpm/test_daos_post.sh b/ci/rpm/test_daos_post.sh new file mode 100755 index 00000000000..5f98e6d045c --- /dev/null +++ b/ci/rpm/test_daos_post.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# This is a post test processing script for post processing the +# run_utest.py stage CI run + +set -uex + +test_log_dir="${1:-}" +first_node="${NODELIST%%,*}" + +if [ -z "$test_log_dir" ]; then + echo "test_daos_post: The test log directory argument is missing!" + exit 1 +fi + +# Copy logs from test_daos_node.sh execution +rm -rf "$test_log_dir" +mkdir -p "$test_log_dir" +rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ + --filter="include suite_dmg.log" \ + --filter="include daos_server_helper.log" \ + --filter="include daos_control.log" \ + --filter="include daos_agent.log" \ + --filter="include daos_server.log.*" \ + --filter="exclude *" "${test_log_dir}/" From d8d25ab9e28f645b2c73e4f1d91ddd0b5fbfda1c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 17:56:40 -0400 Subject: [PATCH 06/19] Test artifact dir name. Skip-cancel-previous-builds: true Skip-functional-test: true SKip-fault-injection-testing: true Skip-unit-tests: true Signed-off-by: Phil Henderson --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index de7b257e7a4..774382bd72b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1291,9 +1291,9 @@ pipeline { nextVersion: next_version(), instRpms: 'mercury-libfabric', nodeList: env.NODELIST, - alwaysScript: 'ci/rpm/test_daos_post.sh test_rpms_leap_logs', + alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15.5\'', archiveArtifactsArgs: [ - artifacts: 'test_rpms_leap_logs/'] + artifacts: '\'Test RPMs on Leap 15.5/\''] ) ) } From 33213bbae20b479c08b6f6dbf17654d589e7500b Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 18:42:59 -0400 Subject: [PATCH 07/19] Test RPM updates skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 8 +++++--- ci/rpm/test_daos_post.sh | 17 +++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 774382bd72b..674f8ef7bb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1277,11 +1277,12 @@ pipeline { label: params.CI_UNIT_VM1_LABEL, jobStatus: job_status_internal, nextVersion: next_version(), + imageVersion: 'el9.6', instRpms: 'mercury-libfabric', nodeList: env.NODELIST, - alwaysScript: 'ci/rpm/test_daos_post.sh test_rpms_el_logs', + alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on EL 9.6\'', archiveArtifactsArgs: [ - artifacts: 'test_rpms_el_logs/'] + artifacts: 'Test RPMs on EL 9.6/'] ), 'Test RPMs on Leap 15.5': scriptedTestRpmStage( name: 'Test RPMs on Leap 15.5', @@ -1289,11 +1290,12 @@ pipeline { label: params.CI_UNIT_VM1_LABEL, jobStatus: job_status_internal, nextVersion: next_version(), + imageVersion: 'leap15.5', instRpms: 'mercury-libfabric', nodeList: env.NODELIST, alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15.5\'', archiveArtifactsArgs: [ - artifacts: '\'Test RPMs on Leap 15.5/\''] + artifacts: 'Test RPMs on Leap 15.5/'] ) ) } diff --git a/ci/rpm/test_daos_post.sh b/ci/rpm/test_daos_post.sh index 5f98e6d045c..e1e9fa9b3ef 100755 --- a/ci/rpm/test_daos_post.sh +++ b/ci/rpm/test_daos_post.sh @@ -8,18 +8,19 @@ set -uex test_log_dir="${1:-}" first_node="${NODELIST%%,*}" -if [ -z "$test_log_dir" ]; then +if [ -z "${test_log_dir}" ]; then echo "test_daos_post: The test log directory argument is missing!" exit 1 fi # Copy logs from test_daos_node.sh execution -rm -rf "$test_log_dir" -mkdir -p "$test_log_dir" +rm -rf "${test_log_dir}" +mkdir -p "${test_log_dir}/configs" +rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/etc/daos/ \ + --filter="include daos_*.yml*" \ + --filter="exclude *" "${test_log_dir}/configs/" +mkdir -p "${test_log_dir}/logs" rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ --filter="include suite_dmg.log" \ - --filter="include daos_server_helper.log" \ - --filter="include daos_control.log" \ - --filter="include daos_agent.log" \ - --filter="include daos_server.log.*" \ - --filter="exclude *" "${test_log_dir}/" + --filter="include daos_*.log*" \ + --filter="exclude *" "${test_log_dir}/logs/" From e57a1e0efbee3752a5675aab3bd15884e7471957 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 22:48:16 -0400 Subject: [PATCH 08/19] Hack to install pciutils-3.13 skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 674f8ef7bb9..e23b43cbaa1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1291,7 +1291,7 @@ pipeline { jobStatus: job_status_internal, nextVersion: next_version(), imageVersion: 'leap15.5', - instRpms: 'mercury-libfabric', + instRpms: 'mercury-libfabric pciutils-3.13.0-150300.13.12.1.x86_64', nodeList: env.NODELIST, alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15.5\'', archiveArtifactsArgs: [ From 7b33944d9f67c6cf2e25d0e9d26b13ea31105b53 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 23:06:31 -0400 Subject: [PATCH 09/19] Collect log for test_daos_node.sh execution sjip-cancel-previous-builds: true skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-9: true skip-fault-injection-testing: true Signed-off-by: Phil Henderson --- ci/rpm/test_daos.sh | 10 +++++----- ci/rpm/test_daos_post.sh | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ci/rpm/test_daos.sh b/ci/rpm/test_daos.sh index 6e1f415d237..67fba6bc44d 100755 --- a/ci/rpm/test_daos.sh +++ b/ci/rpm/test_daos.sh @@ -9,9 +9,9 @@ set -uex IFS=" " read -r -a nodelist <<< "${NODELIST//,/ }" mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -# shellcheck disable=SC2029 ssh "$SSH_KEY_ARGS" jenkins@"${nodelist[0]}" \ - "NODE=${nodelist[0]} \ - DAOS_PKG_VERSION=$DAOS_PKG_VERSION \ - PYTHON_VERSION=\"${PYTHON_VERSION}\" \ - $(cat "$mydir/test_daos_node.sh")" + "NODE=${nodelist[0]} \ + DAOS_PKG_VERSION=$DAOS_PKG_VERSION \ + PYTHON_VERSION=${PYTHON_VERSION} \ + bash -s 2>&1 | tee /tmp/test_daos_rpms.log" \ + < "$mydir/test_daos_node.sh" diff --git a/ci/rpm/test_daos_post.sh b/ci/rpm/test_daos_post.sh index e1e9fa9b3ef..392e1245960 100755 --- a/ci/rpm/test_daos_post.sh +++ b/ci/rpm/test_daos_post.sh @@ -24,3 +24,6 @@ rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ --filter="include suite_dmg.log" \ --filter="include daos_*.log*" \ --filter="exclude *" "${test_log_dir}/logs/" +rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ + --filter="include test_daos_rpms.log" \ + --filter="exclude *" "${test_log_dir}/" From a643198b05e97201975cfa258a7f366274e31329 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 23:52:09 -0400 Subject: [PATCH 10/19] Log collection hardening skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-9: true skip-fault-injection-testing: true Signed-off-by: Phil Henderson --- ci/rpm/test_daos.sh | 4 ++++ ci/rpm/test_daos_node.sh | 3 +++ ci/rpm/test_daos_post.sh | 6 +++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ci/rpm/test_daos.sh b/ci/rpm/test_daos.sh index 67fba6bc44d..0f732aa140e 100755 --- a/ci/rpm/test_daos.sh +++ b/ci/rpm/test_daos.sh @@ -9,9 +9,13 @@ set -uex IFS=" " read -r -a nodelist <<< "${NODELIST//,/ }" mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +# shellcheck disable=SC2029 ssh "$SSH_KEY_ARGS" jenkins@"${nodelist[0]}" \ "NODE=${nodelist[0]} \ DAOS_PKG_VERSION=$DAOS_PKG_VERSION \ PYTHON_VERSION=${PYTHON_VERSION} \ bash -s 2>&1 | tee /tmp/test_daos_rpms.log" \ < "$mydir/test_daos_node.sh" + + # bash -s > /tmp/test_daos_rpms.log 2>&1" + # < "$mydir/test_daos_node.sh" diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index ce16704c884..4cc9728d018 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -29,6 +29,9 @@ if [ -n "$DAOS_PKG_VERSION" ]; then fi set -uex + +echo "${PRETTY_NAME:-Unknown OS}" + sudo $YUM -y install daos-client"$DAOS_PKG_VERSION" if rpm -q daos-server; then echo "daos-server RPM should not be installed as a dependency of daos-client" diff --git a/ci/rpm/test_daos_post.sh b/ci/rpm/test_daos_post.sh index 392e1245960..1ab04d9d63f 100755 --- a/ci/rpm/test_daos_post.sh +++ b/ci/rpm/test_daos_post.sh @@ -18,12 +18,12 @@ rm -rf "${test_log_dir}" mkdir -p "${test_log_dir}/configs" rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/etc/daos/ \ --filter="include daos_*.yml*" \ - --filter="exclude *" "${test_log_dir}/configs/" + --filter="exclude *" "${test_log_dir}/configs/" || true mkdir -p "${test_log_dir}/logs" rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ --filter="include suite_dmg.log" \ --filter="include daos_*.log*" \ - --filter="exclude *" "${test_log_dir}/logs/" + --filter="exclude *" "${test_log_dir}/logs/" || true rsync -v -dpt -z -e "ssh ${SSH_KEY_ARGS}" jenkins@"${first_node}":/tmp/ \ --filter="include test_daos_rpms.log" \ - --filter="exclude *" "${test_log_dir}/" + --filter="exclude *" "${test_log_dir}/" || true From 19629f9635cdc042146c3baaaf3af3cab0a38c4c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 3 Jul 2026 00:23:29 -0400 Subject: [PATCH 11/19] Remove hack skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e23b43cbaa1..674f8ef7bb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1291,7 +1291,7 @@ pipeline { jobStatus: job_status_internal, nextVersion: next_version(), imageVersion: 'leap15.5', - instRpms: 'mercury-libfabric pciutils-3.13.0-150300.13.12.1.x86_64', + instRpms: 'mercury-libfabric', nodeList: env.NODELIST, alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15.5\'', archiveArtifactsArgs: [ From dcfa2102f682464f5e176d901bd8e3ada2dc6d32 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Mon, 6 Jul 2026 12:13:29 -0400 Subject: [PATCH 12/19] Retain test_daos_node.sh exit status skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- ci/rpm/test_daos.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ci/rpm/test_daos.sh b/ci/rpm/test_daos.sh index 0f732aa140e..bd0d3e5e62a 100755 --- a/ci/rpm/test_daos.sh +++ b/ci/rpm/test_daos.sh @@ -11,11 +11,6 @@ mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" # shellcheck disable=SC2029 ssh "$SSH_KEY_ARGS" jenkins@"${nodelist[0]}" \ - "NODE=${nodelist[0]} \ - DAOS_PKG_VERSION=$DAOS_PKG_VERSION \ - PYTHON_VERSION=${PYTHON_VERSION} \ - bash -s 2>&1 | tee /tmp/test_daos_rpms.log" \ + "bash -lc 'NODE=${nodelist[0]} DAOS_PKG_VERSION=$DAOS_PKG_VERSION PYTHON_VERSION=$PYTHON_VERSION \ + bash -s 2>&1 | tee /tmp/test_daos_rpms.log; exit \${PIPESTATUS[0]}'" \ < "$mydir/test_daos_node.sh" - - # bash -s > /tmp/test_daos_rpms.log 2>&1" - # < "$mydir/test_daos_node.sh" From 42d0f64903c83cd40c7b214ad0e1043f29db914c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 7 Jul 2026 10:59:48 -0400 Subject: [PATCH 13/19] Run RPM test on Leap 15.6 skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 22 +++++++++++----------- src/tests/ftest/harness/setup.yaml | 3 ++- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 674f8ef7bb9..62adc9dd8ec 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1271,31 +1271,31 @@ pipeline { artifacts: 'nlt_logs/fault-injection/', allowEmptyArchive: true] ), - 'Test RPMs on EL 9.6': scriptedTestRpmStage( - name: 'Test RPMs on EL 9.6', - runStage: shouldStageRun('Test RPMs on EL 9.6'), + 'Test RPMs on EL 9': scriptedTestRpmStage( + name: 'Test RPMs on EL 9', + runStage: shouldStageRun('Test RPMs on EL 9'), label: params.CI_UNIT_VM1_LABEL, jobStatus: job_status_internal, nextVersion: next_version(), imageVersion: 'el9.6', instRpms: 'mercury-libfabric', nodeList: env.NODELIST, - alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on EL 9.6\'', + alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on EL 9\'', archiveArtifactsArgs: [ - artifacts: 'Test RPMs on EL 9.6/'] + artifacts: 'Test RPMs on EL 9/'] ), - 'Test RPMs on Leap 15.5': scriptedTestRpmStage( - name: 'Test RPMs on Leap 15.5', - runStage: shouldStageRun('Test RPMs on Leap 15.5'), + 'Test RPMs on Leap 15': scriptedTestRpmStage( + name: 'Test RPMs on Leap 15', + runStage: shouldStageRun('Test RPMs on Leap 15'), label: params.CI_UNIT_VM1_LABEL, jobStatus: job_status_internal, nextVersion: next_version(), - imageVersion: 'leap15.5', + imageVersion: 'leap15.6', instRpms: 'mercury-libfabric', nodeList: env.NODELIST, - alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15.5\'', + alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15\'', archiveArtifactsArgs: [ - artifacts: 'Test RPMs on Leap 15.5/'] + artifacts: 'Test RPMs on Leap 15/'] ) ) } diff --git a/src/tests/ftest/harness/setup.yaml b/src/tests/ftest/harness/setup.yaml index b744014dab9..c7d753601d1 100644 --- a/src/tests/ftest/harness/setup.yaml +++ b/src/tests/ftest/harness/setup.yaml @@ -7,7 +7,8 @@ server_config: engines_per_host: 1 engines: 0: - storage: auto + class: ram + scm_mount: /mnt/daos server_manager: storage_prepare_timeout: 60 storage_format_timeout: 60 From 5171bd6c950147ddf3509b8d4de206ae748204f3 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 7 Jul 2026 11:18:28 -0400 Subject: [PATCH 14/19] Fix missing satge name updates. skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 62adc9dd8ec..62eb5644af0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -61,8 +61,8 @@ void updateRunStage() { 'Functional on SLES 15', 'Functional on Ubuntu 20.04', 'Fault injection testing', - 'Test RPMs on EL 9.6', - 'Test RPMs on Leap 15.5', + 'Test RPMs on EL 9', + 'Test RPMs on Leap 15', 'Test Hardware', 'Functional Hardware Medium', 'Functional Hardware Medium MD on SSD', @@ -195,7 +195,7 @@ void updateRunStage() { } // These stages are unrelated to functional testing and should be skipped for (stage in ['Unit Tests', 'Fault injection testing', - 'Test RPMs on EL 9.6', 'Test RPMs on Leap 15.5']) { + 'Test RPMs on EL 9', 'Test RPMs on Leap 15']) { runStage[stage] = false reasons[stage] = "Quick functional testing" } @@ -637,12 +637,12 @@ pipeline { booleanParam(name: bashName('Fault injection testing'), defaultValue: true, description: 'Run the Fault injection testing stage.') - booleanParam(name: bashName('Test RPMs on EL 9.6'), + booleanParam(name: bashName('Test RPMs on EL 9'), defaultValue: true, description: 'Run the Test RPMs on EL 9.6 stage.') - booleanParam(name: bashName('Test RPMs on Leap 15.5'), + booleanParam(name: bashName('Test RPMs on Leap 15'), defaultValue: true, - description: 'Run the Test RPMs on Leap 15.5 stage.') + description: 'Run the Test RPMs on Leap 15.6 stage.') booleanParam(name: bashName('Test Hardware'), defaultValue: true, description: 'Run the Test Hardware stage.') From 081152702b2876b147f05f2ba0eff94f0659a4e1 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 7 Jul 2026 11:20:14 -0400 Subject: [PATCH 15/19] Fixes skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi test_setup_vm Signed-off-by: Phil Henderson --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 62eb5644af0..ab657424ca0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -639,10 +639,10 @@ pipeline { description: 'Run the Fault injection testing stage.') booleanParam(name: bashName('Test RPMs on EL 9'), defaultValue: true, - description: 'Run the Test RPMs on EL 9.6 stage.') + description: 'Run the Test RPMs on EL 9 stage.') booleanParam(name: bashName('Test RPMs on Leap 15'), defaultValue: true, - description: 'Run the Test RPMs on Leap 15.6 stage.') + description: 'Run the Test RPMs on Leap 15 stage.') booleanParam(name: bashName('Test Hardware'), defaultValue: true, description: 'Run the Test Hardware stage.') From ed9ea3dc1920d5939d9f95b3185d7ec12d4b92f6 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 7 Jul 2026 14:58:01 -0400 Subject: [PATCH 16/19] Remove unintended change. skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi Signed-off-by: Phil Henderson --- src/tests/ftest/harness/setup.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tests/ftest/harness/setup.yaml b/src/tests/ftest/harness/setup.yaml index c7d753601d1..b744014dab9 100644 --- a/src/tests/ftest/harness/setup.yaml +++ b/src/tests/ftest/harness/setup.yaml @@ -7,8 +7,7 @@ server_config: engines_per_host: 1 engines: 0: - class: ram - scm_mount: /mnt/daos + storage: auto server_manager: storage_prepare_timeout: 60 storage_format_timeout: 60 From 5d2695c399fe58542c641959bfd207f9580741c5 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 8 Jul 2026 23:29:12 -0400 Subject: [PATCH 17/19] Applying feedback. skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi Signed-off-by: Phil Henderson --- ci/rpm/test_daos_post.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/rpm/test_daos_post.sh b/ci/rpm/test_daos_post.sh index 1ab04d9d63f..ec3b0c1b2c9 100755 --- a/ci/rpm/test_daos_post.sh +++ b/ci/rpm/test_daos_post.sh @@ -1,7 +1,7 @@ #!/bin/bash # This is a post test processing script for post processing the -# run_utest.py stage CI run +# test_daos.sh stage CI run set -uex From 7b2f7e440949b3b4d60000e68e20e67a8d3cd27d Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 9 Jul 2026 03:02:06 -0400 Subject: [PATCH 18/19] Groovylint fixes skip-unit-tests: true skip-test-hardware: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi Signed-off-by: Phil Henderson --- Jenkinsfile | 61 ++++++++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 47e03225d81..8067838a933 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,6 +35,7 @@ String bashName(String name) { } // Update the runStage map +/* groovylint-disable-next-line MethodSize */ void updateRunStage() { Map reasons = [:] @@ -78,20 +79,20 @@ void updateRunStage() { // Initialize the run state of each stage using the parameter stage keys for (name in stageOrder) { value = params.get(bashName(name), null) - if (value instanceof Boolean && !name.startsWith('CI_')) { + if (value != null && value.class == Boolean && !name.startsWith('CI_')) { runStage[name] = value - reasons[name] = "parameter selection or default" + reasons[name] = 'parameter selection or default' } } // Debug - String buildCause = currentBuild.getBuildCauses().toString() - println("updateRunStage: Build cause: ${buildCause}") + List buildCauses = currentBuild.buildCauses + println("updateRunStage: Build cause: ${buildCauses}") println("updateRunStage: Started by user: ${startedByUser()}") // Handle landing builds if (startedByLanding()) { - println("updateRunStage: Detected landing build, overwriting defaults") + println('updateRunStage: Detected landing build, overwriting defaults') for (stage in runStage.keySet()) { if (stage in ['Pre-build', 'Python Bandit check', 'Build', 'Unit Tests', 'Test'] || stage.contains('Build on') @@ -104,7 +105,7 @@ void updateRunStage() { } else { runStage[stage] = false } - reasons[stage] = "landing build" + reasons[stage] = 'landing build' } displayRunStage(reasons) return @@ -112,11 +113,11 @@ void updateRunStage() { // Handle doc-only changes: Only run default or selected build stages if (docOnlyChange(target_branch)) { - println("updateRunStage: Detected doc-only change, skipping testing") + println('updateRunStage: Detected doc-only change, skipping testing') for (stage in runStage.keySet()) { if (stage in ['Unit Tests', 'Test', 'Test Hardware']) { runStage[stage] = false - reasons[stage] = "doc-only change" + reasons[stage] = 'doc-only change' } } displayRunStage(reasons) @@ -125,13 +126,13 @@ void updateRunStage() { // Handle user setting CI_RPM_TEST_VERSION or specifying RPM-test-version if (rpmTestVersion()) { - println("updateRunStage: Detected RPM test version, skipping build/RPM test stages") + println('updateRunStage: Detected RPM test version, skipping build/RPM test stages') for (stage in runStage.keySet()) { if (stage.contains('Build') || stage.contains('Unit Tests') || stage.contains('Test RPMs')) { runStage[stage] = false - reasons[stage] = "RPM test version" + reasons[stage] = 'RPM test version' } } displayRunStage(reasons) @@ -140,14 +141,14 @@ void updateRunStage() { // Handle user setting CI_BUILD_PACKAGES_ONLY if (params.CI_BUILD_PACKAGES_ONLY) { - println("updateRunStage: Detected CI_BUILD_PACKAGES_ONLY, skipping unit test stages") + println('updateRunStage: Detected CI_BUILD_PACKAGES_ONLY, skipping unit test stages') for (stage in runStage.keySet()) { if (stage.contains('Unit Tests')) { runStage[stage] = false - reasons[stage] = "CI_BUILD_PACKAGES_ONLY" + reasons[stage] = 'CI_BUILD_PACKAGES_ONLY' } else if (stage.contains('Build')) { runStage[stage] = true - reasons[stage] = "CI_BUILD_PACKAGES_ONLY" + reasons[stage] = 'CI_BUILD_PACKAGES_ONLY' } } displayRunStage(reasons) @@ -157,7 +158,7 @@ void updateRunStage() { // Handle user setting CI_IGNORE_SKIP_COMMIT_PRAGMAS if (params.CI_IGNORE_SKIP_COMMIT_PRAGMAS) { println( - "updateRunStage: Detected CI_IGNORE_SKIP_COMMIT_PRAGMAS, ignoring skip commit pragmas") + 'updateRunStage: Detected CI_IGNORE_SKIP_COMMIT_PRAGMAS, ignoring skip commit pragmas') displayRunStage(reasons) return } @@ -165,12 +166,13 @@ void updateRunStage() { // Update stage running based on commit pragmas println("updateRunStage: Converting env.pragmas string back into a Map: ${env.pragmas}") Map commitPragmas = envToPragmas() - println("updateRunStage: Checking skip commit pragmas from commit message:") + println('updateRunStage: Checking skip commit pragmas from commit message:') commitPragmas.each { key, value -> println(" ${key}: ${value}") } for (stage in runStage.keySet()) { List skipPragmas = getStageNameSkipPragmas(stage) + /* groovylint-disable-next-line NestedForLoop */ for (pragma in skipPragmas) { // commitPragmas will already contain lower case keys from pragmasToMap() println("updateRunStage: ${stage} checking for a ${pragma} commit pragma") @@ -188,17 +190,18 @@ void updateRunStage() { // Handle quick functional commit pragma if (quickFunctional()) { - println("updateRunStage: Detected quick functional testing") + println('updateRunStage: Detected quick functional testing') // These stages must be run for functional testing for (stage in ['Pre-build', 'Python Bandit check', 'Build']) { runStage[stage] = true - reasons[stage] = "Quick functional testing" + reasons[stage] = 'Quick functional testing' } // These stages are unrelated to functional testing and should be skipped + /* groovylint-disable-next-line BracesForForLoop */ for (stage in ['Unit Tests', 'Fault injection testing', 'Test RPMs on EL 9', 'Test RPMs on Leap 15']) { runStage[stage] = false - reasons[stage] = "Quick functional testing" + reasons[stage] = 'Quick functional testing' } // Build stages should only be run if their RPMs are needed String hwBuildStage = 'Build on ' @@ -226,13 +229,13 @@ void updateRunStage() { // Initially skip all the build stages for (stage in testBuildStage.values().toSet()) { runStage[stage] = false - reasons[stage] = "Quick functional testing" + reasons[stage] = 'Quick functional testing' } // The mapped build stage must be run to generate RPMs for the functional test stage for (stage in testBuildStage.keySet()) { if (runStage[stage]) { runStage[testBuildStage[stage]] = true - reasons[testBuildStage[stage]] = "Quick functional testing" + reasons[testBuildStage[stage]] = 'Quick functional testing' } } } @@ -242,7 +245,7 @@ void updateRunStage() { // Log which stages will be run and why based on the current state of the runStage map void displayRunStage(Map reasons = [:]) { - println("Stage run conditions:") + println('Stage run conditions:') for (stage in runStage.keySet()) { String reason = reasons.get(stage, 'default') if (runStage[stage]) { @@ -254,6 +257,7 @@ void displayRunStage(Map reasons = [:]) { } // Get a list of skip commit pragmas to check for a given stage name +/* groovylint-disable-next-line MethodSize */ List getStageNameSkipPragmas(String stageName) { String stagePragma = "skip-${stageName.replaceAll(' ', '-').toLowerCase()}" List pragmas = [] @@ -262,13 +266,11 @@ List getStageNameSkipPragmas(String stageName) { if (stageName in ['Cancel Previous Builds', 'Pre-build']) { // Add skip pragma for this stage pragmas.add(stagePragma) - } else if (stageName == 'Python Bandit check') { // Add skip pragma for this stage pragmas.add(stagePragma) // Compatibility with existing commit pragmas pragmas.add(stagePragma.replace('-bandit-check', '-bandit')) - } else if (stageName.contains('Build')) { // Add skip pragma for parent stage if (stageName != 'Build') { @@ -280,7 +282,6 @@ List getStageNameSkipPragmas(String stageName) { if (stagePragma.contains('build-on-')) { pragmas.add(stagePragma.replace('build-on-', 'build-')) } - } else if (stageName.contains('Unit Test') || stageName.contains('NLT')) { // Add skip pragma for parent stage if (stageName != 'Unit Tests') { @@ -292,7 +293,6 @@ List getStageNameSkipPragmas(String stageName) { if (stagePragma.contains('-with-')) { pragmas.add(stagePragma.replace('-with-', '-')) } - } else if (stageName == 'Test' || stageName.contains('Functional on') || stageName.contains('Fault injection') || stageName.contains('Test RPMs')) { // Add skip pragma for parent stage @@ -317,7 +317,6 @@ List getStageNameSkipPragmas(String stageName) { } // Add skip pragma for this stage pragmas.add(stagePragma) - } else if (stageName.contains('Hardware') || stageName.contains('Cluster Box')) { // Add skip pragma for parent stage if (stageName != 'Test Hardware') { @@ -341,6 +340,7 @@ List getStageNameSkipPragmas(String stageName) { List distros = ['el', 'leap', 'sles', 'ubuntu'] List copyPragmas = pragmas.clone() for (distro in distros) { + /* groovylint-disable-next-line NestedForLoop */ for (_pragma in copyPragmas) { if (_pragma.contains("-${distro}-")) { Integer _index = pragmas.indexOf(_pragma) @@ -365,7 +365,7 @@ List getStageNameSkipPragmas(String stageName) { // Initialize the runStage map with the current state of the build parameters and any commit // pragmas related to skipping/running stages. Should only be called once per build. -def setupRunStage() { +void setupRunStage() { pragmasToEnv() update_default_commit_pragmas() updateRunStage() @@ -1068,7 +1068,8 @@ pipeline { // NLT memchecks the valgrind-tagged build, not the shared -race one. unstash 'opt-daos-valgrind' job_step_update( - unitTest(timeout_time: 60 * cachedCommitPragma(pragma: 'NLT-repeat', def_val: '1').toInteger(), + unitTest(timeout_time: 60 * cachedCommitPragma(pragma: 'NLT-repeat', + def_val: '1').toInteger(), inst_repos: daosRepos(), test_script: 'ci/unit/test_nlt.sh' + ' --system-ram-reserved 4' + @@ -1077,6 +1078,7 @@ pipeline { ' --log-usage-save nltir.xml' + ' --log-usage-export nltr.json' + ' --class-name nlt' + + /* groovylint-disable-next-line LineLength */ " --repeat ${cachedCommitPragma(pragma: 'NLT-repeat', def_val: '1')}" + /* groovylint-disable-next-line LineLength */ (cachedCommitPragma(pragma: 'NLT-repeat-failfast', def_val: 'false').toLowerCase() == 'true' ? ' --failfast' : '') + @@ -1274,6 +1276,7 @@ pipeline { imageVersion: 'el9.7', provEnvVars: 'VM_CPUS=14', unitTestPostArgs: [ + /* groovylint-disable-next-line DuplicateListLiteral */ artifacts: ['nlt_logs/'], testResults: 'nlt-junit.xml', with_valgrind: '', @@ -1446,7 +1449,7 @@ pipeline { pragma_suffix:'-cb-medium-md-on-ssd', label: params.FUNCTIONAL_CLUSTER_BOX_MEDIUM_LABEL, next_version: next_version(), - stage_tags: "cb,medium", + stage_tags: 'cb,medium', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto_md_on_ssd', node_count: 5, From 44643c8b7e07307e461bd9b652cde758e91242ff Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 14 Jul 2026 10:49:22 -0400 Subject: [PATCH 19/19] Applying feddback and pipeline-lib updates skip-unit-tests: true skip-functional-on-el-8: false skip-functional-on-leap-15: false skip-functional-on-sles-15: false test-tag: test_load_mpi Signed-off-by: Phil Henderson --- Jenkinsfile | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8067838a933..523c26b9744 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1262,19 +1262,20 @@ pipeline { label: params.CI_FI_1_LABEL, jobStatus: job_status_internal, distro: 'el9', - timeoutTime: 240, - testScript: 'ci/unit/test_nlt.sh --memcheck no' + - ' --system-ram-reserved 4 --server-debug WARN' + - ' --log-usage-import nltr.json' + - ' --log-usage-save nltr.xml' + - ' --class-name fault-injection fi', - withValgrind: '', - alwaysScript: 'ci/unit/test_nlt_post.sh', - testResults: 'nlt-junit.xml', - unstashOpt: true, - unstashTests: false, - imageVersion: 'el9.7', - provEnvVars: 'VM_CPUS=14', + unitTestArgs: [ + timeout_time: 240, + test_script: 'ci/unit/test_nlt.sh --memcheck no' + + ' --system-ram-reserved 4 --server-debug WARN' + + ' --log-usage-import nltr.json' + + ' --log-usage-save nltr.xml' + + ' --class-name fault-injection fi', + always_script: 'ci/unit/test_nlt_post.sh', + testResults: 'nlt-junit.xml', + unstash_opt: true, + unstash_tests: false, + image_version: 'el9.7', + prov_env_vars: 'VM_CPUS=14' + ], unitTestPostArgs: [ /* groovylint-disable-next-line DuplicateListLiteral */ artifacts: ['nlt_logs/'], @@ -1290,11 +1291,12 @@ pipeline { runStage: shouldStageRun('Test RPMs on EL 9'), label: params.CI_UNIT_VM1_LABEL, jobStatus: job_status_internal, + testRpmArgs: [ + target: 'el9.6', + inst_rpms: 'mercury-libfabric', + NODELIST: env.NODELIST], nextVersion: next_version(), - imageVersion: 'el9.6', - instRpms: 'mercury-libfabric', - nodeList: env.NODELIST, - alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on EL 9\'', + postScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on EL 9\'', archiveArtifactsArgs: [ artifacts: 'Test RPMs on EL 9/'] ), @@ -1303,11 +1305,12 @@ pipeline { runStage: shouldStageRun('Test RPMs on Leap 15'), label: params.CI_UNIT_VM1_LABEL, jobStatus: job_status_internal, + testRpmArgs: [ + target: 'leap15.6', + inst_rpms: 'mercury-libfabric', + NODELIST: env.NODELIST], nextVersion: next_version(), - imageVersion: 'leap15.6', - instRpms: 'mercury-libfabric', - nodeList: env.NODELIST, - alwaysScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15\'', + postScript: 'ci/rpm/test_daos_post.sh \'Test RPMs on Leap 15\'', archiveArtifactsArgs: [ artifacts: 'Test RPMs on Leap 15/'] )