diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index c58b0fbed99..997048a1a7f 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -29,7 +29,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python environment - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.11' - name: Install extra python packages @@ -90,7 +90,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python environment - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.11' - name: Add parser @@ -144,7 +144,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python environment - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.11' - name: Install python packages @@ -163,7 +163,7 @@ jobs: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Python environment - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.11' - name: Install extra python packages @@ -211,7 +211,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python environment - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.11' - name: Install extra python packages diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 2a3fe856301..1aadcd101ef 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -1,7 +1,7 @@ name: Unit testing on: - pull_request: + workflow_dispatch: permissions: {} diff --git a/Jenkinsfile b/Jenkinsfile index dd086e88a0e..357f489d6a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -70,7 +70,8 @@ void updateRunStage() { 'Functional Hardware Medium Verbs Provider MD on SSD', 'Functional Hardware Medium UCX Provider', 'Functional Hardware Large', - 'Functional Hardware Large MD on SSD' + 'Functional Hardware Large MD on SSD', + 'Functional Cluster Box Medium MD on SSD' ] // Initialize the run state of each stage using the parameter stage keys @@ -218,7 +219,9 @@ void updateRunStage() { 'Functional Hardware Medium Verbs Provider MD on SSD': hwBuildStage, 'Functional Hardware Medium UCX Provider': hwBuildStage, 'Functional Hardware Large': hwBuildStage, - 'Functional Hardware Large MD on SSD': hwBuildStage] + 'Functional Hardware Large MD on SSD': hwBuildStage, + 'Functional Cluster Box Medium MD on SSD': hwBuildStage, + ] // Initially skip all the build stages for (stage in testBuildStage.values().toSet()) { runStage[stage] = false @@ -314,8 +317,8 @@ List getStageNameSkipPragmas(String stageName) { // Add skip pragma for this stage pragmas.add(stagePragma) - } else if (stageName.contains('Hardware')) { - // + } else if (stageName.contains('Hardware') || stageName.contains('Cluster Box')) { + // Add skip pragma for parent stage if (stageName != 'Test Hardware') { pragmas.add('skip-test-hardware') } @@ -682,6 +685,9 @@ pipeline { booleanParam(name: bashName('Functional Hardware Large MD on SSD'), defaultValue: true, description: 'Run the Functional Hardware Large MD on SSD stage.') + booleanParam(name: bashName('Functional Cluster Box Medium MD on SSD'), + defaultValue: true, + description: 'Run the Functional Cluster Box test stage') string(name: 'CI_UNIT_VM1_LABEL', defaultValue: 'ci_vm1', description: 'Label to use for 1 VM node unit and RPM tests') @@ -712,6 +718,9 @@ pipeline { string(name: 'FUNCTIONAL_HARDWARE_LARGE_LABEL', defaultValue: 'ci_nvme9', description: 'Label to use for 9 node Functional Hardware Large (MD on SSD) stages') + string(name: 'FUNCTIONAL_CLUSTER_BOX_MEDIUM_LABEL', + defaultValue: 'cluster_box', + description: 'Label to use for the Functional Cluster Box stages') string(name: 'CI_STORAGE_PREP_LABEL', defaultValue: '', description: 'Label for cluster to do a DAOS Storage Preparation') @@ -729,8 +738,10 @@ pipeline { stage('Set Description') { steps { script { - if (params.CI_BUILD_DESCRIPTION) { - buildDescription params.CI_BUILD_DESCRIPTION + String description = params.CI_BUILD_DESCRIPTION ?: + cachedCommitPragma('Build-description', '') + if (description) { + buildDescription description } } } @@ -961,7 +972,7 @@ pipeline { deps_build: false) + ' --build-arg DAOS_PACKAGES_BUILD=no ' + ' --build-arg DAOS_KEEP_SRC=yes ' + - " -t ${sanitized_JOB_NAME()}-leap15-gcc" + + " -t ${sanitized_JOB_NAME()}-leap15-gcc" + " -t ${sanitized_JOB_NAME()}-leap15" + ' --build-arg POINT_RELEASE=.6' + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" @@ -1560,6 +1571,21 @@ pipeline { job_status: job_status_internal, image_version: 'el9.7' ), + 'Functional Cluster Box Medium MD on SSD': getFunctionalTestStage( + name: 'Functional Cluster Box Medium MD on SSD', + runStage: shouldStageRun('Functional Cluster Box Medium MD on SSD'), + pragma_suffix:'-cb-medium-md-on-ssd', + label: params.FUNCTIONAL_CLUSTER_BOX_MEDIUM_LABEL, + next_version: next_version(), + stage_tags: "cb,medium", + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto_md_on_ssd', + node_count: 5, + run_if_pr: true, + run_if_landing: false, + job_status: job_status_internal, + image_version: 'el9.7' + ), ) } } diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md new file mode 100644 index 00000000000..2f777d8f41d --- /dev/null +++ b/PR_DESCRIPTION.md @@ -0,0 +1,193 @@ +# Add allow-numa-imbalance flag to config generate + +## Summary + +This PR adds a new `--allow-numa-imbalance` / `-n` flag to the `dmg config generate` and `daos_server config generate` commands. This flag enables generation of server configurations where NVMe devices are distributed equally across engines regardless of NUMA affinity. + +## Problem Statement + +Currently, the config generation commands enforce balanced NVMe device distribution that respects NUMA affinity. When generating a config: + +1. The tool calculates the lowest common number of SSDs across all NUMA nodes +2. It restricts each engine to use only that minimum number of SSDs from its local NUMA node +3. Extra SSDs on NUMA nodes with more devices are excluded from the config + +This prevents maximizing device utilization in heterogeneous hardware environments where: +- Different NUMA nodes have different numbers of NVMe devices +- Hardware constraints result in uneven device distribution +- Users want to utilize all available SSDs even if it requires cross-NUMA access + +## Solution + +Add a `--allow-numa-imbalance` flag that: + +1. **Collects all SSDs**: Gathers all available NVMe devices from all NUMA nodes +2. **Distributes equally**: Allocates SSDs evenly across all engines, ignoring NUMA boundaries +3. **Handles remainders**: Uses maximum divisible number of SSDs; remainder SSDs are not used +4. **Warns about performance**: Flag description includes warning about suboptimal performance +5. **Sets server config parameter**: The generated config includes `allow_numa_imbalance: true` to allow the server to start with this configuration +6. **Maintains backward compatibility**: The default behavior (without the flag) remains unchanged + +### Example Behavior + +**Before (without flag) - NUMA-aware balancing:** +- NUMA-0: 4 SSDs available → 2 SSDs used (limited to minimum) +- NUMA-1: 2 SSDs available → 2 SSDs used +- Total: 4 of 6 SSDs used + +**After (with flag) - Equal distribution:** +- NUMA-0: 4 SSDs available → 3 SSDs assigned (may include devices from NUMA-1) +- NUMA-1: 2 SSDs available → 3 SSDs assigned (may include devices from NUMA-0) +- Total: All 6 SSDs used, distributed equally + +**With remainder (7 SSDs / 2 engines):** +- All 7 SSDs collected, but only 6 used (3 per engine) +- 1 SSD remains unused (not evenly divisible) +- Warning logged: "using 6 SSDs (3 per engine), 1 SSDs will not be used" + +## Changes Made + +### 1. Command-line Interface (`src/control/common/cmdutil/auto.go`) +- Added `AllowNumaImbalance bool` field to `ConfGenCmd` struct +- Added short flag `-n` and long flag `--allow-numa-imbalance` +- Flag description: "Redistribute NVMe devices equally across engines regardless of NUMA affinity" + +### 2. Control Library (`src/control/lib/control/auto.go`) +- Added `AllowNumaImbalance bool` field to `ConfGenerateReq` struct +- Modified `correctSSDCounts()` function to accept `allowImbalance` parameter + - When `false`, performs NUMA-aware balancing (existing behavior) + - When `true`, calls new `distributeSSDs()` function +- Added new `distributeSSDs()` function: + - Collects all SSDs from all NUMA nodes + - Validates that VMD devices are not present (cannot distribute VMD) + - Calculates SSDs per engine (total / nrEngines) + - If remainder exists, uses only maximum divisible number + - Logs notice when SSDs are unused due to remainder + - Distributes SSDs equally across engines + - Logs detailed information about distribution +- Updated `genEngineConfigs()` to pass the flag to `correctSSDCounts()` +- Modified `genServerConfig()` to call `WithAllowNumaImbalance()` on generated config + +### 3. Tests (`src/control/lib/control/auto_test.go`) +- Updated `TestControl_AutoConfig_correctSSDCounts` to test the new parameter +- Added test case "allow imbalance distributes equally": + - 4 SSDs on NUMA-0 + 2 on NUMA-1 = 6 total + - Distributes to 3 SSDs per engine + - Verifies equal distribution +- Added test case "allow imbalance with remainder discards extras": + - 5 SSDs on NUMA-0 + 2 on NUMA-1 = 7 total + - Uses 6 SSDs (3 per engine), 1 SSD unused + - Verifies remainder handling +- Added test case "allow imbalance with 8 SSDs across 2 engines": + - 5 SSDs on NUMA-0 + 3 on NUMA-1 = 8 total + - Distributes to 4 SSDs per engine + - Verifies equal distribution works +- Existing test cases verify default behavior still works correctly + +### 4. Documentation (`docs/admin/deployment.md`) +- Added flag to both `daos_server config generate` and `dmg config generate` help output +- Added detailed description explaining: + - Default NUMA-aware balancing behavior + - Effect of the flag (equal distribution regardless of NUMA) + - Handling of remainders (maximum divisible number used) + - **Performance warning**: May result in suboptimal performance + - That it sets `allow_numa_imbalance: true` in the generated config + +## Use Cases + +This feature is useful for: + +1. **Development/testing environments** with heterogeneous hardware +2. **Hardware upgrades** where NVMe devices are added incrementally to specific nodes +3. **Hardware failures** where some devices are unavailable on certain NUMA nodes +4. **Specialized configurations** requiring maximum utilization despite imbalance +5. **Cost-optimized deployments** with mixed hardware specifications + +## Example Usage + +### Before (without flag) +```bash +$ daos_server config generate +# NUMA-0: 4 SSDs available, 2 used (limited to minimum) +# NUMA-1: 2 SSDs available, 2 used +# Total: 4 of 6 SSDs utilized +``` + +### After (with flag) +```bash +$ daos_server config generate --allow-numa-imbalance +# Collects all 6 SSDs and redistributes equally: +# - Engine 0: Gets 3 SSDs (regardless of which NUMA node they're on) +# - Engine 1: Gets 3 SSDs (regardless of which NUMA node they're on) +# - Config includes: allow_numa_imbalance: true +# Total: All 6 SSDs utilized +``` + +### DMG remote generation +```bash +$ dmg config generate -l host1,host2 --allow-numa-imbalance +# Generates config allowing imbalanced NVMe distribution across the hostset +``` + +## Testing + +### Unit Tests +- Added test case for `correctSSDCounts()` with imbalance flag +- Verified existing tests still pass (default behavior unchanged) + +### Manual Testing +Run the following to verify: +```bash +# Test without flag (default behavior) +daos_server config generate + +# Test with flag (new behavior) +daos_server config generate --allow-numa-imbalance + +# Verify generated config contains: allow_numa_imbalance: true +``` + +## Backward Compatibility + +✅ **Fully backward compatible** +- Default behavior unchanged (flag defaults to `false`) +- Existing scripts and automation continue to work +- No breaking changes to API or config format + +## Future Enhancements + +Potential follow-up work: +1. Add validation to warn users about performance implications of imbalanced configs +2. Provide recommendations for device distribution optimization +3. Add metrics to monitor imbalance effects on production systems + +## Checklist + +- [x] Code changes implemented +- [x] Unit tests added/updated +- [x] Documentation updated +- [x] Commit message follows guidelines +- [x] Signed-off-by line included +- [ ] CI/CD tests pass +- [ ] Review by control plane experts + +## Related Issues + +- Addresses user requests for config generation in heterogeneous environments +- Related to DAOS-16979 (hugepage/NUMA imbalance handling) + +--- + +**Testing Instructions for Reviewers:** + +1. Verify flag appears in help output: + ```bash + daos_server config generate --help | grep numa + dmg config generate --help | grep numa + ``` + +2. Test with imbalanced hardware (if available) or mock setup + +3. Verify generated config includes `allow_numa_imbalance: true` when flag is used + +4. Confirm default behavior (without flag) still balances SSDs correctly diff --git a/docs/admin/deployment.md b/docs/admin/deployment.md index 633a48a83c8..29381f4db12 100644 --- a/docs/admin/deployment.md +++ b/docs/admin/deployment.md @@ -298,6 +298,8 @@ Help Options: MD-on-SSD config -f, --fabric-ports= Allow custom fabric interface ports to be specified for each engine config section. Comma separated port numbers, one per engine + -n, --allow-numa-imbalance Distribute NVMe devices equally across engines regardless of NUMA + affinity (may result in suboptimal performance due to cross-NUMA access) --skip-prep Skip preparation of devices during scan. ``` @@ -344,6 +346,8 @@ Help Options: MD-on-SSD config -f, --fabric-ports= Allow custom fabric interface ports to be specified for each engine config section. Comma separated port numbers, one per engine + -n, --allow-numa-imbalance Distribute NVMe devices equally across engines regardless of NUMA + affinity (may result in suboptimal performance due to cross-NUMA access) ``` The `daos_server` service must be running on the remote storage servers and as such a minimal @@ -404,6 +408,17 @@ this option is set then a MD-on-SSD config will be generated. - `--fabric-ports` enables custom port numbers to be assigned to each engine's fabric settings. Comma separated list must contain enough numbers to cover all engines generated in config. +- `--allow-numa-imbalance` allows the generation of a server config with NVMe devices distributed +equally across engines regardless of NUMA affinity. By default, config generation respects NUMA +affinity and balances the number of SSDs assigned to each engine by limiting all engines to use +only the lowest common number of SSDs across NUMA nodes. When this flag is set, all available SSDs +are collected and distributed equally across all engines, ignoring NUMA node boundaries. If the total +number of SSDs is not evenly divisible by the number of engines, only the maximum divisible number +of SSDs are used (e.g., 7 SSDs / 2 engines = 3 per engine, 1 SSD unused). This maximizes device +utilization in heterogeneous environments but **may result in suboptimal performance** due to +cross-NUMA memory access. The generated config will include `allow_numa_imbalance: true` to allow +the server to start with this configuration. + The text generated by the command and output to stdout can be copied and used as the server config file on relevant hosts (normally by copying to `/etc/daos/daos_server.yml` and (re)starting service). diff --git a/requirements-ftest.txt b/requirements-ftest.txt index 7b86d613036..0a89809bcc9 100644 --- a/requirements-ftest.txt +++ b/requirements-ftest.txt @@ -2,6 +2,7 @@ avocado-framework==82 avocado-framework-plugin-result-html==82 avocado-framework-plugin-varianter-yaml-to-mux==82 clustershell -paramiko distro +paramiko +setuptools<82 torch diff --git a/src/cart/crt_swim.c b/src/cart/crt_swim.c index f2a81ee7b6d..70f5f334246 100644 --- a/src/cart/crt_swim.c +++ b/src/cart/crt_swim.c @@ -1642,8 +1642,10 @@ crt_swim_rank_check(struct crt_grp_priv *grp_priv, d_rank_t rank, uint64_t incar } crt_swim_csm_unlock(csm); - if (updated) + if (updated) { + swim_member_reset(csm->csm_ctx, rank); crt_swim_notify_rank_state(rank, &state_prev, &state); + } return rc; } diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h index 4095aea3c55..ab14572fa1f 100644 --- a/src/client/dfuse/dfuse.h +++ b/src/client/dfuse/dfuse.h @@ -1,6 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -91,8 +91,22 @@ struct dfuse_eq { struct d_slab_type *de_read_slab; struct d_slab_type *de_pre_read_slab; struct d_slab_type *de_write_slab; + + ATOMIC uint32_t de_empty_polls; }; +/* + * Wake up the EQ progress thread for newly queued work. Reset de_empty_polls + * first so any inherited backoff is cleared before the sem_post, otherwise + * the freshly submitted event could be delayed by a stale backoff window. + */ +static inline void +dfuse_eq_wakeup(struct dfuse_eq *eqt) +{ + atomic_store_relaxed(&eqt->de_empty_polls, 0); + sem_post(&eqt->de_sem); +} + /* Maximum size dfuse expects for read requests, this is not a limit but rather what is expected * This is the maximum size expected from the kernel, increasing this without changing kernel * behavior will have no effect. diff --git a/src/client/dfuse/dfuse_core.c b/src/client/dfuse/dfuse_core.c index 01c6a2e7c27..bf898de7cbf 100644 --- a/src/client/dfuse/dfuse_core.c +++ b/src/client/dfuse/dfuse_core.c @@ -1,6 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * (C) Copyright 2025-2026 Google LLC. * * SPDX-License-Identifier: BSD-2-Clause-Patent @@ -11,12 +11,45 @@ #include "dfuse_common.h" #include "dfuse.h" +#define DFUSE_EQ_BACKOFF_START 64 +#define DFUSE_EQ_BACKOFF_MIN_US 50 +#define DFUSE_EQ_BACKOFF_MAX_US 5000 + +static int64_t +dfuse_eq_backoff_us(uint32_t empty_polls) +{ + uint32_t backoff = DFUSE_EQ_BACKOFF_MIN_US; + uint32_t steps; + + if (empty_polls < DFUSE_EQ_BACKOFF_START) + return 0; + + steps = (empty_polls - DFUSE_EQ_BACKOFF_START) / DFUSE_EQ_BACKOFF_START; + while (steps-- > 0 && backoff < DFUSE_EQ_BACKOFF_MAX_US) + backoff *= 2; + + if (backoff > DFUSE_EQ_BACKOFF_MAX_US) + backoff = DFUSE_EQ_BACKOFF_MAX_US; + + return backoff; +} + +static void +dfuse_eq_empty_poll(struct dfuse_eq *eqt) +{ + uint32_t empty_polls = atomic_load_relaxed(&eqt->de_empty_polls); + + if (empty_polls < UINT32_MAX) + atomic_store_relaxed(&eqt->de_empty_polls, empty_polls + 1); +} + /* Async progress thread. * * A number of threads are created at launch, each thread having its own event queue with a * semaphore to wakeup, posted for each entry added to the event queue and once for shutdown. * When there are no entries on the eq then the thread will yield in the semaphore, when there - * are pending events it'll spin in eq_poll() for completion. All pending events should be + * are pending events it'll poll for completion. Repeated empty polls use a bounded timeout in + * daos_eq_poll() so CRT progress continues without busy polling. All pending events should be * completed before thread exit, should exit be called with pending events. */ static void * @@ -29,6 +62,7 @@ dfuse_progress_thread(void *arg) while (1) { int rc; int i; + int64_t poll_timeout; for (i = 0; i < to_consume; i++) { cont: @@ -55,8 +89,11 @@ dfuse_progress_thread(void *arg) return NULL; } - rc = daos_eq_poll(eqt->de_eq, 1, DAOS_EQ_NOWAIT, 128, &dev[0]); + poll_timeout = dfuse_eq_backoff_us(atomic_load_relaxed(&eqt->de_empty_polls)); + rc = daos_eq_poll(eqt->de_eq, 1, poll_timeout, 128, &dev[0]); if (rc >= 1) { + atomic_store_relaxed(&eqt->de_empty_polls, 0); + for (i = 0; i < rc; i++) { struct dfuse_event *ev; @@ -67,8 +104,10 @@ dfuse_progress_thread(void *arg) } else if (rc < 0) { DFUSE_TRA_WARNING(eqt, "Error from daos_eq_poll, " DF_RC, DP_RC(rc)); to_consume = 0; + dfuse_eq_empty_poll(eqt); } else { to_consume = 0; + dfuse_eq_empty_poll(eqt); } } return NULL; @@ -1207,6 +1246,7 @@ dfuse_fs_init(struct dfuse_info *dfuse_info) eqt->de_handle = dfuse_info; + atomic_store_relaxed(&eqt->de_empty_polls, 0); DFUSE_TRA_UP(eqt, dfuse_info, "event_queue"); /* Create the semaphore before the eq as there's no way to check if sem_init() diff --git a/src/client/dfuse/ops/fgetattr.c b/src/client/dfuse/ops/fgetattr.c index 6fdee73515c..20a1f9893e2 100644 --- a/src/client/dfuse/ops/fgetattr.c +++ b/src/client/dfuse/ops/fgetattr.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -58,7 +59,7 @@ dfuse_cb_getattr(fuse_req_t req, struct dfuse_inode_entry *ie) if (rc != 0) D_GOTO(ev, rc); - sem_post(&eqt->de_sem); + dfuse_eq_wakeup(eqt); return; ev: diff --git a/src/client/dfuse/ops/read.c b/src/client/dfuse/ops/read.c index f2f6ff8fcdd..9a55a8e08f6 100644 --- a/src/client/dfuse/ops/read.c +++ b/src/client/dfuse/ops/read.c @@ -352,7 +352,7 @@ chunk_fetch(fuse_req_t req, struct dfuse_obj_hdl *oh, struct read_chunk_data *cd goto err; /* Send a message to the async thread to wake it up and poll for events */ - sem_post(&eqt->de_sem); + dfuse_eq_wakeup(eqt); /* Now ensure there are more descriptors for the next request */ d_slab_restock(eqt->de_read_slab); @@ -556,7 +556,7 @@ dfuse_cb_read(fuse_req_t req, fuse_ino_t ino, size_t len, off_t position, struct } /* Send a message to the async thread to wake it up and poll for events */ - sem_post(&eqt->de_sem); + dfuse_eq_wakeup(eqt); /* Now ensure there are more descriptors for the next request */ d_slab_restock(eqt->de_read_slab); @@ -663,7 +663,7 @@ dfuse_pre_read(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie) goto err; /* Send a message to the async thread to wake it up and poll for events */ - sem_post(&eqt->de_sem); + dfuse_eq_wakeup(eqt); /* Now ensure there are more descriptors for the next request */ d_slab_restock(eqt->de_pre_read_slab); diff --git a/src/client/dfuse/ops/write.c b/src/client/dfuse/ops/write.c index 8339c05a9b4..dc0bae1b34f 100644 --- a/src/client/dfuse/ops/write.c +++ b/src/client/dfuse/ops/write.c @@ -133,7 +133,7 @@ dfuse_cb_write(fuse_req_t req, fuse_ino_t ino, struct fuse_bufvec *bufv, off_t p DFUSE_REPLY_WRITE(oh, req, len); /* Send a message to the async thread to wake it up and poll for events */ - sem_post(&eqt->de_sem); + dfuse_eq_wakeup(eqt); /* Now ensure there are more descriptors for the next request */ d_slab_restock(eqt->de_write_slab); diff --git a/src/control/cmd/ddb/command_completers_test.go b/src/control/cmd/ddb/command_completers_test.go index 834fbfb048a..70f7b131d54 100644 --- a/src/control/cmd/ddb/command_completers_test.go +++ b/src/control/cmd/ddb/command_completers_test.go @@ -1,3 +1,9 @@ +// +// (C) Copyright 2026 Hewlett Packard Enterprise Development LP +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + package main import ( @@ -18,7 +24,7 @@ func createFile(t *testing.T, filePath string) { fd, err := os.Create(filePath) if err != nil { - t.Fatalf("Failed to create test vos file %s: %v", filePath, err) + t.Fatalf("failed to create test vos file %s: %v", filePath, err) } fd.Close() } @@ -27,14 +33,14 @@ func createDirAll(t *testing.T, dirPath string) { t.Helper() if err := os.MkdirAll(dirPath, 0755); err != nil { - t.Fatalf("Failed to create test pool directory %s: %v", dirPath, err) + t.Fatalf("failed to create test pool directory %s: %v", dirPath, err) } } -func testSetup(t *testing.T) (tmpDir string, teardown func()) { +func testSetup(t *testing.T) string { t.Helper() - tmpDir, teardown = test.CreateTestDir(t) + tmpDir := t.TempDir() for _, dir := range testPoolDirs { createDirAll(t, filepath.Join(tmpDir, dir)) @@ -51,12 +57,11 @@ func testSetup(t *testing.T) (tmpDir string, teardown func()) { createDirAll(t, filepath.Join(tmpDir, "bar", "baz")) createFile(t, filepath.Join(tmpDir, "bar", "baz", "no_vos")) - return + return tmpDir } -func TestListVosFiles(t *testing.T) { - tmpDir, teardown := testSetup(t) - t.Cleanup(teardown) +func TestDdb_listVosFiles(t *testing.T) { + tmpDir := testSetup(t) for name, tc := range map[string]struct { args string @@ -118,12 +123,12 @@ func TestListVosFiles(t *testing.T) { } { t.Run(name, func(t *testing.T) { results := listVosFiles(tc.args) - test.AssertStringsEqual(t, tc.expRes, results, "listDirVos results do not match expected") + test.AssertStringsEqual(t, tc.expRes, results, "unexpected listVosFiles results") }) } } -func TestFilterSuggestions(t *testing.T) { +func TestDdb_filterSuggestions(t *testing.T) { // The test cases are designed to cover various prefix scenarios. // It should notably cover the case where the prefix is a single character that matches the // second character of a suggestion, which is a special case in the appendSuggestion @@ -169,7 +174,7 @@ func TestFilterSuggestions(t *testing.T) { } { t.Run(name, func(t *testing.T) { results := filterSuggestions(tc.prefix, initialSuggestions, additionalSuggestions) - test.AssertStringsEqual(t, tc.expRes, results, "filterSuggestions results do not match expected") + test.AssertStringsEqual(t, tc.expRes, results, "unexpected filterSuggestions results") }) } } diff --git a/src/control/cmd/ddb/ddb_commands_test.go b/src/control/cmd/ddb/ddb_commands_test.go new file mode 100644 index 00000000000..a01c5c16d05 --- /dev/null +++ b/src/control/cmd/ddb/ddb_commands_test.go @@ -0,0 +1,395 @@ +// +// (C) Copyright 2026 Hewlett Packard Enterprise Development LP +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +//go:build test_stubs + +package main + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/daos-stack/daos/src/control/common/test" +) + +func TestDdb_HelpCmds(t *testing.T) { + for name, tc := range map[string]struct { + cmdStr string + helpSubStr string + }{ + "help for 'ls' command": { + cmdStr: "ls", + helpSubStr: "Usage:\n ls [flags] [path]\n", + }, + "help for 'open' command": { + cmdStr: "open", + helpSubStr: "Usage:\n open [flags] path\n", + }, + // TODO(follow-up PR): Add help tests for the remaining commands. + } { + t.Run(name, func(t *testing.T) { + ctx := newTestContext(t) + + // Create a temporary config file with the help command + tmpCfgDir := t.TempDir() + tmpCfgFile := path.Join(tmpCfgDir, "ddb-cmd_file.txt") + if err := os.WriteFile(tmpCfgFile, []byte(fmt.Sprintf("%s --help", tc.cmdStr)), 0644); err != nil { + t.Fatalf("failed to write temp config file: %v", err) + } + + // Run the help command with a command file + args := test.JoinArgs(nil, "--cmd_file="+tmpCfgFile) + stdoutCmdFile, err := captureStdout(func() error { + return runDdb(ctx, args) + }) + if err != nil { + t.Fatalf("unexpected error when running '%s --help' via command file: want nil, got %v", tc.cmdStr, err) + } + test.AssertTrue(t, strings.Contains(stdoutCmdFile, tc.helpSubStr), + fmt.Sprintf("expected stdout to contain %q: got\n%s", tc.helpSubStr, stdoutCmdFile)) + + // Run the help command with a command line + args = test.JoinArgs(nil, tc.cmdStr, "--help") + stdoutCmdLine, err := captureStdout(func() error { + return runDdb(ctx, args) + }) + if err != nil { + t.Fatalf("unexpected error when running '%s --help' via command line: want nil, got %v", tc.cmdStr, err) + } + test.AssertTrue(t, strings.Contains(stdoutCmdLine, tc.helpSubStr), + fmt.Sprintf("expected stdout to contain %q: got\n%s", tc.helpSubStr, stdoutCmdLine)) + + // Compare command line and command file outputs + test.AssertEqual(t, stdoutCmdFile, stdoutCmdLine, + fmt.Sprintf("unexpected help output mismatch between command file and command line for '%s'", tc.cmdStr)) + }) + } +} + +func TestDdb_Cmds(t *testing.T) { + // Helper factories for command stub functions — declared here to avoid + // anonymous functions nested inside the test table. + + lsFnChecking := func(t *testing.T, wantPath string, wantRecursive, wantDetails bool) func(string, bool, bool) error { + return func(path string, recursive, details bool) error { + fmt.Println("ls called") + test.CmpAny(t, "path", wantPath, path) + test.CmpAny(t, "recursive", wantRecursive, recursive) + test.CmpAny(t, "details", wantDetails, details) + return nil + } + } + + openFnChecking := func(t *testing.T, wantPath, wantDbPath string, wantWriteMode bool) func(string, string, bool) error { + return func(path, dbPath string, writeMode bool) error { + fmt.Println("open called") + test.CmpAny(t, "path", wantPath, path) + test.CmpAny(t, "db_path", wantDbPath, dbPath) + test.CmpAny(t, "write_mode", wantWriteMode, writeMode) + return nil + } + } + + featureFnCheckingShow := func(t *testing.T, wantShow bool) func(string, string, string, string, bool) error { + return func(_, _, _, _ string, show bool) error { + fmt.Println("feature called") + test.CmpAny(t, "show", wantShow, show) + return nil + } + } + + dtxAggrFnChecking := func(t *testing.T, wantPath string, wantCmtTime uint64, wantCmtDate string) func(string, uint64, string) error { + return func(path string, cmtTime uint64, cmtDate string) error { + fmt.Println("dtx_aggr called") + test.CmpAny(t, "path", wantPath, path) + test.CmpAny(t, "cmtTime", wantCmtTime, cmtTime) + test.CmpAny(t, "cmtDate", wantCmtDate, cmtDate) + return nil + } + } + + for name, tc := range map[string]struct { + args []string + setup func(*testing.T) + expStdout []string + expErr error + // skipCmdLine skips the command-line sub-test with a message. Use when + // a flag is shared between the CLI layer and the grumble command: go-flags + // consumes it before grumble can see it, making a clean command-line test + // impossible for that particular flag. + skipCmdLine string + }{ + "ls invalid options": { + args: []string{"ls", "--bar"}, + expErr: ddbTestErr("invalid flag: --bar"), + }, + "ls default": { + args: []string{"ls"}, + setup: func(t *testing.T) { + ddb_run_ls_Fn = lsFnChecking(t, "", false, false) + }, + expStdout: []string{"ls called"}, + }, + "ls path": { + args: []string{"ls", "/[0]"}, + setup: func(t *testing.T) { + ddb_run_ls_Fn = lsFnChecking(t, "/[0]", false, false) + }, + expStdout: []string{"ls called"}, + }, + "ls long recursive opt": { + args: []string{"ls", "--recursive"}, + setup: func(t *testing.T) { + ddb_run_ls_Fn = lsFnChecking(t, "", true, false) + }, + expStdout: []string{"ls called"}, + }, + "ls short details opt": { + args: []string{"ls", "-d"}, + setup: func(t *testing.T) { + ddb_run_ls_Fn = lsFnChecking(t, "", false, true) + }, + expStdout: []string{"ls called"}, + }, + "ls details long opt": { + args: []string{"ls", "--details"}, + setup: func(t *testing.T) { + ddb_run_ls_Fn = lsFnChecking(t, "", false, true) + }, + expStdout: []string{"ls called"}, + }, + + // --- open command --- + // Note: the -w/--write_mode and -p/--db_path flags of the grumble 'open' + // command share names with CLI-level flags that are consumed by go-flags + // before reaching grumble in command-line mode. The command-line test for + // those flags would silently test wrong values. They are correctly exercised + // in command-file mode; see TestRun for CLI-level flag coverage. + "open default": { + args: []string{"open", "/path/to/vos-0"}, + setup: func(t *testing.T) { + ddb_run_open_Fn = openFnChecking(t, "/path/to/vos-0", "", false) + }, + expStdout: []string{"open called"}, + }, + "open write mode": { + args: []string{"open", "-w", "/path/to/vos-0"}, + skipCmdLine: "-w is consumed by the CLI write_mode flag before reaching grumble", + setup: func(t *testing.T) { + ddb_run_open_Fn = openFnChecking(t, "/path/to/vos-0", "", true) + }, + expStdout: []string{"open called"}, + }, + "open with db path": { + args: []string{"open", "-p", "/sysdb", "/path/to/vos-0"}, + skipCmdLine: "-p is consumed by the CLI db_path flag before reaching grumble", + setup: func(t *testing.T) { + ddb_run_open_Fn = openFnChecking(t, "/path/to/vos-0", "/sysdb", false) + }, + expStdout: []string{"open called"}, + }, + + // --- feature command --- + // feature --show: verifies the show flag is forwarded to the C layer. + "feature show": { + args: []string{"feature", "--show"}, + setup: func(t *testing.T) { + ddb_run_feature_Fn = featureFnCheckingShow(t, true) + }, + expStdout: []string{"feature called"}, + }, + // feature --enable: verifies that the enable string reaches ddb_feature_string2flags. + "feature enable": { + args: []string{"feature", "--enable=myflag"}, + setup: func(t *testing.T) { + var capturedFlag string + ddb_feature_string2flags_Fn = func(s string) (uint64, uint64, error) { + capturedFlag = s + return 0, 0, nil + } + ddb_run_feature_Fn = func(path, dbPath, enable, disable string, show bool) error { + fmt.Println("feature called") + test.CmpAny(t, "enable flag string", "myflag", capturedFlag) + return nil + } + }, + expStdout: []string{"feature called"}, + }, + // feature --disable: verifies that the disable string reaches ddb_feature_string2flags. + "feature disable": { + args: []string{"feature", "--disable=otherflag"}, + setup: func(t *testing.T) { + var capturedFlag string + ddb_feature_string2flags_Fn = func(s string) (uint64, uint64, error) { + capturedFlag = s + return 0, 0, nil + } + ddb_run_feature_Fn = func(path, dbPath, enable, disable string, show bool) error { + fmt.Println("feature called") + test.CmpAny(t, "disable flag string", "otherflag", capturedFlag) + return nil + } + }, + expStdout: []string{"feature called"}, + }, + + // --- dtx_aggr command --- + // The Run handler in ddb_commands.go enforces that exactly one of --cmt_time or + // --cmt_date is provided. These tests exercise that Go-layer validation. + "dtx_aggr both cmt_time and cmt_date": { + args: []string{"dtx_aggr", "--cmt_time=0", "--cmt_date=2024-01-01"}, + expErr: ddbTestErr("mutually exclusive"), + }, + "dtx_aggr neither cmt_time nor cmt_date": { + args: []string{"dtx_aggr"}, + expErr: ddbTestErr("has to be defined"), + }, + "dtx_aggr cmt_time": { + args: []string{"dtx_aggr", "--cmt_time=1000"}, + setup: func(t *testing.T) { + ddb_run_dtx_aggr_Fn = dtxAggrFnChecking(t, "", 1000, "") + }, + expStdout: []string{"dtx_aggr called"}, + }, + "dtx_aggr cmt_date": { + args: []string{"dtx_aggr", "--cmt_date=2024-01-01"}, + setup: func(t *testing.T) { + ddb_run_dtx_aggr_Fn = dtxAggrFnChecking(t, "", 0, "2024-01-01") + }, + expStdout: []string{"dtx_aggr called"}, + }, + "dtx_aggr with path": { + args: []string{"dtx_aggr", "--cmt_time=0", "[0]"}, + setup: func(t *testing.T) { + ddb_run_dtx_aggr_Fn = dtxAggrFnChecking(t, "[0]", 0, "") + }, + expStdout: []string{"dtx_aggr called"}, + }, + + // --- close command --- + "close": { + args: []string{"close"}, + setup: func(t *testing.T) { + ddb_run_close_Fn = func() error { + fmt.Println("close called") + return nil + } + }, + expStdout: []string{"close called"}, + }, + + // --- version command --- + "version": { + args: []string{"version"}, + setup: func(t *testing.T) { + ddb_run_version_Fn = func() error { + fmt.Println("version called") + return nil + } + }, + expStdout: []string{"version called"}, + }, + + // TODO(follow-up PR): Add TestCmds cases for the remaining commands. + // Each new test case follows the same pattern as the cases above: set the + // corresponding ddb_run__Fn hook in setup() to verify argument passing, + // then add the case to this table. + // Commands still to be covered: superblock_dump, value_dump, rm, + // value_load, ilog_dump, ilog_commit, ilog_clear, dtx_dump, dtx_cmt_clear, + // smd_sync, vea_dump, vea_update, dtx_act_commit, dtx_act_abort, rm_pool, + // dtx_act_discard_invalid, dev_list, dev_replace, dtx_stat, prov_mem. + } { + t.Run(name, func(t *testing.T) { + checkCmd := func(t *testing.T, stdout string, err error) { + t.Helper() + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + for _, msg := range tc.expStdout { + test.AssertTrue(t, strings.Contains(stdout, msg), + fmt.Sprintf("expected stdout to contain %q: got\n%s", msg, stdout)) + } + } + + t.Run("command-line", func(t *testing.T) { + if tc.skipCmdLine != "" { + t.Skipf("skipping command-line mode: %s", tc.skipCmdLine) + } + ctx := newTestContext(t) + if tc.setup != nil { + tc.setup(t) + } + stdout, err := captureStdout(func() error { + return runDdb(ctx, tc.args) + }) + checkCmd(t, stdout, err) + }) + + t.Run("command-file", func(t *testing.T) { + tmpDir := t.TempDir() + cmdFile := filepath.Join(tmpDir, "cmds.txt") + cmdLine := strings.Join(tc.args, " ") + if err := os.WriteFile(cmdFile, []byte(cmdLine), 0644); err != nil { + t.Fatalf("failed to write command file: %v", err) + } + ctx := newTestContext(t) + if tc.setup != nil { + tc.setup(t) + } + stdout, err := captureStdout(func() error { + return runDdb(ctx, []string{"--cmd_file=" + cmdFile}) + }) + checkCmd(t, stdout, err) + }) + }) + } +} + +func TestDdb_ManPage(t *testing.T) { + // Expected sections and commands present in every man page rendering. + expSections := []string{ + manArgsHeader, + manCmdsHeader, + manPathSection[:20], + manMdOnSsdSection[:20], + manLoggingSection[:20], + ".B ls\n", + ".B open\n", + } + + // manpage to stdout: must contain all section headers and known commands. + ctx := newTestContext(t) + stdout, err := captureStdout(func() error { + return runDdb(ctx, []string{"manpage"}) + }) + test.CmpErr(t, nil, err) + test.AssertStringContains(t, stdout, expSections...) + + // --output flag: man page is written to a file, stdout is empty. + tmpDir := t.TempDir() + outFile := filepath.Join(tmpDir, "ddb.groff") + + ctx = newTestContext(t) + stdout, err = captureStdout(func() error { + return runDdb(ctx, []string{"manpage", "--output=" + outFile}) + }) + if err != nil { + t.Fatalf("unexpected error when running 'manpage --output': want nil, got %v", err) + } + test.AssertTrue(t, stdout == "", + fmt.Sprintf("expected empty stdout when --output is set: got\n%s", stdout)) + + content, err := os.ReadFile(outFile) + if err != nil { + t.Fatalf("failed to read output file: %v", err) + } + test.AssertStringContains(t, string(content), expSections...) +} diff --git a/src/control/cmd/ddb/main.go b/src/control/cmd/ddb/main.go index 67cc415069b..97e2603b76a 100644 --- a/src/control/cmd/ddb/main.go +++ b/src/control/cmd/ddb/main.go @@ -365,37 +365,45 @@ func run(ctx *DdbContext, log *logging.LeveledLogger, opts cliOptions, parser *f return result } -func main() { - // Set the traceback level such that a crash results in - // a coredump (when ulimit -c is set appropriately). - debug.SetTraceback("crash") - - // Must be called before any write to stdout. - if err := logging.DisableCStdoutBuffering(); err != nil { - exitWithError(err) - } - - ctx := &DdbContext{} - opts, parser, err := parseOpts(os.Args[1:], ctx) +// runDdb contains the core ddb execution logic. It is separate from main() so +// that it can be tested without triggering os.Exit. main() handles only +// OS-level setup (traceback, stdout buffering) and calls exitWithError on +// failure; runDdb returns errors instead. +func runDdb(ctx *DdbContext, args []string) error { + opts, parser, err := parseOpts(args, ctx) if errors.Is(err, errHelpRequested) { - return + return nil } if err != nil { - exitWithError(err) + return err } if opts.Version { fmt.Printf("ddb version %s\n", build.DaosVersion) - return + return nil } - log, err := newLogger(opts) - if err != nil { - exitWithError(errors.Wrap(err, loggerInitErr)) + var log *logging.LeveledLogger + if log, err = newLogger(opts); err != nil { + return errors.Wrap(err, loggerInitErr) } log.Debug("Logging facilities initialized") - if err = run(ctx, log, opts, parser); err != nil { + return run(ctx, log, opts, parser) +} + +func main() { + // Set the traceback level such that a crash results in + // a coredump (when ulimit -c is set appropriately). + debug.SetTraceback("crash") + + // Must be called before any write to stdout. + if err := logging.DisableCStdoutBuffering(); err != nil { + exitWithError(err) + } + + ctx := &DdbContext{} + if err := runDdb(ctx, os.Args[1:]); err != nil { exitWithError(err) } } diff --git a/src/control/cmd/ddb/main_test.go b/src/control/cmd/ddb/main_test.go new file mode 100644 index 00000000000..03c761b94b8 --- /dev/null +++ b/src/control/cmd/ddb/main_test.go @@ -0,0 +1,616 @@ +// +// (C) Copyright 2026 Hewlett Packard Enterprise Development LP +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +//go:build test_stubs + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/daos-stack/daos/src/control/common/test" + "github.com/daos-stack/daos/src/control/logging" + "github.com/daos-stack/daos/src/control/server/engine" +) + +func TestDdb_parseOpts(t *testing.T) { + for name, tc := range map[string]struct { + args []string + checkFunc func(opts *cliOptions) error + expStdout []string + expErr error + }{ + "General help message": { + args: []string{"--help"}, + expStdout: []string{ + "Usage:\n ddb [OPTIONS] [ddb_command] [ddb_command_args...]\n", + "VOS Paths:\n", + "Available Commands:\n", + }, + expErr: errHelpRequested, + }, + "General help message with opt": { + args: []string{"-w", "--help"}, + expStdout: []string{ + "Usage:\n ddb [OPTIONS] [ddb_command] [ddb_command_args...]\n", + "VOS Paths:\n", + "Available Commands:\n", + }, + expErr: errHelpRequested, + }, + "Unknown commands with help": { + args: []string{"foo", "--help"}, + expErr: errUnknownCmd, + }, + "Unknown commands with help and opt": { + args: []string{"-w", "foo", "--help"}, + expErr: errUnknownCmd, + }, + "Default option values": { + args: []string{"ls", "-d", "-r"}, + checkFunc: func(opts *cliOptions) error { + if opts.Debug != "" { + return fmt.Errorf("expected Debug to be empty, got %q", opts.Debug) + } + if opts.WriteMode { + return fmt.Errorf("expected WriteMode to be false") + } + if opts.CmdFile != "" { + return fmt.Errorf("expected CmdFile to be empty") + } + if opts.SysdbPath != "" { + return fmt.Errorf("expected SysdbPath to be empty") + } + if opts.VosPath != "" { + return fmt.Errorf("expected VosPath to be empty") + } + if opts.Args.RunCmd != "ls" { + return fmt.Errorf("expected RunCmd to be 'ls', got %q", opts.Args.RunCmd) + } + if opts.Args.RunCmdArgs[0] != "-d" { + return fmt.Errorf("expected first RunCmdArgs to be '-d', got %q", opts.Args.RunCmdArgs[0]) + } + if opts.Args.RunCmdArgs[1] != "-r" { + return fmt.Errorf("expected second RunCmdArgs to be '-r', got %q", opts.Args.RunCmdArgs[1]) + } + return nil + }, + }, + "Short miss vos path error": { + args: []string{"-p", "/foo", "ls"}, + expErr: ddbTestErr(vosPathMissErr), + }, + "Long miss vos path error": { + args: []string{"--db_path=/bar", "ls"}, + expErr: ddbTestErr(vosPathMissErr), + }, + "Short cmd args error": { + args: []string{"-f", "/foo/bar.cmd", "ls"}, + expErr: ddbTestErr(runCmdArgsErr), + }, + "Long cmd args error": { + args: []string{"--cmd_file=/foo/bar.cmd", "ls"}, + expErr: ddbTestErr(runCmdArgsErr), + }, + "Short vos path miss error": { + args: []string{"-p", "/foo"}, + expErr: ddbTestErr(vosPathMissErr), + }, + "Long vos path miss error": { + args: []string{"--db_path=/foo"}, + expErr: ddbTestErr(vosPathMissErr), + }, + "Long debug option": { + args: []string{"--debug=DEBUG", "ls"}, + checkFunc: func(opts *cliOptions) error { + if opts.Debug != "DEBUG" { + return fmt.Errorf("expected Debug to be 'DEBUG', got %q", opts.Debug) + } + return nil + }, + }, + "Short write option": { + args: []string{"-w", "ls"}, + checkFunc: func(opts *cliOptions) error { + if !opts.WriteMode { + return fmt.Errorf("expected WriteMode to be true") + } + return nil + }, + }, + "Long write option": { + args: []string{"--write_mode", "ls"}, + checkFunc: func(opts *cliOptions) error { + if !opts.WriteMode { + return fmt.Errorf("expected WriteMode to be true") + } + return nil + }, + }, + "Short vos path option": { + args: []string{"-s", "/foo/vos-0", "ls"}, + checkFunc: func(opts *cliOptions) error { + if opts.VosPath != "/foo/vos-0" { + return fmt.Errorf("expected VosPath to be '/foo/vos-0', got %q", opts.VosPath) + } + return nil + }, + }, + "Long vos path option": { + args: []string{"--vos_path=/foo/vos-0", "ls"}, + checkFunc: func(opts *cliOptions) error { + if opts.VosPath != "/foo/vos-0" { + return fmt.Errorf("expected VosPath to be '/foo/vos-0', got %q", opts.VosPath) + } + return nil + }, + }, + "Short db path option": { + args: []string{"-s", "/foo/vos-0", "-p", "/bar", "ls"}, + checkFunc: func(opts *cliOptions) error { + if opts.VosPath != "/foo/vos-0" { + return fmt.Errorf("expected VosPath to be '/foo/vos-0', got %q", opts.VosPath) + } + if opts.SysdbPath != "/bar" { + return fmt.Errorf("expected SysdbPath to be '/bar', got %q", opts.SysdbPath) + } + return nil + }, + }, + "Long db path option": { + args: []string{"--vos_path=/foo/vos-0", "--db_path=/bar", "ls"}, + checkFunc: func(opts *cliOptions) error { + if opts.VosPath != "/foo/vos-0" { + return fmt.Errorf("expected VosPath to be '/foo/vos-0', got %q", opts.VosPath) + } + if opts.SysdbPath != "/bar" { + return fmt.Errorf("expected SysdbPath to be '/bar', got %q", opts.SysdbPath) + } + return nil + }, + }, + "Short version option": { + args: []string{"-v"}, + checkFunc: func(opts *cliOptions) error { + if !opts.Version { + return fmt.Errorf("expected Version to be true") + } + return nil + }, + }, + "Long version option": { + args: []string{"--version"}, + checkFunc: func(opts *cliOptions) error { + if !opts.Version { + return fmt.Errorf("expected Version to be true") + } + return nil + }, + }, + } { + t.Run(name, func(t *testing.T) { + ctx := newTestContext(t) + + opts, stdout, err := runCmdToStdout(ctx, tc.args) + test.CmpErr(t, tc.expErr, err) + + for _, msg := range tc.expStdout { + test.AssertTrue(t, strings.Contains(stdout, msg), + fmt.Sprintf("expected stdout to contain %q: got\n%s", msg, stdout)) + } + + if tc.expErr != nil { + return + } + + if tc.checkFunc != nil { + if err := tc.checkFunc(&opts); err != nil { + t.Fatal(err) + } + } + }) + } +} + +// openFnChecking returns a ddb_run_open_Fn stub that verifies the vos path and +// sysdb path arguments passed to the open call. called is set to true when the +// stub is invoked, allowing the caller to assert that open was called. +func openFnChecking(t *testing.T, wantPath, wantDbPath string, called *bool) func(string, string, bool) error { + return func(path, dbPath string, _ bool) error { + *called = true + test.CmpAny(t, "vos path", wantPath, path) + test.CmpAny(t, "sysdb path", wantDbPath, dbPath) + return nil + } +} + +// openFnCheckingWriteMode returns a ddb_run_open_Fn stub that verifies the +// write_mode argument passed to the open call. called is set to true when the +// stub is invoked, allowing the caller to assert that open was called. +func openFnCheckingWriteMode(t *testing.T, wantWriteMode bool, called *bool) func(string, string, bool) error { + return func(_ string, _ string, writeMode bool) error { + *called = true + test.CmpAny(t, "write_mode", wantWriteMode, writeMode) + return nil + } +} + +// openFnMustNotBeCalled is a ddb_run_open_Fn stub that fails the test if +// the open function is called at all (used to verify no-auto-open behavior). +func openFnMustNotBeCalled(_ string, _ string, _ bool) error { + return fmt.Errorf("open should not have been called") +} + +// openFnAllowedOnce returns a ddb_run_open_Fn stub that allows the open +// function to be called exactly once (used to verify the 'open' command +// itself calls open but the CLI does not pre-open). +func openFnAllowedOnce() func(string, string, bool) error { + count := 0 + return func(_ string, _ string, _ bool) error { + count++ + if count > 1 { + return fmt.Errorf("open pre-opened by CLI (called %d times)", count) + } + return nil + } +} + +// openFnFailing is a ddb_run_open_Fn stub that always returns an error, +// used to verify that open failures are propagated correctly. +func openFnFailing(_ string, _ string, _ bool) error { + return fmt.Errorf("simulated open failure") +} + +// TestRun covers the non-interactive command-line execution paths of runDdb(). +// Interactive mode is intentionally not tested: it delegates entirely to +// grumble's app.Run(), which requires a real terminal and is hard to automate. +func TestDdb_runDdb(t *testing.T) { + for name, tc := range map[string]struct { + args []string + setup func(*testing.T) + expStdout []string + expErr error + }{ + "Version output": { + args: []string{"-v"}, + expStdout: []string{"ddb version"}, + }, + "Long version output": { + args: []string{"--version"}, + expStdout: []string{"ddb version"}, + }, + "Unknown command": { + args: []string{"foo"}, + expErr: errUnknownCmd, + }, + "Unknown command with write option": { + args: []string{"-w", "foo"}, + expErr: errUnknownCmd, + }, + "Open called with short vos path and db path": { + args: []string{"-s", "/foo/vos-0", "-p", "/bar", "ls"}, + setup: func(t *testing.T) { + var called bool + ddb_run_open_Fn = openFnChecking(t, "/foo/vos-0", "/bar", &called) + t.Cleanup(func() { test.AssertTrue(t, called, "open was not called") }) + }, + }, + "Open called with long vos path and db path": { + args: []string{"--vos_path=/foo/vos-0", "--db_path=/bar", "ls"}, + setup: func(t *testing.T) { + var called bool + ddb_run_open_Fn = openFnChecking(t, "/foo/vos-0", "/bar", &called) + t.Cleanup(func() { test.AssertTrue(t, called, "open was not called") }) + }, + }, + "Open called with write mode": { + args: []string{"-w", "-s", "/foo/vos-0", "ls"}, + setup: func(t *testing.T) { + var called bool + ddb_run_open_Fn = openFnCheckingWriteMode(t, true, &called) + t.Cleanup(func() { test.AssertTrue(t, called, "open was not called") }) + }, + }, + "No auto-open for feature command": { + // noAutoOpen is keyed on opts.Args.RunCmd which is empty in command-file + // mode, so this case only applies to command-line mode. + args: []string{"-s", "/foo/vos-0", "feature", "--show"}, + setup: func(t *testing.T) { + ddb_run_open_Fn = openFnMustNotBeCalled + }, + }, + "No auto-open for open command": { + // The CLI should NOT pre-open when the 'open' command is issued; only the + // command itself should call ctx.Open (exactly once). + // Only valid for command-line mode (see note above). + args: []string{"-s", "/foo/vos-0", "open", "/foo/vos-0"}, + setup: func(t *testing.T) { + ddb_run_open_Fn = openFnAllowedOnce() + }, + }, + "No auto-open for smd_sync": { + args: []string{"-s", "/foo/vos-0", "smd_sync"}, + setup: func(t *testing.T) { + ddb_run_open_Fn = openFnMustNotBeCalled + }, + }, + "Init failure": { + args: []string{"ls"}, + expErr: ddbTestErr(ctxInitErr), + setup: func(t *testing.T) { + ddb_init_RC = -1 // non-zero triggers DER_UNKNOWN + }, + }, + "Open failure": { + args: []string{"-s", "/foo/vos-0", "ls"}, + expErr: ddbTestErr("Error opening VOS path"), + setup: func(t *testing.T) { + ddb_run_open_Fn = openFnFailing + }, + }, + } { + t.Run(name, func(t *testing.T) { + ctx := newTestContext(t) + if tc.setup != nil { + tc.setup(t) + } + stdout, err := captureStdout(func() error { + return runDdb(ctx, tc.args) + }) + test.CmpErr(t, tc.expErr, err) + if tc.expErr == nil { + test.AssertStringContains(t, stdout, tc.expStdout...) + } + }) + } +} + +// TestRunCommandFile covers command-file execution paths of runDdb(). +// It exercises the same behavior as TestRun for cases where command-file +// mode produces the same result as command-line mode. +func TestDdb_runDdbCommandFile(t *testing.T) { + for name, tc := range map[string]struct { + flags []string // CLI flags (before --cmd_file) + cmdLine string // line written to the temporary command file + setup func(*testing.T) + expStdout []string + expErr error + }{ + "Unknown command": { + cmdLine: "foo", + expErr: errUnknownCmd, + }, + "Unknown command with write option": { + flags: []string{"-w"}, + cmdLine: "foo", + expErr: errUnknownCmd, + }, + "Open called with short vos path and db path": { + flags: []string{"-s", "/foo/vos-0", "-p", "/bar"}, + cmdLine: "ls", + setup: func(t *testing.T) { + var called bool + ddb_run_open_Fn = openFnChecking(t, "/foo/vos-0", "/bar", &called) + t.Cleanup(func() { test.AssertTrue(t, called, "open was not called") }) + }, + }, + "Open called with long vos path and db path": { + flags: []string{"--vos_path=/foo/vos-0", "--db_path=/bar"}, + cmdLine: "ls", + setup: func(t *testing.T) { + var called bool + ddb_run_open_Fn = openFnChecking(t, "/foo/vos-0", "/bar", &called) + t.Cleanup(func() { test.AssertTrue(t, called, "open was not called") }) + }, + }, + "Open called with write mode": { + flags: []string{"-w", "-s", "/foo/vos-0"}, + cmdLine: "ls", + setup: func(t *testing.T) { + var called bool + ddb_run_open_Fn = openFnCheckingWriteMode(t, true, &called) + t.Cleanup(func() { test.AssertTrue(t, called, "open was not called") }) + }, + }, + } { + t.Run(name, func(t *testing.T) { + tmpDir := t.TempDir() + cmdFile := filepath.Join(tmpDir, "cmds.txt") + if err := os.WriteFile(cmdFile, []byte(tc.cmdLine), 0644); err != nil { + t.Fatalf("failed to write command file: %v", err) + } + ctx := newTestContext(t) + if tc.setup != nil { + tc.setup(t) + } + args := append(tc.flags, "--cmd_file="+cmdFile) + stdout, err := captureStdout(func() error { + return runDdb(ctx, args) + }) + test.CmpErr(t, tc.expErr, err) + if tc.expErr == nil { + test.AssertStringContains(t, stdout, tc.expStdout...) + } + }) + } +} + +// TestRunMultiLineCommandFile verifies that runFileCmds iterates over multiple +// lines in a command file, executing each one in sequence. +func TestDdb_runFileCmds(t *testing.T) { + ctx := newTestContext(t) + var lsCalled, versionCalled bool + ddb_run_ls_Fn = func(path string, recursive bool, details bool) error { + lsCalled = true + return nil + } + ddb_run_version_Fn = func() error { + versionCalled = true + return nil + } + + tmpDir := t.TempDir() + cmdFile := filepath.Join(tmpDir, "cmds.txt") + if err := os.WriteFile(cmdFile, []byte("ls\nversion\n"), 0644); err != nil { + t.Fatalf("failed to write command file: %v", err) + } + + if err := runDdb(ctx, []string{"--cmd_file=" + cmdFile}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + test.AssertTrue(t, lsCalled, "ls was not called") + test.AssertTrue(t, versionCalled, "version was not called") +} + +func TestDdb_strToLogLevels(t *testing.T) { + for name, tc := range map[string]struct { + input string + expCliLevel logging.LogLevel + expEngineLevel engine.LogLevel + expErr bool + }{ + "TRACE": {input: "TRACE", expCliLevel: logging.LogLevelTrace, expEngineLevel: engine.LogLevelDbug}, + "DEBUG": {input: "DEBUG", expCliLevel: logging.LogLevelDebug, expEngineLevel: engine.LogLevelDbug}, + "DBUG": {input: "DBUG", expCliLevel: logging.LogLevelDebug, expEngineLevel: engine.LogLevelDbug}, + "INFO": {input: "INFO", expCliLevel: logging.LogLevelInfo, expEngineLevel: engine.LogLevelInfo}, + "NOTE": {input: "NOTE", expCliLevel: logging.LogLevelNotice, expEngineLevel: engine.LogLevelNote}, + "NOTICE": {input: "NOTICE", expCliLevel: logging.LogLevelNotice, expEngineLevel: engine.LogLevelNote}, + "WARN": {input: "WARN", expCliLevel: logging.LogLevelNotice, expEngineLevel: engine.LogLevelWarn}, + "ERROR": {input: "ERROR", expCliLevel: logging.LogLevelError, expEngineLevel: engine.LogLevelErr}, + "ERR": {input: "ERR", expCliLevel: logging.LogLevelError, expEngineLevel: engine.LogLevelErr}, + "CRIT": {input: "CRIT", expCliLevel: logging.LogLevelError, expEngineLevel: engine.LogLevelCrit}, + "ALRT": {input: "ALRT", expCliLevel: logging.LogLevelError, expEngineLevel: engine.LogLevelAlrt}, + "FATAL": {input: "FATAL", expCliLevel: logging.LogLevelError, expEngineLevel: engine.LogLevelEmrg}, + "EMRG": {input: "EMRG", expCliLevel: logging.LogLevelError, expEngineLevel: engine.LogLevelEmrg}, + "EMIT": {input: "EMIT", expCliLevel: logging.LogLevelError, expEngineLevel: engine.LogLevelEmit}, + "lowercase debug": { + input: "debug", expCliLevel: logging.LogLevelDebug, expEngineLevel: engine.LogLevelDbug, + }, + "invalid level": {input: "INVALID", expErr: true}, + "empty string": {input: "", expErr: true}, + } { + t.Run(name, func(t *testing.T) { + cliLevel, engineLevel, err := strToLogLevels(tc.input) + if tc.expErr { + if err == nil { + t.Fatalf("expected an error for input %q: got nil", tc.input) + } + return + } + if err != nil { + t.Fatalf("unexpected error for input %q: %v", tc.input, err) + } + test.AssertTrue(t, cliLevel == tc.expCliLevel, + fmt.Sprintf("unexpected CLI log level for input %q: want %v, got %v", tc.input, tc.expCliLevel, cliLevel)) + test.AssertTrue(t, engineLevel == tc.expEngineLevel, + fmt.Sprintf("unexpected engine log level for input %q: want %v, got %v", tc.input, tc.expEngineLevel, engineLevel)) + }) + } +} + +// TestNewLogger verifies the newLogger code paths: default level, explicit debug +// level, invalid level, and all three LogDir branches (valid dir, non-existent +// path, path that is a file rather than a directory). +func TestDdb_newLogger(t *testing.T) { + t.Run("no LogDir default level", func(t *testing.T) { + log, err := newLogger(cliOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if log == nil { + t.Fatal("expected non-nil logger") + } + }) + + t.Run("explicit valid debug level", func(t *testing.T) { + log, err := newLogger(cliOptions{Debug: "DEBUG"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if log == nil { + t.Fatal("expected non-nil logger") + } + }) + + t.Run("invalid debug level", func(t *testing.T) { + _, err := newLogger(cliOptions{Debug: "INVALID"}) + if err == nil { + t.Fatal("expected error for invalid log level, got nil") + } + }) + + t.Run("valid LogDir", func(t *testing.T) { + tmpDir := t.TempDir() + log, err := newLogger(cliOptions{LogDir: tmpDir}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if log == nil { + t.Fatal("expected non-nil logger") + } + }) + + t.Run("non-existent LogDir", func(t *testing.T) { + _, err := newLogger(cliOptions{LogDir: "/non/existent/path"}) + if err == nil { + t.Fatal("expected error for non-existent log dir, got nil") + } + }) + + t.Run("LogDir is a file", func(t *testing.T) { + tmpDir := t.TempDir() + tmpFile := filepath.Join(tmpDir, "not-a-dir") + if err := os.WriteFile(tmpFile, []byte(""), 0644); err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + _, err := newLogger(cliOptions{LogDir: tmpFile}) + if err == nil { + t.Fatal("expected error when LogDir is a file, got nil") + } + }) +} + +// TestClosePoolIfOpen verifies that closePoolIfOpen only calls Close when the +// pool is actually open, and that it tolerates a Close error (log only, no panic). +func TestDdb_closePoolIfOpen(t *testing.T) { + log := logging.NewCommandLineLogger() + + t.Run("pool not open, close not called", func(t *testing.T) { + ctx := newTestContext(t) + ddb_pool_is_open_RC = false + ddb_run_close_Fn = func() error { + t.Fatal("Close should not be called when pool is not open") + return nil + } + closePoolIfOpen(ctx, log) + }) + + t.Run("pool open, close called", func(t *testing.T) { + ctx := newTestContext(t) + ddb_pool_is_open_RC = true + closeCalled := false + ddb_run_close_Fn = func() error { + closeCalled = true + return nil + } + closePoolIfOpen(ctx, log) + test.AssertTrue(t, closeCalled, "expected Close to have been called when pool is open") + }) + + t.Run("pool open, close returns error", func(t *testing.T) { + ctx := newTestContext(t) + ddb_pool_is_open_RC = true + ddb_run_close_Fn = func() error { + return fmt.Errorf("close failed") + } + // Should not panic; the error is only logged. + closePoolIfOpen(ctx, log) + }) +} diff --git a/src/control/cmd/ddb/test_helpers.go b/src/control/cmd/ddb/test_helpers.go new file mode 100644 index 00000000000..3a171fec36c --- /dev/null +++ b/src/control/cmd/ddb/test_helpers.go @@ -0,0 +1,72 @@ +// +// (C) Copyright 2026 Hewlett Packard Enterprise Development LP +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +//go:build test_stubs + +package main + +import ( + "bytes" + "io" + "os" + "testing" +) + +type ddbTestErr string + +func (dte ddbTestErr) Error() string { + return string(dte) +} + +const ( + errUnknownCmd = ddbTestErr("Unknown command:") +) + +// newTestContext creates a fresh DdbContext for use in tests, resetting all +// stub variables to their zero values to ensure test isolation. +func newTestContext(t *testing.T) *DdbContext { + t.Helper() + resetDdbStubs() + return &DdbContext{} +} + +// captureStdout replaces os.Stdout with a pipe, runs fn, restores os.Stdout, +// and returns the captured output along with any error from fn. +func captureStdout(fn func() error) (output string, err error) { + var result bytes.Buffer + r, w, _ := os.Pipe() + done := make(chan struct{}) + go func() { + _, _ = io.Copy(&result, r) + close(done) + }() + stdout := os.Stdout + os.Stdout = w + // Deferred so cleanup and output capture always run, even if fn() exits + // via runtime.Goexit() (e.g., t.Fatalf called from a stub via test.CmpAny). + defer func() { + w.Close() + <-done + os.Stdout = stdout + output = result.String() + }() + + err = fn() + return +} + +// runCmdToStdout calls parseOpts with the given args and captures stdout +// output. Returns the parsed options, stdout output, and error. +func runCmdToStdout(ctx *DdbContext, args []string) (cliOptions, string, error) { + var opts cliOptions + stdout, err := captureStdout(func() error { + var e error + opts, _, e = parseOpts(args, ctx) + return e + }) + + return opts, stdout, err +} diff --git a/src/control/common/cmdutil/auto.go b/src/control/common/cmdutil/auto.go index 26bd56eba49..60c2195ea52 100644 --- a/src/control/common/cmdutil/auto.go +++ b/src/control/common/cmdutil/auto.go @@ -8,14 +8,15 @@ type deprecatedParams struct { type ConfGenCmd struct { deprecatedParams - MgmtSvcReplicas string `default:"localhost" short:"r" long:"ms-replicas" description:"Comma separated list of MS replica addresses to host management service"` - NrEngines int `short:"e" long:"num-engines" description:"Set the number of DAOS Engine sections to be populated in the config file output. If unset then the value will be set to the number of NUMA nodes on storage hosts in the DAOS system."` - SCMOnly bool `short:"s" long:"scm-only" description:"Create a SCM-only config without NVMe SSDs."` - NetClass string `default:"infiniband" short:"c" long:"net-class" description:"Set the network device class to be used" choice:"ethernet" choice:"infiniband"` - NetProvider string `short:"p" long:"net-provider" description:"Set the network fabric provider to be used"` - UseTmpfsSCM bool `short:"t" long:"use-tmpfs-scm" description:"Use tmpfs for scm rather than PMem"` - ExtMetadataPath string `short:"m" long:"control-metadata-path" description:"External storage path to store control metadata. Set this to a persistent location and specify --use-tmpfs-scm to create an MD-on-SSD config"` - FabricPorts string `short:"f" long:"fabric-ports" description:"Allow custom fabric interface ports to be specified for each engine config section. Comma separated port numbers, one per engine"` + MgmtSvcReplicas string `default:"localhost" short:"r" long:"ms-replicas" description:"Comma separated list of MS replica addresses to host management service"` + NrEngines int `short:"e" long:"num-engines" description:"Set the number of DAOS Engine sections to be populated in the config file output. If unset then the value will be set to the number of NUMA nodes on storage hosts in the DAOS system."` + SCMOnly bool `short:"s" long:"scm-only" description:"Create a SCM-only config without NVMe SSDs."` + NetClass string `default:"infiniband" short:"c" long:"net-class" description:"Set the network device class to be used" choice:"ethernet" choice:"infiniband"` + NetProvider string `short:"p" long:"net-provider" description:"Set the network fabric provider to be used"` + UseTmpfsSCM bool `short:"t" long:"use-tmpfs-scm" description:"Use tmpfs for scm rather than PMem"` + ExtMetadataPath string `short:"m" long:"control-metadata-path" description:"External storage path to store control metadata. Set this to a persistent location and specify --use-tmpfs-scm to create an MD-on-SSD config"` + FabricPorts string `short:"f" long:"fabric-ports" description:"Allow custom fabric interface ports to be specified for each engine config section. Comma separated port numbers, one per engine"` + AllowNumaImbalance bool `short:"n" long:"allow-numa-imbalance" description:"Distribute NVMe devices equally across engines regardless of NUMA affinity (may result in suboptimal performance due to cross-NUMA access)"` } // CheckDeprecated will check for deprecated parameters and update as needed. diff --git a/src/control/common/test/utils.go b/src/control/common/test/utils.go index 5b731550b9c..3f149baa41e 100644 --- a/src/control/common/test/utils.go +++ b/src/control/common/test/utils.go @@ -1,6 +1,6 @@ // // (C) Copyright 2018-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent @@ -150,6 +150,16 @@ func CmpAny(t *testing.T, desc string, want, got any, cmpOpts ...cmp.Option) { } } +// AssertStringContains asserts that s contains each of the provided substrings. +func AssertStringContains(t *testing.T, s string, substrings ...string) { + t.Helper() + for _, sub := range substrings { + if !strings.Contains(s, sub) { + t.Fatalf("expected string to contain %q: got\n%s", sub, s) + } + } +} + // SplitFile separates file content into contiguous sections separated by // a blank line. func SplitFile(path string) (sections [][]string, err error) { diff --git a/src/control/lib/control/auto.go b/src/control/lib/control/auto.go index d07a0a3e9c7..ae79ebc4ce9 100644 --- a/src/control/lib/control/auto.go +++ b/src/control/lib/control/auto.go @@ -70,8 +70,10 @@ type ( // Generate config with a tmpfs RAM-disk SCM. UseTmpfsSCM bool `json:"UseTmpfsSCM"` // Location to persist control-plane metadata, will generate MD-on-SSD config. - ExtMetadataPath string `json:"ExtMetadataPath"` - Log logging.Logger `json:"-"` + ExtMetadataPath string `json:"ExtMetadataPath"` + // Allow NUMA imbalance when NVMe devices are not evenly distributed. + AllowNumaImbalance bool `json:"AllowNumaImbalance"` + Log logging.Logger `json:"-"` } // ConfGenerateResp contains the generated server config. @@ -916,7 +918,12 @@ func filterDevicesByAffinity(req ConfGenerateReq, nd *networkDetails, sd *storag return nodeSet, nil } -func correctSSDCounts(log logging.Logger, sd *storageDetails) error { +func correctSSDCounts(log logging.Logger, sd *storageDetails, allowImbalance bool) error { + if allowImbalance { + log.Debug("allow-numa-imbalance enabled, distributing SSDs equally across engines") + return distributeSSDs(log, sd) + } + // calculate ssd count lowest value across numa nodes minSSDsInCfg, err := lowestCommonNrSSDs(sd.NumaSSDs.keys(), sd.NumaSSDs) if err != nil { @@ -950,6 +957,57 @@ func correctSSDCounts(log logging.Logger, sd *storageDetails) error { return nil } +// distributeSSDs allocates all available SSDs equally across engines, ignoring NUMA affinity. +// Each engine receives the same number of SSDs. If the total number of SSDs is not evenly +// divisible by the number of engines, only the maximum divisible number of SSDs are used and +// remainder SSDs are not included in the generated configuration. +func distributeSSDs(log logging.Logger, sd *storageDetails) error { + numaIDs := sd.NumaSSDs.keys() + if len(numaIDs) == 0 { + return errors.New("no numa nodes with SSDs found") + } + + // Collect all SSDs from all NUMA nodes + var allSSDs []string + for _, numaID := range numaIDs { + ssds := sd.NumaSSDs[numaID] + if ssds.HasVMD() { + // VMD devices cannot be distributed across NUMA nodes + log.Debugf("NUMA-%d has VMD devices, skipping distribution", numaID) + return errors.New("cannot distribute VMD devices across NUMA nodes, " + + "use without --allow-numa-imbalance flag") + } + allSSDs = append(allSSDs, ssds.Strings()...) + } + + totalSSDs := len(allSSDs) + nrEngines := len(numaIDs) + ssdsPerEngine := totalSSDs / nrEngines + remainder := totalSSDs % nrEngines + ssdsToUse := ssdsPerEngine * nrEngines + + if remainder > 0 { + log.Noticef("total SSDs (%d) not evenly divisible by engines (%d); "+ + "using %d SSDs (%d per engine), %d SSDs will not be used", + totalSSDs, nrEngines, ssdsToUse, ssdsPerEngine, remainder) + } + + log.Debugf("distributing %d SSDs equally across %d engines (%d per engine)", + ssdsToUse, nrEngines, ssdsPerEngine) + + // Distribute SSDs equally across engines, using only the divisible portion + idx := 0 + for i, numaID := range numaIDs { + engineSSDs := allSSDs[idx : idx+ssdsPerEngine] + sd.NumaSSDs[numaID] = hardware.MustNewPCIAddressSet(engineSSDs...) + log.Debugf("assigned %d SSDs to NUMA-%d (engine %d): %v", + ssdsPerEngine, numaID, i, engineSSDs) + idx += ssdsPerEngine + } + + return nil +} + func getSCMTier(log logging.Logger, numaID, nrNumaNodes int, sd *storageDetails) (*storage.TierConfig, error) { scmTier := storage.NewTierConfig().WithStorageClass(sd.scmCls.String()). WithScmMountPoint(fmt.Sprintf("%s%d", scmMountPrefix, numaID)) @@ -1056,7 +1114,7 @@ func genEngineConfigs(req ConfGenerateReq, newEngineCfg newEngineCfgFn, nodeSet // if nvme is enabled, make ssd counts consistent across numa groupings if !req.SCMOnly { - if err := correctSSDCounts(req.Log, sd); err != nil { + if err := correctSSDCounts(req.Log, sd, req.AllowNumaImbalance); err != nil { return nil, err } } @@ -1231,7 +1289,8 @@ func genServerConfig(req ConfGenerateReq, ecs []*engine.Config, tc *threadCounts WithMgmtSvcReplicas(req.MgmtSvcReplicas...). WithFabricProvider(ecs[0].Fabric.Provider). WithEngines(ecs...). - WithControlLogFile(defaultControlLogFile) + WithControlLogFile(defaultControlLogFile). + WithAllowNumaImbalance(req.AllowNumaImbalance) for idx := range cfg.Engines { tiers := cfg.Engines[idx].Storage.Tiers diff --git a/src/control/lib/control/auto_test.go b/src/control/lib/control/auto_test.go index 596b0d3c5de..2eb11f444ae 100644 --- a/src/control/lib/control/auto_test.go +++ b/src/control/lib/control/auto_test.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -1073,9 +1073,10 @@ func TestControl_AutoConfig_filterDevicesByAffinity(t *testing.T) { func TestControl_AutoConfig_correctSSDCounts(t *testing.T) { for name, tc := range map[string]struct { - sd storageDetails - expErr error - expSD storageDetails // expected details after updates + sd storageDetails + allowImbalance bool + expErr error + expSD storageDetails // expected details after updates }{ "no ssds": { sd: storageDetails{ @@ -1108,12 +1109,84 @@ func TestControl_AutoConfig_correctSSDCounts(t *testing.T) { }, }, }, + "allow imbalance distributes equally": { + sd: storageDetails{ + NumaSCMs: numaSCMsMap{ + 0: []string{"/dev/pmem0"}, + 1: []string{"/dev/pmem1"}, + }, + NumaSSDs: numaSSDsMap{ + 0: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(0, 1, 2, 3)...), + 1: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(4, 5)...), + }, + }, + allowImbalance: true, + expSD: storageDetails{ + NumaSCMs: numaSCMsMap{ + 0: []string{"/dev/pmem0"}, + 1: []string{"/dev/pmem1"}, + }, + // 6 total SSDs distributed equally: 3 per engine + NumaSSDs: numaSSDsMap{ + 0: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(0, 1, 2)...), + 1: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(3, 4, 5)...), + }, + }, + }, + "allow imbalance with remainder discards extras": { + sd: storageDetails{ + NumaSCMs: numaSCMsMap{ + 0: []string{"/dev/pmem0"}, + 1: []string{"/dev/pmem1"}, + }, + NumaSSDs: numaSSDsMap{ + 0: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(0, 1, 2, 3, 4)...), + 1: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(5, 6)...), + }, + }, + allowImbalance: true, + expSD: storageDetails{ + NumaSCMs: numaSCMsMap{ + 0: []string{"/dev/pmem0"}, + 1: []string{"/dev/pmem1"}, + }, + // 7 total SSDs: uses 6 (3 per engine), 1 remainder not used + NumaSSDs: numaSSDsMap{ + 0: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(0, 1, 2)...), + 1: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(3, 4, 5)...), + }, + }, + }, + "allow imbalance with 8 SSDs across 2 engines": { + sd: storageDetails{ + NumaSCMs: numaSCMsMap{ + 0: []string{"/dev/pmem0"}, + 1: []string{"/dev/pmem1"}, + }, + NumaSSDs: numaSSDsMap{ + 0: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(0, 1, 2, 3, 4)...), + 1: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(5, 6, 7)...), + }, + }, + allowImbalance: true, + expSD: storageDetails{ + NumaSCMs: numaSCMsMap{ + 0: []string{"/dev/pmem0"}, + 1: []string{"/dev/pmem1"}, + }, + // 8 total SSDs distributed equally: 4 per engine + NumaSSDs: numaSSDsMap{ + 0: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(0, 1, 2, 3)...), + 1: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(4, 5, 6, 7)...), + }, + }, + }, } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) defer test.ShowBufferOnFailure(t, buf) - gotErr := correctSSDCounts(log, &tc.sd) + gotErr := correctSSDCounts(log, &tc.sd, tc.allowImbalance) test.CmpErr(t, tc.expErr, gotErr) if tc.expErr != nil { return diff --git a/src/include/cart/swim.h b/src/include/cart/swim.h index c805c31b1aa..03d9c5c7a92 100644 --- a/src/include/cart/swim.h +++ b/src/include/cart/swim.h @@ -1,6 +1,7 @@ /* * Copyright (c) 2016 UChicago Argonne, LLC * (C) Copyright 2018-2023 Intel Corporation. + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -297,6 +298,19 @@ int swim_net_glitch_update(struct swim_context *ctx, swim_id_t id, */ void swim_member_del(struct swim_context *ctx, swim_id_t id); +/** + * Reset a SWIM member, typically when updating its incarnation number and + * setting its state to SWIM_MEMBER_ALIVE. + * + * @param[in] ctx SWIM context pointer from swim_init() + * @param[in] id IDs of member to reset + */ +static inline void +swim_member_reset(struct swim_context *ctx, swim_id_t id) +{ + swim_member_del(ctx, id); +} + /** @} */ #ifdef __cplusplus diff --git a/src/mgmt/srv_pool.c b/src/mgmt/srv_pool.c index 371c9e9922c..3c8a16237ea 100644 --- a/src/mgmt/srv_pool.c +++ b/src/mgmt/srv_pool.c @@ -12,11 +12,59 @@ #define D_LOGFAC DD_FAC(mgmt) +#include #include +#include #include #include "srv_internal.h" +static size_t +pool_destroy_local_scm_size(uuid_t pool_uuid) +{ + struct smd_pool_info *pool_info = NULL; + uint64_t eng_local_scm = 0; + int rc; + + if (!bio_nvme_configured(SMD_DEV_TYPE_META)) + return 0; + + rc = smd_pool_get_info(pool_uuid, &pool_info); + if (rc != 0 || pool_info == NULL) + return 0; + + if (pool_info->spi_scm_sz > 0 && dss_tgt_nr > 0) + eng_local_scm = pool_info->spi_scm_sz * dss_tgt_nr; + + smd_pool_free_info(pool_info); + return eng_local_scm; +} + +static uint32_t +pool_destroy_rpc_timeout(crt_rpc_t *td_req, uuid_t pool_uuid) +{ + uint32_t default_timeout; + uint32_t timeout; + size_t gib; + size_t eng_local_scm_size; + int rc; + + rc = crt_req_get_timeout(td_req, &default_timeout); + D_ASSERTF(rc == 0, "crt_req_get_timeout: " DF_RC "\n", DP_RC(rc)); + + eng_local_scm_size = pool_destroy_local_scm_size(pool_uuid); + if (eng_local_scm_size == 0) + return default_timeout; + + gib = eng_local_scm_size / ((size_t)1024 * 1024 * 1024); + if (gib < 1024) + timeout = 90; + else + timeout = 180; + + return max(timeout, default_timeout); +} + /** Destroy the pool on the specified ranks. */ int ds_mgmt_tgt_pool_destroy_ranks(uuid_t pool_uuid, d_rank_list_t *filter_ranks) @@ -27,6 +75,7 @@ ds_mgmt_tgt_pool_destroy_ranks(uuid_t pool_uuid, d_rank_list_t *filter_ranks) unsigned int opc; int topo; int rc; + uint32_t timeout; uint8_t mgmt_ver; rc = ds_mgmt_rpc_protocol(&mgmt_ver); @@ -45,6 +94,11 @@ ds_mgmt_tgt_pool_destroy_ranks(uuid_t pool_uuid, d_rank_list_t *filter_ranks) D_ASSERT(td_in != NULL); uuid_copy(td_in->td_pool_uuid, pool_uuid); + timeout = pool_destroy_rpc_timeout(td_req, pool_uuid); + crt_req_set_timeout(td_req, timeout); + D_DEBUG(DB_MGMT, DF_UUID ": setting pool destroy CoRPC timeout: %u sec\n", + DP_UUID(pool_uuid), timeout); + rc = dss_rpc_send(td_req); if (rc == 0 && DAOS_FAIL_CHECK(DAOS_POOL_DESTROY_FAIL_CORPC)) rc = -DER_TIMEDOUT; diff --git a/src/mgmt/srv_target.c b/src/mgmt/srv_target.c index e82d32d5713..e637916170d 100644 --- a/src/mgmt/srv_target.c +++ b/src/mgmt/srv_target.c @@ -29,40 +29,61 @@ static char *newborns_path; /** directory for destroyed pool */ static char *zombies_path; -/* ds_pooltgts* - * dpt_creates_ht tracks in-flight pool tgt creates - * tgt create inserts a record into creates_ht ; and during tgt allocation - * periodically checks if a tgt destroy is requested. - * tgt destroy checks if a record exists, modifies it to ask create to stop ; - * waits for create to remove the record (indicating create is done). - * In-memory, not persistent. +/* + * ds_pooltgts keeps two independent in-memory synchronization domains: + * - create vs destroy: dpt_creates_ht tracks in-flight creates so destroy can + * request cancellation and wait until preallocation exits. + * - destroy vs destroy: dpt_destroys_ht tracks in-flight destroys so only one + * destroy handler runs per pool UUID at a time. */ struct ds_pooltgts { - ABT_mutex dpt_mutex; - ABT_cond dpt_cv; + ABT_mutex dpt_create_mutex; + ABT_cond dpt_create_cv; struct d_hash_table dpt_creates_ht; + ABT_mutex dpt_destroy_mutex; + ABT_cond dpt_destroy_cv; + struct d_hash_table dpt_destroys_ht; }; - -struct ds_pooltgts_rec { +struct ds_pooltgts_create_rec { uuid_t dptr_uuid; bool cancel_create; /* ask create hdlr to stop prealloc */ d_list_t dptr_hlink; /* in hash table */ }; +struct ds_pooltgts_destroy_rec { + uuid_t dptr_uuid; + d_list_t dptr_hlink; /* in hash table */ +}; + static struct ds_pooltgts *pooltgts; -static inline struct ds_pooltgts_rec * -pooltgts_obj(d_list_t *rlink) +static inline struct ds_pooltgts_create_rec * +pooltgts_create_obj(d_list_t *rlink) { - return container_of(rlink, struct ds_pooltgts_rec, dptr_hlink); + return container_of(rlink, struct ds_pooltgts_create_rec, dptr_hlink); +} + +static inline struct ds_pooltgts_destroy_rec * +pooltgts_destroy_obj(d_list_t *rlink) +{ + return container_of(rlink, struct ds_pooltgts_destroy_rec, dptr_hlink); } static bool pooltgts_cmp_keys(struct d_hash_table *htable, d_list_t *rlink, const void *key, unsigned int ksize) { - struct ds_pooltgts_rec *ptrec = pooltgts_obj(rlink); + struct ds_pooltgts_create_rec *ptrec = pooltgts_create_obj(rlink); + + return uuid_compare(key, ptrec->dptr_uuid) == 0; +} + +static bool +pooltgts_destroy_cmp_keys(struct d_hash_table *htable, d_list_t *rlink, const void *key, + unsigned int ksize) +{ + struct ds_pooltgts_destroy_rec *ptrec = pooltgts_destroy_obj(rlink); return uuid_compare(key, ptrec->dptr_uuid) == 0; } @@ -71,6 +92,10 @@ static d_hash_table_ops_t pooltgts_hops = { .hop_key_cmp = pooltgts_cmp_keys, }; +static d_hash_table_ops_t pooltgts_destroy_hops = { + .hop_key_cmp = pooltgts_destroy_cmp_keys, +}; + static int destroy_cb(const char *path, const struct stat *sb, int flag, struct FTW *ftwbuf) @@ -476,32 +501,55 @@ ds_mgmt_tgt_setup(void) } } - /* create lock/cv and hash table to track outstanding pool creates */ + /* Create-side synchronization: create mutex/cv + in-flight create hash. */ D_ALLOC_PTR(pooltgts); if (pooltgts == NULL) { D_GOTO(err_zombies, rc = -DER_NOMEM); } - rc = ABT_mutex_create(&pooltgts->dpt_mutex); + rc = ABT_mutex_create(&pooltgts->dpt_create_mutex); if (rc != ABT_SUCCESS) { - D_ERROR("failed to create pooltgts mutex: %d\n", rc); + D_ERROR("failed to create create mutex: %d\n", rc); rc = dss_abterr2der(rc); - goto err_pooltgts; + goto err_pooltgts_alloc; } - rc = ABT_cond_create(&pooltgts->dpt_cv); + rc = ABT_cond_create(&pooltgts->dpt_create_cv); if (rc != ABT_SUCCESS) { - D_ERROR("failed to create pooltgts cv: %d\n", rc); + D_ERROR("failed to create create cv: %d\n", rc); rc = dss_abterr2der(rc); - goto err_mutex; + goto err_create_mutex; } + + /* Destroy-side synchronization: destroy mutex/cv + in-flight destroy hash. */ + rc = ABT_mutex_create(&pooltgts->dpt_destroy_mutex); + if (rc != ABT_SUCCESS) { + D_ERROR("failed to create destroy mutex: %d\n", rc); + rc = dss_abterr2der(rc); + goto err_create_cv; + } + + rc = ABT_cond_create(&pooltgts->dpt_destroy_cv); + if (rc != ABT_SUCCESS) { + D_ERROR("failed to create destroy cv: %d\n", rc); + rc = dss_abterr2der(rc); + goto err_destroy_mutex; + } + rc = d_hash_table_create_inplace(D_HASH_FT_NOLOCK, 6 /* bits */, NULL /* priv */, &pooltgts_hops, &pooltgts->dpt_creates_ht); if (rc) { D_ERROR("failed to create hash table (creates) "DF_RC"\n", DP_RC(rc)); - goto err_cv; + goto err_destroy_cv; + } + + rc = d_hash_table_create_inplace(D_HASH_FT_NOLOCK, 6 /* bits */, NULL /* priv */, + &pooltgts_destroy_hops, &pooltgts->dpt_destroys_ht); + if (rc) { + D_ERROR("failed to create hash table (destroys) " DF_RC "\n", DP_RC(rc)); + goto err_creates_ht; } rc = subtree_destroy(newborns_path); @@ -516,11 +564,18 @@ ds_mgmt_tgt_setup(void) rc); return 0; -err_cv: - ABT_cond_free(&pooltgts->dpt_cv); -err_mutex: - ABT_mutex_free(&pooltgts->dpt_mutex); -err_pooltgts: +err_creates_ht: + d_hash_table_destroy_inplace(&pooltgts->dpt_creates_ht, true); +err_destroy_cv: + ABT_cond_free(&pooltgts->dpt_destroy_cv); +err_destroy_mutex: + ABT_mutex_free(&pooltgts->dpt_destroy_mutex); + +err_create_cv: + ABT_cond_free(&pooltgts->dpt_create_cv); +err_create_mutex: + ABT_mutex_free(&pooltgts->dpt_create_mutex); +err_pooltgts_alloc: D_FREE(pooltgts); err_zombies: D_FREE(zombies_path); @@ -540,8 +595,14 @@ ds_mgmt_tgt_cleanup(void) D_ERROR("failed to destroy table: dpt_creates_ht: "DF_RC"\n", DP_RC(rc)); } - ABT_cond_free(&pooltgts->dpt_cv); - ABT_mutex_free(&pooltgts->dpt_mutex); + rc = d_hash_table_destroy_inplace(&pooltgts->dpt_destroys_ht, true); + if (rc) { + D_ERROR("failed to destroy table: dpt_destroys_ht: " DF_RC "\n", DP_RC(rc)); + } + ABT_cond_free(&pooltgts->dpt_destroy_cv); + ABT_mutex_free(&pooltgts->dpt_destroy_mutex); + ABT_cond_free(&pooltgts->dpt_create_cv); + ABT_mutex_free(&pooltgts->dpt_create_mutex); D_FREE(pooltgts); D_FREE(zombies_path); D_FREE(newborns_path); @@ -638,7 +699,7 @@ ds_mgmt_tgt_create_aggregator(crt_rpc_t *source, crt_rpc_t *result, struct tgt_create_args { char *tca_newborn; char *tca_path; - struct ds_pooltgts_rec *tca_ptrec; + struct ds_pooltgts_create_rec *tca_ptrec; struct dss_xstream *tca_dx; daos_size_t tca_scm_size; daos_size_t tca_nvme_size; @@ -752,11 +813,11 @@ ds_mgmt_hdlr_tgt_create(crt_rpc_t *tc_req) D_GOTO(out_reply, rc = -DER_NOMEM); uuid_copy(tca.tca_ptrec->dptr_uuid, tc_in->tc_pool_uuid); tca.tca_ptrec->cancel_create = false; - ABT_mutex_lock(pooltgts->dpt_mutex); + ABT_mutex_lock(pooltgts->dpt_create_mutex); rc = d_hash_rec_insert(&pooltgts->dpt_creates_ht, tca.tca_ptrec->dptr_uuid, sizeof(uuid_t), &tca.tca_ptrec->dptr_hlink, true); - ABT_mutex_unlock(pooltgts->dpt_mutex); + ABT_mutex_unlock(pooltgts->dpt_create_mutex); if (rc == -DER_EXIST) { D_ERROR(DF_UUID": already creating or cleaning up\n", DP_UUID(tc_in->tc_pool_uuid)); @@ -900,11 +961,11 @@ ds_mgmt_hdlr_tgt_create(crt_rpc_t *tc_req) } D_FREE(tca.tca_newborn); D_FREE(tca.tca_path); - ABT_mutex_lock(pooltgts->dpt_mutex); + ABT_mutex_lock(pooltgts->dpt_create_mutex); d_hash_rec_delete_at(&pooltgts->dpt_creates_ht, &tca.tca_ptrec->dptr_hlink); - ABT_cond_signal(pooltgts->dpt_cv); - ABT_mutex_unlock(pooltgts->dpt_mutex); + ABT_cond_signal(pooltgts->dpt_create_cv); + ABT_mutex_unlock(pooltgts->dpt_create_mutex); D_DEBUG(DB_MGMT, DF_UUID" record removed from dpt_creates_ht\n", DP_UUID(tca.tca_ptrec->dptr_uuid)); out_rec: @@ -1017,8 +1078,10 @@ ds_mgmt_hdlr_tgt_destroy(crt_rpc_t *td_req) { struct mgmt_tgt_destroy_in *td_in; struct mgmt_tgt_destroy_out *td_out; - char *path; + struct ds_pooltgts_destroy_rec *dtrec = NULL; + char *path = NULL; int rc; + bool destroy_registered = false; /** incoming request buffer */ td_in = crt_req_get(td_req); @@ -1029,11 +1092,14 @@ ds_mgmt_hdlr_tgt_destroy(crt_rpc_t *td_req) td_out = crt_reply_get(td_req); D_ASSERT(td_in != NULL && td_out != NULL); - /* If create in-flight, request it be canceled ; then wait */ - ABT_mutex_lock(pooltgts->dpt_mutex); + /* + * If create is in flight for this UUID, request cancel and wait until the + * create handler removes its record from dpt_creates_ht. + */ + ABT_mutex_lock(pooltgts->dpt_create_mutex); do { d_list_t *rec = NULL; - struct ds_pooltgts_rec *ptrec = NULL; + struct ds_pooltgts_create_rec *ptrec = NULL; uint32_t nreqs = 0; rec = d_hash_rec_find(&pooltgts->dpt_creates_ht, @@ -1041,14 +1107,54 @@ ds_mgmt_hdlr_tgt_destroy(crt_rpc_t *td_req) if (!rec) break; - ptrec = pooltgts_obj(rec); + ptrec = pooltgts_create_obj(rec); nreqs++; D_DEBUG(DB_MGMT, DF_UUID": busy creating tgts, ask to cancel " "(request %u)\n", DP_UUID(td_in->td_pool_uuid), nreqs); ptrec->cancel_create = true; - ABT_cond_wait(pooltgts->dpt_cv, pooltgts->dpt_mutex); + ABT_cond_wait(pooltgts->dpt_create_cv, pooltgts->dpt_create_mutex); } while (1); - ABT_mutex_unlock(pooltgts->dpt_mutex); + ABT_mutex_unlock(pooltgts->dpt_create_mutex); + + /* + * Serialize destroy handlers for the same pool UUID. The first destroy + * inserts into dpt_destroys_ht and proceeds; later destroy requests wait. + */ + ABT_mutex_lock(pooltgts->dpt_destroy_mutex); + do { + d_list_t *rec; + + rec = d_hash_rec_find(&pooltgts->dpt_destroys_ht, td_in->td_pool_uuid, + sizeof(uuid_t)); + if (rec != NULL) { + D_DEBUG(DB_MGMT, DF_UUID ": waiting for in-flight target destroy\n", + DP_UUID(td_in->td_pool_uuid)); + ABT_cond_wait(pooltgts->dpt_destroy_cv, pooltgts->dpt_destroy_mutex); + continue; + } + + D_ALLOC_PTR(dtrec); + if (dtrec == NULL) { + rc = -DER_NOMEM; + ABT_mutex_unlock(pooltgts->dpt_destroy_mutex); + goto out; + } + uuid_copy(dtrec->dptr_uuid, td_in->td_pool_uuid); + + rc = d_hash_rec_insert(&pooltgts->dpt_destroys_ht, dtrec->dptr_uuid, sizeof(uuid_t), + &dtrec->dptr_hlink, true); + if (rc != 0) { + D_FREE(dtrec); + dtrec = NULL; + ABT_mutex_unlock(pooltgts->dpt_destroy_mutex); + goto out; + } + + destroy_registered = true; + break; + } while (1); + ABT_mutex_unlock(pooltgts->dpt_destroy_mutex); + D_INFO(DF_UUID ": ready to destroy targets\n", DP_UUID(td_in->td_pool_uuid)); if (engine_in_check()) { @@ -1066,8 +1172,8 @@ ds_mgmt_hdlr_tgt_destroy(crt_rpc_t *td_req) /** generate path to the target directory */ rc = ds_mgmt_file(dss_storage_path, td_in->td_pool_uuid, NULL, NULL, &path); - if (rc) - D_GOTO(out, rc); + if (rc != 0) + goto out; /** check whether the target exists */ rc = access(path, F_OK); @@ -1077,7 +1183,7 @@ ds_mgmt_hdlr_tgt_destroy(crt_rpc_t *td_req) if (rc != 0 && rc != -DER_BUSY) { D_ERROR(DF_UUID": kill pool service VOS pool: "DF_RC"\n", DP_UUID(td_in->td_pool_uuid), DP_RC(rc)); - goto out_path; + goto out; } rc = tgt_destroy(td_req->cr_input, path); } else if (errno == ENOENT) { @@ -1089,8 +1195,8 @@ ds_mgmt_hdlr_tgt_destroy(crt_rpc_t *td_req) * failed, so flush again. */ rc = ds_mgmt_file(zombies_path, td_in->td_pool_uuid, NULL, NULL, &zombie); - if (rc) - D_GOTO(out, rc); + if (rc != 0) + goto out; rc = ds_mgmt_dir_fsync(zombie); if (rc == -DER_NONEXIST) rc = 0; @@ -1099,9 +1205,15 @@ ds_mgmt_hdlr_tgt_destroy(crt_rpc_t *td_req) rc = daos_errno2der(errno); } -out_path: - D_FREE(path); out: + D_FREE(path); + if (destroy_registered) { + ABT_mutex_lock(pooltgts->dpt_destroy_mutex); + d_hash_rec_delete_at(&pooltgts->dpt_destroys_ht, &dtrec->dptr_hlink); + ABT_cond_broadcast(pooltgts->dpt_destroy_cv); + ABT_mutex_unlock(pooltgts->dpt_destroy_mutex); + } + D_FREE(dtrec); td_out->td_rc = rc; crt_reply_send(td_req); } diff --git a/src/tests/ftest/cart/utest/utest_swim.c b/src/tests/ftest/cart/utest/utest_swim.c index 8f57fd7717b..3844352b619 100644 --- a/src/tests/ftest/cart/utest/utest_swim.c +++ b/src/tests/ftest/cart/utest/utest_swim.c @@ -1,6 +1,6 @@ /* * (C) Copyright 2020-2024 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -20,16 +20,32 @@ #include #include "../cart/crt_internal.h" +static bool +rank_is_suspected(struct swim_context *ctx, swim_id_t id) +{ + struct swim_item *item; + bool found = false; + + TAILQ_FOREACH(item, &ctx->sc_suspects, si_link) + { + if (item->si_id == id) { + found = true; + break; + } + } + + return found; +} + static void -test_swim(void **state) +test_setup(crt_context_t *crt_ctx) { - crt_context_t crt_ctx; int rc; rc = crt_init(NULL, CRT_FLAG_BIT_SERVER | CRT_FLAG_BIT_AUTO_SWIM_DISABLE); assert_int_equal(rc, 0); - rc = crt_context_create(&crt_ctx); + rc = crt_context_create(crt_ctx); assert_int_equal(rc, 0); rc = crt_swim_init(0); @@ -37,6 +53,27 @@ test_swim(void **state) rc = crt_rank_self_set(0, 1 /* group_version_min */); assert_int_equal(rc, 0); +} + +static void +test_teardown(crt_context_t crt_ctx) +{ + int rc; + + crt_swim_fini(); + rc = crt_context_destroy(crt_ctx, 0); + assert_int_equal(rc, 0); + rc = crt_finalize(); + assert_int_equal(rc, 0); +} + +static void +test_swim(void **state) +{ + crt_context_t crt_ctx; + int rc; + + test_setup(&crt_ctx); rc = crt_swim_rank_add(crt_grp_pub2priv(NULL), 1, d_hlc_get()); assert_int_equal(rc, 0); @@ -50,11 +87,60 @@ test_swim(void **state) rc = crt_swim_rank_add(crt_grp_pub2priv(NULL), 0, d_hlc_get()); assert_int_equal(rc, -DER_ALREADY); - crt_swim_fini(); - rc = crt_context_destroy(crt_ctx, 0); + test_teardown(crt_ctx); +} + +/* + * Calling crt_swim_rank_check() on a suspected member with a newer incarnation + * should mark the member ALIVE and remove it from the suspect list. + */ +static void +test_swim_rank_check_clears_suspect(void **state) +{ + struct crt_grp_priv *grp_priv; + struct crt_swim_membs *csm; + struct swim_context *ctx; + struct swim_member_update upd; + crt_context_t crt_ctx; + uint64_t incarnation; + int rc; + + test_setup(&crt_ctx); + + grp_priv = crt_grp_pub2priv(NULL); + csm = &grp_priv->gp_membs_swim; + ctx = csm->csm_ctx; + + /* Add the member to be suspected (rank 1) and a gossip source (rank 2). */ + incarnation = d_hlc_get(); + rc = crt_swim_rank_add(grp_priv, 1, incarnation); assert_int_equal(rc, 0); - rc = crt_finalize(); + + rc = crt_swim_rank_add(grp_priv, 2, d_hlc_get()); assert_int_equal(rc, 0); + + /* + * Deliver a SUSPECT update about rank 1 from rank 2 (an alive, hence + * trustable, member) so that rank 1 is added to the suspect list. + */ + upd.smu_id = 1; + upd.smu_state.sms_incarnation = incarnation; + upd.smu_state.sms_status = SWIM_MEMBER_SUSPECT; + upd.smu_state.sms_delay = 0; + + rc = swim_updates_parse(ctx, 2 /* from */, 1 /* id */, &upd, 1); + assert_int_equal(rc, 0); + assert_true(rank_is_suspected(ctx, 1)); + + /* + * crt_swim_rank_check() with a newer incarnation must clear the + * suspicion of rank 1. + */ + rc = crt_swim_rank_check(grp_priv, 1, incarnation + 1); + assert_int_equal(rc, 0); + assert_false(rank_is_suspected(ctx, 1)); + + test_teardown(crt_ctx); } static int @@ -82,7 +168,8 @@ fini_tests(void **state) int main(int argc, char **argv) { const struct CMUnitTest tests[] = { - cmocka_unit_test(test_swim), + cmocka_unit_test(test_swim), + cmocka_unit_test(test_swim_rank_check_clears_suspect), }; d_register_alt_assert(mock_assert); diff --git a/src/tests/ftest/container/boundary.py b/src/tests/ftest/container/boundary.py index a6687a7a93a..915b9a4cd7a 100644 --- a/src/tests/ftest/container/boundary.py +++ b/src/tests/ftest/container/boundary.py @@ -1,6 +1,6 @@ """ (C) Copyright 2022-2023 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -104,9 +104,10 @@ def create_pools(self, num_pools, num_containers): self.fail('{} pool create threads failed'.format(num_failed)) self.log.info('Created %d pools', num_pools) - # Create all containers for all pools in parallel + # Create all containers for all pools in parallel (specifying max_workers because of the + # 2-second sleep in every thread) container_manager = ThreadManager( - self.create_container_and_test, self.get_remaining_time() - 30) + self.create_container_and_test, self.get_remaining_time() - 30, max_workers=1024) all_pool_cont_args = list(itertools.product(self.pool, range(num_containers))) self.random.shuffle(all_pool_cont_args) for pool, cont_num in all_pool_cont_args: diff --git a/src/tests/ftest/control/config_generate_output.py b/src/tests/ftest/control/config_generate_output.py index 35ce1cf521f..d62c2b57d93 100644 --- a/src/tests/ftest/control/config_generate_output.py +++ b/src/tests/ftest/control/config_generate_output.py @@ -1,5 +1,6 @@ ''' (C) Copyright 2018-2024 Intel Corporation. + (C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent ''' @@ -9,6 +10,7 @@ from apricot import TestWithServers from dmg_utils import DmgCommand from exception_utils import CommandFailure +from storage_utils import storage_numa_nodes class ConfigGenerateOutput(TestWithServers): @@ -24,6 +26,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.def_provider = "ofi+tcp" + self.allow_numa_imbalance = False # Data structure that store expected values. self.numa_node_to_pci_addrs = defaultdict(set) @@ -32,6 +35,16 @@ def __init__(self, *args, **kwargs): self.interface_to_providers = defaultdict(set) self.interface_set = set() + def setUp(self): + """Set up each test case.""" + super().setUp() + + # Determine the number of available numa nodes. This is be used to determine if the + # --allow-numa-imbalance flag is needed when generating running config generate. + self.allow_numa_imbalance = False + if len(storage_numa_nodes(self)) > 1: + self.allow_numa_imbalance = True + def prepare_expected_data(self): """Prepare expected values. @@ -147,7 +160,8 @@ def verify_ms_replica(self, host_port_input, failure_expected=None): try: result = dmg.config_generate( - mgmt_svc_replicas=host_port_input, net_provider=self.def_provider) + mgmt_svc_replicas=host_port_input, net_provider=self.def_provider, + allow_numa_imbalance=self.allow_numa_imbalance) except CommandFailure as err: errors.append("Unexpected failure! {}".format(err)) @@ -186,7 +200,7 @@ def test_basic_config(self): 5. Repeat for all engines. :avocado: tags=all,full_regression - :avocado: tags=hw,large + :avocado: tags=hw,hw_vmd,medium :avocado: tags=control,dmg_config_generate :avocado: tags=ConfigGenerateOutput,test_basic_config """ @@ -195,7 +209,8 @@ def test_basic_config(self): # 1. Call dmg config generate. result = self.get_dmg_command().config_generate( - mgmt_svc_replicas="wolf-a", net_provider=self.def_provider) + mgmt_svc_replicas="wolf-a", net_provider=self.def_provider, + allow_numa_imbalance=self.allow_numa_imbalance) generated_yaml = yaml.safe_load(result.stdout) errors = [] @@ -264,7 +279,7 @@ def test_tmpfs_scm_config(self): 7. Repeat for all engines. :avocado: tags=all,full_regression - :avocado: tags=hw,large + :avocado: tags=hw,hw_vmd,medium :avocado: tags=control,dmg_config_generate :avocado: tags=ConfigGenerateOutput,test_tmpfs_scm_config """ @@ -276,7 +291,7 @@ def test_tmpfs_scm_config(self): # Call dmg config generate. result = self.get_dmg_command().config_generate( mgmt_svc_replicas="wolf-a", net_provider=self.def_provider, use_tmpfs_scm=True, - control_metadata_path=self.test_dir) + control_metadata_path=self.test_dir, allow_numa_imbalance=self.allow_numa_imbalance) if result.exit_status != 0: errors.append("Config generate failed with use_tmpfs_scm = True!") generated_yaml = yaml.safe_load(result.stdout) @@ -332,7 +347,7 @@ def test_mgmt_svc_replicas_single(self): """Test --ms-replica with single MS replica with and without port. :avocado: tags=all,full_regression - :avocado: tags=hw,large + :avocado: tags=hw,hw_vmd,medium :avocado: tags=control,dmg_config_generate,mgmt_svc_replicas :avocado: tags=ConfigGenerateOutput,test_mgmt_svc_replicas_single """ @@ -350,7 +365,7 @@ def test_mgmt_svc_replicas_odd(self): """Test --ms-replicas with odd number of MS replicas. :avocado: tags=all,full_regression - :avocado: tags=hw,large + :avocado: tags=hw,hw_vmd,medium :avocado: tags=control,dmg_config_generate,mgmt_svc_replicas :avocado: tags=ConfigGenerateOutput,test_mgmt_svc_replicas_odd """ @@ -368,7 +383,7 @@ def test_mgmt_svc_replicas_invalid(self): """Test --ms-replicas with invalid port. :avocado: tags=all,full_regression - :avocado: tags=hw,large + :avocado: tags=hw,hw_vmd,medium :avocado: tags=control,dmg_config_generate,mgmt_svc_replicas :avocado: tags=ConfigGenerateOutput,test_mgmt_svc_replicas_invalid """ @@ -391,7 +406,7 @@ def test_mgmt_svc_replicas_same_ap_repeated(self): """Test --ms-replicas with the same MS replicas repeated. :avocado: tags=all,full_regression - :avocado: tags=hw,large + :avocado: tags=hw,hw_vmd,medium :avocado: tags=control,dmg_config_generate,mgmt_svc_replicas :avocado: tags=ConfigGenerateOutput,test_mgmt_svc_replicas_same_ap_repeated """ @@ -412,7 +427,7 @@ def test_num_engines(self): 3. Call dmg config generate --num-engines= Should fail. :avocado: tags=all,full_regression - :avocado: tags=hw,large + :avocado: tags=hw,hw_vmd,medium :avocado: tags=control,dmg_config_generate :avocado: tags=ConfigGenerateOutput,test_num_engines """ @@ -433,7 +448,8 @@ def test_num_engines(self): # Call dmg config generate --num-engines=<1 to max_engine> for num_engines in range(1, max_engine + 1): result = dmg.config_generate( - mgmt_svc_replicas="wolf-a", num_engines=num_engines, net_provider=self.def_provider) + mgmt_svc_replicas="wolf-a", num_engines=num_engines, net_provider=self.def_provider, + allow_numa_imbalance=self.allow_numa_imbalance) generated_yaml = yaml.safe_load(result.stdout) actual_num_engines = len(generated_yaml["engines"]) @@ -445,7 +461,8 @@ def test_num_engines(self): # Verify that max_engine + 1 fails. result = dmg.config_generate( - mgmt_svc_replicas="wolf-a", num_engines=max_engine + 1, net_provider=self.def_provider) + mgmt_svc_replicas="wolf-a", num_engines=max_engine + 1, net_provider=self.def_provider, + allow_numa_imbalance=self.allow_numa_imbalance) if result.exit_status == 0: errors.append("Host + invalid num engines succeeded with {}!".format(max_engine + 1)) @@ -460,7 +477,7 @@ def test_scm_only(self): verify that there's no bdev_list field. :avocado: tags=all,full_regression - :avocado: tags=hw,large + :avocado: tags=hw,hw_vmd,medium :avocado: tags=control,dmg_config_generate :avocado: tags=ConfigGenerateOutput,test_scm_only """ @@ -474,7 +491,8 @@ def test_scm_only(self): # Call dmg config generate with --scm-only=False result = dmg.config_generate( - mgmt_svc_replicas="wolf-a", scm_only=False, net_provider=self.def_provider) + mgmt_svc_replicas="wolf-a", scm_only=False, net_provider=self.def_provider, + allow_numa_imbalance=self.allow_numa_imbalance) if result.exit_status != 0: errors.append("config generate failed with scm_only = False!") generated_yaml = yaml.safe_load(result.stdout) @@ -493,7 +511,8 @@ def test_scm_only(self): # Call dmg config generate with --scm-only=True result = dmg.config_generate( - mgmt_svc_replicas="wolf-a", scm_only=True, net_provider=self.def_provider) + mgmt_svc_replicas="wolf-a", scm_only=True, net_provider=self.def_provider, + allow_numa_imbalance=self.allow_numa_imbalance) if result.exit_status != 0: errors.append("config generate failed with scm_only = True!") generated_yaml = yaml.safe_load(result.stdout) @@ -526,7 +545,7 @@ def test_net_class(self): "fabric_iface". :avocado: tags=all,full_regression - :avocado: tags=hw,large + :avocado: tags=hw,hw_vmd,medium :avocado: tags=control,dmg_config_generate :avocado: tags=ConfigGenerateOutput,test_net_class """ @@ -550,7 +569,7 @@ def test_net_class(self): # dmg config generate should pass. result = dmg.config_generate( mgmt_svc_replicas="wolf-a", num_engines=num_engines, net_class="infiniband", - net_provider=self.def_provider) + net_provider=self.def_provider, allow_numa_imbalance=self.allow_numa_imbalance) if result.exit_status != 0: msg = "config generate failed with --net-class=infiniband "\ @@ -576,7 +595,7 @@ def test_net_class(self): # --net-class=infiniband. Too many engines. Should fail. result = dmg.config_generate( mgmt_svc_replicas="wolf-a", num_engines=ib_count + 1, net_class="infiniband", - net_provider=self.def_provider) + net_provider=self.def_provider, allow_numa_imbalance=self.allow_numa_imbalance) if result.exit_status == 0: msg = "config generate succeeded with --net-class=infiniband num_engines = {}!".format( ib_count + 1) @@ -595,7 +614,7 @@ def test_net_class(self): # dmg config generate should pass. result = dmg.config_generate( mgmt_svc_replicas="wolf-a", num_engines=num_engines, net_class="ethernet", - net_provider=self.def_provider) + net_provider=self.def_provider, allow_numa_imbalance=self.allow_numa_imbalance) if result.exit_status != 0: msg = "config generate failed with --net-class=ethernet --num-engines = {}!".format( @@ -621,7 +640,7 @@ def test_net_class(self): # --net-class=ethernet. Too many engines. Should fail. result = dmg.config_generate( mgmt_svc_replicas="wolf-a", num_engines=eth_count + 1, net_class="ethernet", - net_provider=self.def_provider) + net_provider=self.def_provider, allow_numa_imbalance=self.allow_numa_imbalance) if result.exit_status == 0: msg = "config generate succeeded with --net-class=ethernet, num_engines = {}!".format( eth_count + 1) diff --git a/src/tests/ftest/control/config_generate_run.py b/src/tests/ftest/control/config_generate_run.py index fcd2ad26d0f..ab9f187881c 100644 --- a/src/tests/ftest/control/config_generate_run.py +++ b/src/tests/ftest/control/config_generate_run.py @@ -1,6 +1,6 @@ ''' (C) Copyright 2018-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent ''' @@ -10,6 +10,7 @@ import yaml from apricot import TestWithServers from server_utils import ServerFailed +from storage_utils import storage_numa_nodes class ConfigGenerateRun(TestWithServers): @@ -21,6 +22,21 @@ class ConfigGenerateRun(TestWithServers): :avocado: recursive """ + def __init__(self, *args, **kwargs): + """Initialize a ConfigGenerateRun object.""" + super().__init__(*args, **kwargs) + self.allow_numa_imbalance = False + + def setUp(self): + """Set up each test case.""" + super().setUp() + + # Determine the number of available numa nodes. This is be used to determine if the + # --allow-numa-imbalance flag is needed when generating running config generate. + self.allow_numa_imbalance = False + if len(storage_numa_nodes(self)) > 1: + self.allow_numa_imbalance = True + def test_config_generate_run(self): """Run daos_server with generated server config file. @@ -36,7 +52,7 @@ def test_config_generate_run(self): Note: When running locally, use 50 sec timeout in DaosServerCommand.__init__() :avocado: tags=all,full_regression - :avocado: tags=hw,large + :avocado: tags=hw,hw_vmd,medium :avocado: tags=control,dmg_config_generate :avocado: tags=ConfigGenerateRun,test_config_generate_run """ @@ -58,7 +74,7 @@ def test_config_generate_run(self): result = self.get_dmg_command().config_generate( mgmt_svc_replicas=server_host, num_engines=num_engines, scm_only=scm_only, net_class=net_class, net_provider=net_provider, use_tmpfs_scm=use_tmpfs_scm, - control_metadata_path=control_metadata) + control_metadata_path=control_metadata, allow_numa_imbalance=self.allow_numa_imbalance) try: generated_yaml = yaml.safe_load(result.stdout) diff --git a/src/tests/ftest/control/daos_server_helper.py b/src/tests/ftest/control/daos_server_helper.py index bfdac007d95..5310df70257 100644 --- a/src/tests/ftest/control/daos_server_helper.py +++ b/src/tests/ftest/control/daos_server_helper.py @@ -32,7 +32,7 @@ def test_daos_server_helper_format(self): operations while daos_server is run as normal user. :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=control,basic,daos_server_helper :avocado: tags=DaosPrivHelperTest,test_daos_server_helper_format """ diff --git a/src/tests/ftest/control/dmg_telemetry_nvme.py b/src/tests/ftest/control/dmg_telemetry_nvme.py index 0e211d8d617..376f5130b17 100644 --- a/src/tests/ftest/control/dmg_telemetry_nvme.py +++ b/src/tests/ftest/control/dmg_telemetry_nvme.py @@ -1,5 +1,6 @@ """ (C) Copyright 2021-2023 Intel Corporation. +(C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -32,7 +33,7 @@ def test_nvme_telemetry_metrics(self): Verify the telemetry engine NVMe metrics. :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=control,telemetry,nvme :avocado: tags=TestWithTelemetryNvme,test_nvme_telemetry_metrics """ @@ -68,7 +69,7 @@ def test_telemetry_list_nvme(self): Verify the dmg telemetry list command. :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=control,telemetry,nvme :avocado: tags=TestWithTelemetryNvme,test_telemetry_list_nvme """ diff --git a/src/tests/ftest/daos_test/rebuild.py b/src/tests/ftest/daos_test/rebuild.py index e1d8de4052b..48a378d02fe 100644 --- a/src/tests/ftest/daos_test/rebuild.py +++ b/src/tests/ftest/daos_test/rebuild.py @@ -1,6 +1,6 @@ ''' (C) Copyright 2018-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent ''' @@ -25,7 +25,7 @@ def test_rebuild_0to10(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_0to10 """ @@ -41,7 +41,7 @@ def test_rebuild_12to15(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,2,medium + :avocado: tags=cb,2,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_12to15 """ @@ -57,7 +57,7 @@ def test_rebuild_16(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_16 """ @@ -73,7 +73,7 @@ def test_rebuild_17(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_17 """ @@ -89,7 +89,7 @@ def test_rebuild_18(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_18 """ @@ -105,7 +105,7 @@ def test_rebuild_19(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_19 """ @@ -121,7 +121,7 @@ def test_rebuild_20(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_20 """ @@ -137,7 +137,7 @@ def test_rebuild_21(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_21 """ @@ -153,7 +153,7 @@ def test_rebuild_22(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_22 """ @@ -169,7 +169,7 @@ def test_rebuild_23(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_23 """ @@ -185,7 +185,7 @@ def test_rebuild_24(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_24 """ @@ -201,7 +201,7 @@ def test_rebuild_25(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_25 """ @@ -217,7 +217,7 @@ def test_rebuild_26(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_26 """ @@ -233,7 +233,7 @@ def test_rebuild_27(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_27 """ @@ -249,7 +249,7 @@ def test_rebuild_28(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_28 """ @@ -265,7 +265,7 @@ def test_rebuild_29(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_29 """ @@ -281,7 +281,7 @@ def test_rebuild_30(self): Core tests for daos_test rebuild :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=unittest,rebuild :avocado: tags=DaosCoreTestRebuild,daos_test,daos_core_test_rebuild,test_rebuild_30 """ diff --git a/src/tests/ftest/daos_test/rebuild.yaml b/src/tests/ftest/daos_test/rebuild.yaml index 6f80a3e7dcc..7ba11b64d84 100644 --- a/src/tests/ftest/daos_test/rebuild.yaml +++ b/src/tests/ftest/daos_test/rebuild.yaml @@ -7,7 +7,9 @@ timeout: 800 timeouts: test_rebuild_0to10: 2000 test_rebuild_12to15: 1500 + test_rebuild_25: 1500 test_rebuild_27: 1500 + test_rebuild_28: 1500 test_rebuild_31: 400 test_rebuild_32: 400 test_rebuild_33: 200 diff --git a/src/tests/ftest/daos_test/suite.py b/src/tests/ftest/daos_test/suite.py index 3be9b5afe7b..77ac83217b5 100644 --- a/src/tests/ftest/daos_test/suite.py +++ b/src/tests/ftest/daos_test/suite.py @@ -1,6 +1,6 @@ """ (C) Copyright 2018-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -24,7 +24,7 @@ def test_daos_degraded_mode(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_degraded_mode """ @@ -56,7 +56,7 @@ def test_daos_pool(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_pool """ @@ -72,7 +72,7 @@ def test_daos_container(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_container """ @@ -88,7 +88,7 @@ def test_daos_epoch(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_epoch """ @@ -104,7 +104,7 @@ def test_daos_single_rdg_tx(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_single_rdg_tx """ @@ -120,7 +120,7 @@ def test_daos_distributed_tx(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_distributed_tx """ @@ -136,7 +136,7 @@ def test_daos_verify_consistency(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_verify_consistency """ @@ -152,7 +152,7 @@ def test_daos_io(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_io """ @@ -168,7 +168,7 @@ def test_daos_ec_io(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_ec_io """ @@ -184,7 +184,7 @@ def test_daos_ec_obj(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_ec_obj """ @@ -200,7 +200,7 @@ def test_daos_object_array(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_object_array """ @@ -216,7 +216,7 @@ def test_daos_array(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_array """ @@ -232,7 +232,7 @@ def test_daos_kv(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_kv """ @@ -248,7 +248,7 @@ def test_daos_capability(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_capability """ @@ -264,7 +264,7 @@ def test_daos_epoch_recovery(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_epoch_recovery """ @@ -280,7 +280,7 @@ def test_daos_md_replication(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_md_replication """ @@ -312,7 +312,7 @@ def test_daos_drain_simple(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test,rebuild :avocado: tags=DaosCoreTest,test_daos_drain_simple """ @@ -344,7 +344,7 @@ def test_daos_rebuild_interactive(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test,rebuild :avocado: tags=DaosCoreTest,test_daos_rebuild_interactive """ @@ -360,7 +360,7 @@ def test_daos_oid_allocator(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_oid_allocator """ @@ -376,7 +376,7 @@ def test_daos_checksum(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_checksum """ @@ -408,7 +408,7 @@ def test_daos_aggregate_ec(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_aggregate_ec """ @@ -424,7 +424,7 @@ def test_daos_degraded_ec(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_degraded_ec """ @@ -440,7 +440,7 @@ def test_daos_dedup(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_dedup """ @@ -456,7 +456,7 @@ def test_daos_upgrade(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test :avocado: tags=DaosCoreTest,test_daos_upgrade """ @@ -472,7 +472,7 @@ def test_daos_pipeline(self): Core tests for daos_test :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium,provider + :avocado: tags=cb,medium,provider :avocado: tags=daos_test,daos_core_test,faults :avocado: tags=DaosCoreTest,test_daos_pipeline """ diff --git a/src/tests/ftest/daos_test/suite.single_host.yaml b/src/tests/ftest/daos_test/suite.single_host.yaml index 0ecec02dfaf..d28066a5fed 100644 --- a/src/tests/ftest/daos_test/suite.single_host.yaml +++ b/src/tests/ftest/daos_test/suite.single_host.yaml @@ -9,7 +9,7 @@ timeout: 600 timeouts: test_daos_degraded_mode: 450 test_daos_management: 110 - test_daos_pool: 180 + test_daos_pool: 360 test_daos_container: 450 test_daos_epoch: 125 test_daos_verify_consistency: 105 diff --git a/src/tests/ftest/daos_test/suite.yaml b/src/tests/ftest/daos_test/suite.yaml index 0a98fd16f0a..622190b1789 100644 --- a/src/tests/ftest/daos_test/suite.yaml +++ b/src/tests/ftest/daos_test/suite.yaml @@ -10,7 +10,7 @@ timeout: 600 timeouts: test_daos_degraded_mode: 450 test_daos_management: 110 - test_daos_pool: 300 + test_daos_pool: 360 test_daos_container: 700 test_daos_epoch: 125 test_daos_verify_consistency: 105 @@ -68,7 +68,7 @@ server_config: storage: auto transport_config: allow_insecure: true - system_ram_reserved: 64 + system_ram_reserved: 16 agent_config: transport_config: diff --git a/src/tests/ftest/harness/basic.py b/src/tests/ftest/harness/basic.py index 0ff102e2074..f60c128969f 100644 --- a/src/tests/ftest/harness/basic.py +++ b/src/tests/ftest/harness/basic.py @@ -1,5 +1,6 @@ """ (C) Copyright 2022-2024 Intel Corporation. + (C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -31,7 +32,7 @@ def test_always_fails_hw(self): """Simple test of apricot test code. :avocado: tags=all - :avocado: tags=hw,large,medium,small + :avocado: tags=hw,cb,large,medium,small :avocado: tags=harness,harness_basic_test :avocado: tags=HarnessBasicTest,always_fails,test_always_fails_hw """ @@ -51,7 +52,7 @@ def test_always_passes_hw(self): """Simple test of apricot test code. :avocado: tags=all - :avocado: tags=hw,hw_vmd,medium,large + :avocado: tags=hw,cb,hw_vmd,medium,large :avocado: tags=harness,harness_basic_test,always_passes :avocado: tags=HarnessBasicTest,test_always_passes_hw """ @@ -91,7 +92,7 @@ def test_load_mpi_hw(self): """Simple test of apricot test code to load the openmpi module. :avocado: tags=all - :avocado: tags=hw,hw_vmd,medium,large + :avocado: tags=hw,cb,hw_vmd,medium,large :avocado: tags=harness,harness_basic_test,load_mpi :avocado: tags=HarnessBasicTest,test_load_mpi_hw """ diff --git a/src/tests/ftest/harness/setup.py b/src/tests/ftest/harness/setup.py index 75598ba20e3..aba050263f4 100644 --- a/src/tests/ftest/harness/setup.py +++ b/src/tests/ftest/harness/setup.py @@ -1,5 +1,6 @@ """ (C) Copyright 2021-2023 Intel Corporation. +(C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -18,7 +19,7 @@ def test_setup(self): Useful for setting up the /etc/daos/daos_server.yml files on multiple hosts. :avocado: tags=all - :avocado: tags=hw,medium,large + :avocado: tags=hw,cb,medium,large :avocado: tags=harness :avocado: tags=HarnessSetupTest,test_setup """ diff --git a/src/tests/ftest/osa/online_parallel_test.py b/src/tests/ftest/osa/online_parallel_test.py index cf473038f55..72debd97543 100644 --- a/src/tests/ftest/osa/online_parallel_test.py +++ b/src/tests/ftest/osa/online_parallel_test.py @@ -1,6 +1,6 @@ """ (C) Copyright 2020-2023 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -193,7 +193,7 @@ def test_osa_online_parallel_test(self): Test Description: Runs multiple OSA commands/IO in parallel :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=osa,checksum,osa_parallel :avocado: tags=OSAOnlineParallelTest,test_osa_online_parallel_test """ diff --git a/src/tests/ftest/pool/create_query.py b/src/tests/ftest/pool/create_query.py index 84593d7a547..9ea6d6dadee 100644 --- a/src/tests/ftest/pool/create_query.py +++ b/src/tests/ftest/pool/create_query.py @@ -1,6 +1,6 @@ """ (C) Copyright 2021-2024 Intel Corporation. -(C) Copyright 2025 Hewlett Packard Enterprise Development LP +(C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -32,7 +32,7 @@ def test_create_and_query(self): pool create. :avocado: tags=all,pr,daily_regression - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=pool :avocado: tags=PoolCreateQueryTests,test_create_and_query """ diff --git a/src/tests/ftest/rebuild/pool_destroy_race.py b/src/tests/ftest/rebuild/pool_destroy_race.py index f225f526f4a..a3bc335d0a9 100644 --- a/src/tests/ftest/rebuild/pool_destroy_race.py +++ b/src/tests/ftest/rebuild/pool_destroy_race.py @@ -1,6 +1,6 @@ """ (C) Copyright 2018-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -35,7 +35,7 @@ def test_pool_destroy_with_io(self): Re-create pool on remaining ranks. :avocado: tags=all,pr - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=pool,rebuild,ior :avocado: tags=RbldPoolDestroyWithIO,test_pool_destroy_with_io """ diff --git a/src/tests/ftest/recovery/check_policy.py b/src/tests/ftest/recovery/check_policy.py index 0ef24dd4c0c..8cd505ab96e 100644 --- a/src/tests/ftest/recovery/check_policy.py +++ b/src/tests/ftest/recovery/check_policy.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -35,7 +35,7 @@ def test_check_policies(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,cat_recov + :avocado: tags=recovery,cat_recov,faults :avocado: tags=DMGCheckPolicyTest,test_check_policies """ # 1. Create a pool. diff --git a/src/tests/ftest/recovery/check_repair.py b/src/tests/ftest/recovery/check_repair.py index 914ce8cb287..734bb4b7e96 100644 --- a/src/tests/ftest/recovery/check_repair.py +++ b/src/tests/ftest/recovery/check_repair.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -31,7 +31,7 @@ def test_check_repair_corner_case(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,cat_recov + :avocado: tags=recovery,cat_recov,faults :avocado: tags=DMGCheckRepairTest,test_check_repair_corner_case """ # 1. Create a pool. diff --git a/src/tests/ftest/recovery/check_start_corner_case.py b/src/tests/ftest/recovery/check_start_corner_case.py index a32cc8cd34b..20363eaea58 100644 --- a/src/tests/ftest/recovery/check_start_corner_case.py +++ b/src/tests/ftest/recovery/check_start_corner_case.py @@ -102,7 +102,7 @@ def test_start_back_to_back(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,cat_recov + :avocado: tags=recovery,cat_recov,faults :avocado: tags=DMGCheckStartCornerCaseTest,test_start_back_to_back """ self.log_step("Create two pools and a container.") @@ -189,7 +189,7 @@ def test_two_pools_healthy(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,cat_recov + :avocado: tags=recovery,cat_recov,faults :avocado: tags=DMGCheckStartCornerCaseTest,test_two_pools_healthy """ # 1. Create three pools and one container. @@ -305,7 +305,7 @@ def test_two_pools_corrupted(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,cat_recov + :avocado: tags=recovery,cat_recov,faults :avocado: tags=DMGCheckStartCornerCaseTest,test_two_pools_corrupted """ # 1. Create three pools and containers. @@ -443,7 +443,7 @@ def test_stale_entry(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,cat_recov + :avocado: tags=recovery,cat_recov,faults :avocado: tags=DMGCheckStartCornerCaseTest,test_stale_entry """ self.log_step("Enable checker while system is Joined and verify the error message.") diff --git a/src/tests/ftest/recovery/check_start_options.py b/src/tests/ftest/recovery/check_start_options.py index fe4c9ee5e20..b486dc32227 100644 --- a/src/tests/ftest/recovery/check_start_options.py +++ b/src/tests/ftest/recovery/check_start_options.py @@ -52,7 +52,7 @@ def test_check_start_reset(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,cat_recov + :avocado: tags=recovery,cat_recov,faults :avocado: tags=DMGCheckStartOptionsTest,test_check_start_reset """ # 1. Create a pool. @@ -168,7 +168,7 @@ def test_check_start_interactive(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,cat_recov + :avocado: tags=recovery,cat_recov,faults :avocado: tags=DMGCheckStartOptionsTest,test_check_start_interactive """ # 1. Create a pool. @@ -288,7 +288,7 @@ def test_check_start_failout(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,cat_recov + :avocado: tags=recovery,cat_recov,faults :avocado: tags=DMGCheckStartOptionsTest,test_check_start_failout """ self.log_step("Create a pool.") @@ -372,7 +372,7 @@ def test_check_start_find_orphans(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,cat_recov + :avocado: tags=recovery,cat_recov,faults :avocado: tags=DMGCheckStartOptionsTest,test_check_start_find_orphans """ # 1. Create a pool and a container. diff --git a/src/tests/ftest/recovery/check_stop.py b/src/tests/ftest/recovery/check_stop.py index c9a911efdb5..6655522ee66 100644 --- a/src/tests/ftest/recovery/check_stop.py +++ b/src/tests/ftest/recovery/check_stop.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -52,7 +52,7 @@ def test_stop_during_repair(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,cat_recov + :avocado: tags=recovery,cat_recov,faults :avocado: tags=DMGCheckStopTest,test_stop_during_repair """ # 1. Create two pools 1 and 2. Create a container in 2. @@ -144,7 +144,7 @@ def test_disable_during_repair(self): :avocado: tags=all,full_regression :avocado: tags=hw,medium - :avocado: tags=recovery,cat_recov + :avocado: tags=recovery,cat_recov,faults :avocado: tags=DMGCheckStopTest,test_disable_during_repair """ # 1. Create a pool and inject fault. diff --git a/src/tests/ftest/server/storage_tiers.py b/src/tests/ftest/server/storage_tiers.py index fad085cd0db..70e5800d1c9 100644 --- a/src/tests/ftest/server/storage_tiers.py +++ b/src/tests/ftest/server/storage_tiers.py @@ -1,6 +1,6 @@ """ (C) Copyright 2020-2023 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP +(C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -29,7 +29,7 @@ def test_tiers(self): Does not test or rely on actual hardware being present. :avocado: tags=all,pr - :avocado: tags=hw,medium + :avocado: tags=cb,medium :avocado: tags=server,storage,storage_tiers :avocado: tags=StorageTiers,test_tiers """ diff --git a/src/tests/ftest/tags.py b/src/tests/ftest/tags.py index b1807126df3..695a2ed06f0 100755 --- a/src/tests/ftest/tags.py +++ b/src/tests/ftest/tags.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ (C) Copyright 2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -17,7 +17,7 @@ THIS_FILE = os.path.realpath(__file__) FTEST_DIR = os.path.dirname(THIS_FILE) MANUAL_TAG = ('manual',) -STAGE_TYPE_TAGS = ('vm', 'hw', 'hw_vmd') +STAGE_TYPE_TAGS = ('vm', 'cb', 'hw', 'hw_vmd') STAGE_SIZE_TAGS = ('medium', 'large') STAGE_FREQUENCY_TAGS = ('all', 'pr', 'daily_regression', 'full_regression') diff --git a/src/tests/ftest/util/dmg_utils.py b/src/tests/ftest/util/dmg_utils.py index e4b51e72c40..5fd46bec0e4 100644 --- a/src/tests/ftest/util/dmg_utils.py +++ b/src/tests/ftest/util/dmg_utils.py @@ -1380,7 +1380,7 @@ def system_stop(self, force=False, ranks=None): def config_generate(self, mgmt_svc_replicas, num_engines=None, scm_only=False, net_class=None, net_provider=None, use_tmpfs_scm=False, - control_metadata_path=None): + control_metadata_path=None, allow_numa_imbalance=False): """Produce a server configuration. Args: @@ -1397,6 +1397,8 @@ def config_generate(self, mgmt_svc_replicas, num_engines=None, scm_only=False, as SCM. Defaults to False. control_metadata_path (str): External directory provided to store control metadata in MD-on-SSD mode. Defaults to None. + allow_numa_imbalance (bool, optional): Whether to allow NUMA imbalance in + the generated config. Returns: CmdResult: Object that contains exit status, stdout, and other @@ -1407,7 +1409,8 @@ def config_generate(self, mgmt_svc_replicas, num_engines=None, scm_only=False, ("config", "generate"), mgmt_svc_replicas=mgmt_svc_replicas, num_engines=num_engines, scm_only=scm_only, net_class=net_class, net_provider=net_provider, use_tmpfs_scm=use_tmpfs_scm, - control_metadata_path=control_metadata_path) + control_metadata_path=control_metadata_path, + allow_numa_imbalance=allow_numa_imbalance) def telemetry_metrics_list(self, host): """List telemetry metrics. diff --git a/src/tests/ftest/util/dmg_utils_base.py b/src/tests/ftest/util/dmg_utils_base.py index 8447115a171..ec185693ebb 100644 --- a/src/tests/ftest/util/dmg_utils_base.py +++ b/src/tests/ftest/util/dmg_utils_base.py @@ -258,6 +258,7 @@ def __init__(self): self.use_tmpfs_scm = FormattedParameter("--use-tmpfs-scm", False) self.control_metadata_path = FormattedParameter( "--control-metadata-path={}", None) + self.allow_numa_imbalance = FormattedParameter("--allow-numa-imbalance", False) class ContSubCommand(CommandWithSubCommand): """Defines an object for the dmg cont sub command.""" diff --git a/src/tests/ftest/util/storage_utils.py b/src/tests/ftest/util/storage_utils.py index 1e231269b97..92ef0d5ba3c 100644 --- a/src/tests/ftest/util/storage_utils.py +++ b/src/tests/ftest/util/storage_utils.py @@ -1,6 +1,6 @@ """ (C) Copyright 2022-2023 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -777,3 +777,19 @@ def _get_interleaved(engines, numa_devices): filter( partial(is_not, None), itertools.chain(*itertools.zip_longest(*numa_devices.values())))) + + +def storage_numa_nodes(test): + """Get the number of unique available storage NUMA nodes + + Args: + test (Test): avocado test object + + Returns: + list: list of unique available storage NUMA nodes + """ + info = StorageInfo(test.log, test.hostlist_servers) + info.scan() + numa_nodes = list(set([device.numa_node for device in info.devices])) + test.log.info("Detected storage NUMA nodes: %s", numa_nodes) + return numa_nodes diff --git a/src/tests/ftest/util/thread_manager.py b/src/tests/ftest/util/thread_manager.py index 6e30c6bdee4..1e93ceeafa3 100644 --- a/src/tests/ftest/util/thread_manager.py +++ b/src/tests/ftest/util/thread_manager.py @@ -1,5 +1,6 @@ """ (C) Copyright 2021-2023 Intel Corporation. +(C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -77,16 +78,19 @@ class ThreadManager(): # pylint: disable=too-many-ancestors,too-few-public-methods """Class to manage running any method as multiple threads.""" - def __init__(self, method, timeout=None): + def __init__(self, method, timeout=None, max_workers=None): """Initialize a ThreadManager object with the the method to run as a thread. Args: method (callable): python method to execute in each thread timeout (int, optional): timeout for all thread execution. Defaults to None. + max_workers (int, optional): maximum number of threads to run concurrently. + Defaults to None. """ self.log = getLogger() self.method = method self.timeout = timeout + self.max_workers = max_workers self.job_kwargs = [] self.futures = {} @@ -112,7 +116,7 @@ def run(self): """ results = [] - with ThreadPoolExecutor() as thread_executor: + with ThreadPoolExecutor(max_workers=self.max_workers) as thread_executor: self.log.info("Submitting %d threads ...", len(self.job_kwargs)) # Keep track of thread ids by assigning an index to each Future object futures = { diff --git a/src/tests/suite/daos_rebuild.c b/src/tests/suite/daos_rebuild.c index 886a2ca6c0f..f51855cb4f5 100644 --- a/src/tests/suite/daos_rebuild.c +++ b/src/tests/suite/daos_rebuild.c @@ -1333,10 +1333,15 @@ rebuild_kill_rank_during_rebuild(void **state) static void rebuild_kill_PS_leader_during_rebuild(void **state) { - test_arg_t *arg = *state; - daos_obj_id_t oids[OBJ_NR]; - d_rank_t leader; - int i; + test_arg_t *arg = *state; + daos_obj_id_t oids[OBJ_NR]; + d_rank_t leader; + d_rank_t non_leader; + int i; + + par_barrier(PAR_COMM_WORLD); + if (arg->myrank != 0) + return; if (!test_runable(arg, 7) || arg->pool.alive_svc->rl_nr < 5) { print_message("need at least 5 svcs, -s5\n"); @@ -1350,25 +1355,30 @@ rebuild_kill_PS_leader_during_rebuild(void **state) oids[i] = dts_oid_set_rank(oids[i], 6); } rebuild_io(arg, oids, OBJ_NR); + non_leader = leader != 6 ? 6 : 5; + print_message("leader=%u non_leader=%u pre_rs_version=%u pre_pool_ver=%u\n", leader, + non_leader, arg->pool.pool_info.pi_rebuild_st.rs_version, + arg->pool.pool_info.pi_map_ver); /* kill non-leader rank */ - if (leader != 6) - daos_kill_server(arg, arg->pool.pool_uuid, arg->group, - arg->pool.alive_svc, 6); - else - daos_kill_server(arg, arg->pool.pool_uuid, arg->group, - arg->pool.alive_svc, 5); - /* hang the rebuild */ - if (arg->myrank == 0) { - daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_LOC, - DAOS_REBUILD_TGT_SCAN_HANG, 0, NULL); - daos_debug_set_params(arg->group, -1, DMG_KEY_FAIL_VALUE, 5, - 0, NULL); - } - sleep(2); + print_message("killing non-leader rank=%u to trigger first rebuild\n", non_leader); + daos_kill_server(arg, arg->pool.pool_uuid, arg->group, arg->pool.alive_svc, non_leader); + + /* Wait for the first rebuild to start scanning */ + test_rebuild_wait_to_scanning_next(&arg, 1); + print_message("first rebuild started: rs_version=%u state=%d errno=%d tobe_obj=" DF_U64 + " rebuilt_obj=" DF_U64 "\n", + arg->pool.pool_info.pi_rebuild_st.rs_version, + arg->pool.pool_info.pi_rebuild_st.rs_state, + arg->pool.pool_info.pi_rebuild_st.rs_errno, + arg->pool.pool_info.pi_rebuild_st.rs_toberb_obj_nr, + arg->pool.pool_info.pi_rebuild_st.rs_obj_nr); + + print_message("killing PS leader rank=%u during active first rebuild\n", leader); rebuild_single_pool_rank(arg, leader, true); sleep(5); + print_message("restart/reintegrate previous PS leader rank=%u\n", leader); reintegrate_single_pool_rank(arg, leader, true); } diff --git a/src/tests/suite/daos_test.h b/src/tests/suite/daos_test.h index 501a943c942..2441fc10a0b 100644 --- a/src/tests/suite/daos_test.h +++ b/src/tests/suite/daos_test.h @@ -423,6 +423,8 @@ test_rebuild_wait_to_start(test_arg_t **args, int args_cnt); void test_rebuild_wait_to_start_next(test_arg_t **args, int args_cnt); void +test_rebuild_wait_to_scanning_next(test_arg_t **args, int args_cnt); +void test_rebuild_wait_to_start_lower(test_arg_t **args, int args_cnt); void test_rebuild_wait_to_error(test_arg_t **args, int args_cnt); diff --git a/src/tests/suite/daos_test_common.c b/src/tests/suite/daos_test_common.c index a28706ed4a4..fff05eb1409 100644 --- a/src/tests/suite/daos_test_common.c +++ b/src/tests/suite/daos_test_common.c @@ -882,6 +882,42 @@ rebuild_pool_started_after_ver(test_arg_t *arg, uint32_t rs_version) } } +/* Determine if pool rebuild has started scanning and rebuild version > rs_version */ +static bool +rebuild_pool_scanning_after_ver(test_arg_t *arg, uint32_t rs_version) +{ + daos_pool_info_t pinfo = {0}; + struct daos_rebuild_status *rst; + int rc; + + pinfo.pi_bits = DPI_REBUILD_STATUS; + rc = test_pool_get_info(arg, &pinfo, NULL /* engine_ranks */); + rst = &pinfo.pi_rebuild_st; + + if (rc != 0) { + print_message("pool query for rebuild status failed, rc=%d, pool " DF_UUIDF "\n", + rc, DP_UUID(arg->pool.pool_uuid)); + return false; + } else { + bool in_progress = (rst->rs_state == DRS_IN_PROGRESS); + bool ver_advanced = (rst->rs_version > rs_version); + bool scanning_seen = (rst->rs_toberb_obj_nr > 0); + + print_message("rebuild for pool " DF_UUIDF + " state=%d rs_version=%u (waiting for > %u), tobe_obj=" DF_U64 + " (waiting for > 0)\n", + DP_UUID(arg->pool.pool_uuid), rst->rs_state, rst->rs_version, + rs_version, rst->rs_toberb_obj_nr); + + if (in_progress && ver_advanced && scanning_seen) { + /* save final pool query info to be able to inspect rebuild status */ + memcpy(&arg->pool.pool_info, &pinfo, sizeof(pinfo)); + return true; + } + return false; + } +} + /* Determine if pool rebuild is busy, and the rebuild version is < rs_version */ static bool rebuild_pool_started_before_ver(test_arg_t *arg, uint32_t rs_version) @@ -1052,6 +1088,24 @@ test_rebuild_started_after(test_arg_t **args, int args_cnt, uint32_t *cur_versio return all_started; } +static bool +test_rebuild_scanning_after(test_arg_t **args, int args_cnt, uint32_t *cur_versions) +{ + bool all_scanning = true; + int i; + + for (i = 0; i < args_cnt; i++) { + bool scanning = true; + + if (!args[i]->pool.destroyed) + scanning = rebuild_pool_scanning_after_ver(args[i], cur_versions[i]); + + if (!scanning) + all_scanning = false; + } + return all_scanning; +} + /* wait until pools start rebuilds with rs_version < current (e.g.,. expecting op:Fail_reclaim) */ void test_rebuild_wait_to_start_lower(test_arg_t **args, int args_cnt) @@ -1094,6 +1148,27 @@ test_rebuild_wait_to_start_next(test_arg_t **args, int args_cnt) D_FREE(cur_versions); } +/* wait until pools start rebuilds with rs_version > current and scanning activity is observed */ +void +test_rebuild_wait_to_scanning_next(test_arg_t **args, int args_cnt) +{ + uint32_t *cur_versions; + int i; + + D_ALLOC_ARRAY(cur_versions, args_cnt); + assert_true(cur_versions != NULL); + for (i = 0; i < args_cnt; i++) + cur_versions[i] = args[i]->pool.pool_info.pi_rebuild_st.rs_version; + + while (!test_rebuild_scanning_after(args, args_cnt, cur_versions)) + sleep(2); + + /* NB: when control reaches here, each pool's current rs_version has been updated + * (for subsequent calls that will rely on it as a baseline) + */ + D_FREE(cur_versions); +} + /* wait until pools start rebuilds with any rs_version > 0 (whatever is current) */ void test_rebuild_wait_to_start(test_arg_t **args, int args_cnt) diff --git a/src/utils/ddb/ddb_vos.c b/src/utils/ddb/ddb_vos.c index c8a84f99351..2f748ec9ca3 100644 --- a/src/utils/ddb/ddb_vos.c +++ b/src/utils/ddb/ddb_vos.c @@ -2311,7 +2311,8 @@ dv_run_prov_mem(const char *db_path, const char *tmpfs_mount, unsigned int tmpfs md_on_ssd = bio_nvme_configured(SMD_DEV_TYPE_META); if (!md_on_ssd) { - D_ERROR("Not in MD-on-SSD mode; skipping memory environment provisioning."); + D_ERROR("Provided db_path is not configured in MD-on-SSD mode."); + rc = -DER_INVAL; goto out; } diff --git a/utils/rpms/daos.changelog b/utils/rpms/daos.changelog index 6ce0f163a31..2c42b637601 100644 --- a/utils/rpms/daos.changelog +++ b/utils/rpms/daos.changelog @@ -1,4 +1,7 @@ %changelog +* Wed Jul 08 2026 Michael Hennecke 2.9.100-5 +- Add a /var/daos directory writable for daos_server + * Mon May 04 2026 Cedric Koch-Hofer 2.9.100-4 - Add gperftools-devel as a build dependency of daos-devel diff --git a/utils/rpms/daos.sh b/utils/rpms/daos.sh index a71991cc525..d4038190d55 100755 --- a/utils/rpms/daos.sh +++ b/utils/rpms/daos.sh @@ -19,6 +19,7 @@ daoshome="${prefix}/lib/daos" server_svc_name="daos_server.service" agent_svc_name="daos_agent.service" sysctl_script_name="10-daos_server.conf" +daos_sys_dir="/var/daos" daos_log_dir="/var/log/daos" VERSION=${daos_version} @@ -157,6 +158,12 @@ getent group daos_metrics >/dev/null || groupadd -r daos_metrics getent group daos_server >/dev/null || groupadd -r daos_server getent group daos_daemons >/dev/null || groupadd -r daos_daemons getent passwd daos_server >/dev/null || useradd -s /sbin/nologin -r -g daos_server -G daos_metrics,daos_daemons daos_server +# Ensure daos_sys_dir exists +if [ ! -d ${daos_sys_dir} ]; then + mkdir -p ${daos_sys_dir} + chown daos_server:daos_daemons ${daos_sys_dir} + chmod 775 ${daos_sys_dir} +fi # Ensure daos_log_dir exists if [ ! -d ${daos_log_dir} ]; then mkdir -p ${daos_log_dir} @@ -310,23 +317,33 @@ if [ ! -d ${daos_log_dir} ]; then chmod 775 ${daos_log_dir} fi EOF +chmod +x "${tmp}/pre_install_client" EXTRA_OPTS+=("--before-install" "${tmp}/pre_install_client") cat << EOF > "${tmp}/post_install_client" +#!/bin/bash +set -x +ldconfig systemctl --no-reload preset daos_agent.service &>/dev/null || : EOF +chmod +x "${tmp}/post_install_client" EXTRA_OPTS+=("--after-install" "${tmp}/post_install_client") cat << EOF > "${tmp}/pre_uninstall_client" systemctl --no-reload disable --now daos_agent.service >& /dev/null || : EOF +chmod +x "${tmp}/pre_uninstall_client" EXTRA_OPTS+=("--before-remove" "${tmp}/pre_uninstall_client") if [[ "${DISTRO:-el8}" =~ suse ]]; then cat << EOF > "${tmp}/post_uninstall_client" +#!/bin/bash +set -x +ldconfig rm -f "/var/lib/systemd/migrated/daos_agent.service" || : /usr/bin/systemctl daemon-reload || : EOF + chmod +x "${tmp}/post_uninstall_client" EXTRA_OPTS+=("--after-remove" "${tmp}/post_uninstall_client") fi diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 9adc6ec0bbc..e559f8394a3 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -14,6 +14,7 @@ %global mercury_version 2.4 %global argobots_version 1.2 %global __python %{__python3} +%global daos_sys_dir "/var/daos" %global daos_log_dir "/var/log/daos" %if (0%{?rhel} >= 8) @@ -24,7 +25,7 @@ Name: daos Version: 2.9.100 -Release: 4%{?relval}%{?dist} +Release: 5%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -407,6 +408,12 @@ getent group daos_metrics >/dev/null || groupadd -r daos_metrics getent group daos_server >/dev/null || groupadd -r daos_server getent group daos_daemons >/dev/null || groupadd -r daos_daemons getent passwd daos_server >/dev/null || useradd -s /sbin/nologin -r -g daos_server -G daos_metrics,daos_daemons daos_server +# Ensure daos_sys_dir exists +if [ ! -d %{daos_sys_dir} ]; then + mkdir -p %{daos_sys_dir} + chown daos_server:daos_daemons %{daos_sys_dir} + chmod 775 %{daos_sys_dir} +fi # Ensure daos_log_dir exists if [ ! -d %{daos_log_dir} ]; then mkdir -p %{daos_log_dir}