From 3dd9d25d3e645237dc476f2b712e02fb22cde934 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Mon, 22 Jun 2026 12:12:16 +0000 Subject: [PATCH 01/20] DAOS-19207 control: check redundancy factor early Signed-off-by: Jan Michalski --- src/control/fault/code/codes.go | 1 + src/control/server/faults.go | 11 ++++++- src/control/server/mgmt_pool.go | 14 ++++++++- src/control/server/mgmt_pool_test.go | 46 +++++++++++++++++++++++++++- 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/src/control/fault/code/codes.go b/src/control/fault/code/codes.go index bb07f0b89ed..f6e7800a49e 100644 --- a/src/control/fault/code/codes.go +++ b/src/control/fault/code/codes.go @@ -161,6 +161,7 @@ const ( ServerPoolHasContainers ServerPoolMemRatioNoRoles ServerBadFaultDomainLabels + ServerPoolTooFewFaultDomains ServerJoinReplaceEnabledPoolRank ServerRankAdminExcluded ServerTransparentHugepageEnabled diff --git a/src/control/server/faults.go b/src/control/server/faults.go index a2e51760026..4359d06f1a6 100644 --- a/src/control/server/faults.go +++ b/src/control/server/faults.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -174,6 +174,15 @@ var FaultPoolMemRatioNoRoles = serverFault( "either remove MD-on-SSD-specific options from the command request or set bdev_roles in "+ "server config file to enable MD-on-SSD") +func FaultPoolTooFewFaultDomains(rdFac int, numDomains int) *fault.Fault { + return serverFault( + code.ServerPoolTooFewFaultDomains, + fmt.Sprintf("pool redundancy factor %d requires at least %d fault domains but only %d are available", + rdFac, rdFac+1, numDomains), + "retry the request with a lower redundancy factor or add more fault domains", + ) +} + func FaultBadFaultDomainLabels(faultPath, addr string, reqLabels, systemLabels []string) *fault.Fault { return serverFault( code.ServerBadFaultDomainLabels, diff --git a/src/control/server/mgmt_pool.go b/src/control/server/mgmt_pool.go index dbe5f0d5ae9..48d5545ef3e 100644 --- a/src/control/server/mgmt_pool.go +++ b/src/control/server/mgmt_pool.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -420,6 +420,18 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq return nil, err } + // Check if the requested redundancy factor can be met with the number of supplied fault domains. + for _, prop := range req.GetProperties() { + if prop.GetNumber() == uint32(daos.PoolPropertyRedunFac) { + rdFac := int(prop.GetNumval()) + domainNr := len(req.FaultDomains) + if rdFac+1 > domainNr { + return nil, FaultPoolTooFewFaultDomains(rdFac, domainNr) + } + break + } + } + if err := svc.calculateCreateStorage(req); err != nil { return nil, err } diff --git a/src/control/server/mgmt_pool_test.go b/src/control/server/mgmt_pool_test.go index 51a3bdc2813..2f2e2747fed 100644 --- a/src/control/server/mgmt_pool_test.go +++ b/src/control/server/mgmt_pool_test.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -118,6 +118,23 @@ func testPoolLabelProp() []*mgmtpb.PoolProperty { } } +func testPoolRedunFacProp() []*mgmtpb.PoolProperty { + return []*mgmtpb.PoolProperty{ + { + Number: daos.PoolPropertyLabel, + Value: &mgmtpb.PoolProperty_Strval{ + Strval: "test", + }, + }, + { + Number: daos.PoolPropertyRedunFac, + Value: &mgmtpb.PoolProperty_Numval{ + Numval: 2, + }, + }, + } +} + func TestServer_MgmtSvc_PoolCreateAlreadyExists(t *testing.T) { for name, tc := range map[string]struct { state system.PoolServiceState @@ -635,6 +652,33 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { }, expErr: FaultPoolNoLabel, }, + "failed creation too few fault domains": { + targetCount: 1, + req: &mgmtpb.PoolCreateReq{ + Uuid: test.MockUUID(1), + TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, + Ranks: []uint32{0}, + Properties: testPoolRedunFacProp(), + }, + expErr: FaultPoolTooFewFaultDomains(2, 1), + }, + "successful creation with rd_fac": { + targetCount: 2, + req: &mgmtpb.PoolCreateReq{ + Uuid: test.MockUUID(1), + TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, + Ranks: []uint32{0, 1}, + Properties: testPoolRedunFacProp(), + }, + drpcRet: &mgmtpb.PoolCreateResp{ + TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, + TgtRanks: []uint32{0, 1}, + }, + expResp: &mgmtpb.PoolCreateResp{ + TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, + TgtRanks: []uint32{0, 1}, + }, + }, } { t.Run(name, func(t *testing.T) { buf.Reset() From 0ede7a676d3a04d2f2c8c4f5710219f316d57b54 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Tue, 23 Jun 2026 19:24:41 +0000 Subject: [PATCH 02/20] DAOS-19207 control: fix domainNr calculation Signed-off-by: Jan Michalski --- src/control/server/mgmt_pool.go | 15 ++++++++------- src/control/system/membership.go | 4 ++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/control/server/mgmt_pool.go b/src/control/server/mgmt_pool.go index 48d5545ef3e..0cbf5a9c26a 100644 --- a/src/control/server/mgmt_pool.go +++ b/src/control/server/mgmt_pool.go @@ -414,17 +414,11 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq return nil, FaultPoolInvalidServiceReps(maxSvcReps) } - // IO engine needs the fault domain tree for placement purposes - req.FaultDomains, err = svc.membership.CompressedFaultDomainTree(req.Ranks...) - if err != nil { - return nil, err - } - // Check if the requested redundancy factor can be met with the number of supplied fault domains. for _, prop := range req.GetProperties() { if prop.GetNumber() == uint32(daos.PoolPropertyRedunFac) { rdFac := int(prop.GetNumval()) - domainNr := len(req.FaultDomains) + domainNr := svc.membership.DomainNr() if rdFac+1 > domainNr { return nil, FaultPoolTooFewFaultDomains(rdFac, domainNr) } @@ -432,6 +426,13 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq } } + // IO engine needs the fault domain tree for placement purposes + svc.log.Debugf("req.Ranks: %v", req.Ranks) + req.FaultDomains, err = svc.membership.CompressedFaultDomainTree(req.Ranks...) + if err != nil { + return nil, err + } + if err := svc.calculateCreateStorage(req); err != nil { return nil, err } diff --git a/src/control/system/membership.go b/src/control/system/membership.go index 691243c0682..06befa8b8cf 100644 --- a/src/control/system/membership.go +++ b/src/control/system/membership.go @@ -785,6 +785,10 @@ func (m *Membership) CompressedFaultDomainTree(ranks ...uint32) ([]uint32, error return append([]uint32{md}, compressTree(subtree)...), nil } +func (m *Membership) DomainNr() int { + return len(m.db.FaultDomainTree().Domains()) +} + const ( DomTreeMetadataHasFaultDom uint32 = (1 << iota) DomTreeMetadataHasPerfDom From 6edfafd011c43884f261cbf04eb5cd596ed0c758 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Tue, 23 Jun 2026 19:55:07 +0000 Subject: [PATCH 03/20] DAOS-19207 control: add and update tests Signed-off-by: Jan Michalski --- src/control/server/mgmt_pool_test.go | 4 ++-- src/control/system/membership_test.go | 29 ++++++++++++++++++++------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/control/server/mgmt_pool_test.go b/src/control/server/mgmt_pool_test.go index 2f2e2747fed..13035d89ba5 100644 --- a/src/control/server/mgmt_pool_test.go +++ b/src/control/server/mgmt_pool_test.go @@ -129,7 +129,7 @@ func testPoolRedunFacProp() []*mgmtpb.PoolProperty { { Number: daos.PoolPropertyRedunFac, Value: &mgmtpb.PoolProperty_Numval{ - Numval: 2, + Numval: 1, }, }, } @@ -660,7 +660,7 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { Ranks: []uint32{0}, Properties: testPoolRedunFacProp(), }, - expErr: FaultPoolTooFewFaultDomains(2, 1), + expErr: FaultPoolTooFewFaultDomains(1, 2), }, "successful creation with rd_fac": { targetCount: 2, diff --git a/src/control/system/membership_test.go b/src/control/system/membership_test.go index 7d3e2db8473..4112e3ba4f4 100644 --- a/src/control/system/membership_test.go +++ b/src/control/system/membership_test.go @@ -1299,13 +1299,15 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { } for name, tc := range map[string]struct { - tree *FaultDomainTree - inputRanks []uint32 - expResult []uint32 - expErr error + tree *FaultDomainTree + inputRanks []uint32 + expResult []uint32 + expErr error + expDomainNr int }{ "nil tree": { - expErr: errors.New("uninitialized fault domain tree"), + expErr: errors.New("uninitialized fault domain tree"), + expDomainNr: 0, }, "root only": { tree: NewFaultDomainTree(), @@ -1315,6 +1317,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { ExpFaultDomainID(0), 0, }, + expDomainNr: 0, }, "single branch, no rank leaves": { tree: NewFaultDomainTree( @@ -1335,6 +1338,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { ExpFaultDomainID(3), 0, }, + expDomainNr: 1, }, "multi branch, no rank leaves": { tree: NewFaultDomainTree( @@ -1374,6 +1378,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { ExpFaultDomainID(8), 0, }, + expDomainNr: 5, }, "single branch with rank leaves": { tree: NewFaultDomainTree( @@ -1395,6 +1400,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { 1, 5, }, + expDomainNr: 1, }, "multi branch with rank leaves": { tree: NewFaultDomainTree( @@ -1442,6 +1448,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { 4, 5, }, + expDomainNr: 6, }, "intermediate domain has name like rank": { tree: NewFaultDomainTree( @@ -1463,6 +1470,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { 1, 1, // rank }, + expDomainNr: 1, }, "request one rank with node only": { tree: NewFaultDomainTree( @@ -1485,6 +1493,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { // ranks 4, }, + expDomainNr: 6, }, "request one rank": { tree: NewFaultDomainTree( @@ -1510,6 +1519,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { // ranks 4, }, + expDomainNr: 6, }, "request multiple ranks": { tree: NewFaultDomainTree( @@ -1550,6 +1560,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { 4, 5, }, + expDomainNr: 6, }, "request nonexistent rank": { tree: NewFaultDomainTree( @@ -1560,8 +1571,9 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { rankDomain("/rack1/pdu3", 4), rankDomain("/rack2/pdu4", 5), ), - inputRanks: []uint32{4, 0, 5, 3, 100}, - expErr: errors.New("rank 100 not found"), + inputRanks: []uint32{4, 0, 5, 3, 100}, + expErr: errors.New("rank 100 not found"), + expDomainNr: 6, }, } { t.Run(name, func(t *testing.T) { @@ -1575,6 +1587,9 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { test.CmpErr(t, tc.expErr, err) + domainNr := membership.DomainNr() + test.AssertEqual(t, tc.expDomainNr, domainNr, "unexpected domain number") + if diff := cmp.Diff(tc.expResult, result); diff != "" { t.Fatalf("(-want, +got): %s", diff) } From ff858017da5104e052fa4a9711c897bcc20914e8 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Tue, 23 Jun 2026 20:04:38 +0000 Subject: [PATCH 04/20] DAOS-19207 control: remove debug line Signed-off-by: Jan Michalski --- src/control/server/mgmt_pool.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/control/server/mgmt_pool.go b/src/control/server/mgmt_pool.go index 0cbf5a9c26a..13d692012bf 100644 --- a/src/control/server/mgmt_pool.go +++ b/src/control/server/mgmt_pool.go @@ -427,7 +427,6 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq } // IO engine needs the fault domain tree for placement purposes - svc.log.Debugf("req.Ranks: %v", req.Ranks) req.FaultDomains, err = svc.membership.CompressedFaultDomainTree(req.Ranks...) if err != nil { return nil, err From 0dc5c08637a897d68499b3e4366309138a83b904 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Wed, 24 Jun 2026 12:52:46 +0000 Subject: [PATCH 05/20] DAOS-19207 control: count domains for a subset of ranks only Signed-off-by: Jan Michalski --- src/control/server/mgmt_pool.go | 5 ++++- src/control/system/membership.go | 14 ++++++++++++-- src/control/system/membership_test.go | 11 ++++++----- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/control/server/mgmt_pool.go b/src/control/server/mgmt_pool.go index 13d692012bf..ad38742f5c6 100644 --- a/src/control/server/mgmt_pool.go +++ b/src/control/server/mgmt_pool.go @@ -415,10 +415,13 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq } // Check if the requested redundancy factor can be met with the number of supplied fault domains. + domainNr, err := svc.membership.DomainNr(req.Ranks...) + if err != nil { + return nil, err + } for _, prop := range req.GetProperties() { if prop.GetNumber() == uint32(daos.PoolPropertyRedunFac) { rdFac := int(prop.GetNumval()) - domainNr := svc.membership.DomainNr() if rdFac+1 > domainNr { return nil, FaultPoolTooFewFaultDomains(rdFac, domainNr) } diff --git a/src/control/system/membership.go b/src/control/system/membership.go index 06befa8b8cf..730ec6401bf 100644 --- a/src/control/system/membership.go +++ b/src/control/system/membership.go @@ -785,8 +785,18 @@ func (m *Membership) CompressedFaultDomainTree(ranks ...uint32) ([]uint32, error return append([]uint32{md}, compressTree(subtree)...), nil } -func (m *Membership) DomainNr() int { - return len(m.db.FaultDomainTree().Domains()) +func (m *Membership) DomainNr(ranks ...uint32) (int, error) { + tree := m.db.FaultDomainTree() + if tree == nil { + return 0, errors.New("uninitialized fault domain tree") + } + + subtree, err := getFaultDomainSubtree(tree, ranks...) + if err != nil { + return 0, err + } + + return len(subtree.Domains()), nil } const ( diff --git a/src/control/system/membership_test.go b/src/control/system/membership_test.go index 4112e3ba4f4..43bcf01e390 100644 --- a/src/control/system/membership_test.go +++ b/src/control/system/membership_test.go @@ -1493,7 +1493,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { // ranks 4, }, - expDomainNr: 6, + expDomainNr: 1, }, "request one rank": { tree: NewFaultDomainTree( @@ -1519,7 +1519,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { // ranks 4, }, - expDomainNr: 6, + expDomainNr: 1, }, "request multiple ranks": { tree: NewFaultDomainTree( @@ -1560,7 +1560,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { 4, 5, }, - expDomainNr: 6, + expDomainNr: 4, }, "request nonexistent rank": { tree: NewFaultDomainTree( @@ -1573,7 +1573,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { ), inputRanks: []uint32{4, 0, 5, 3, 100}, expErr: errors.New("rank 100 not found"), - expDomainNr: 6, + expDomainNr: 0, }, } { t.Run(name, func(t *testing.T) { @@ -1587,7 +1587,8 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { test.CmpErr(t, tc.expErr, err) - domainNr := membership.DomainNr() + domainNr, err := membership.DomainNr(tc.inputRanks...) + test.CmpErr(t, tc.expErr, err) test.AssertEqual(t, tc.expDomainNr, domainNr, "unexpected domain number") if diff := cmp.Diff(tc.expResult, result); diff != "" { From 1ec4d05155607b0a3b1e99eb523d747c8150dae7 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Wed, 24 Jun 2026 13:00:23 +0000 Subject: [PATCH 06/20] DAOS-19207 control: cleanup a test Signed-off-by: Jan Michalski --- src/control/system/membership_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/control/system/membership_test.go b/src/control/system/membership_test.go index 43bcf01e390..7f284035363 100644 --- a/src/control/system/membership_test.go +++ b/src/control/system/membership_test.go @@ -1306,8 +1306,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { expDomainNr int }{ "nil tree": { - expErr: errors.New("uninitialized fault domain tree"), - expDomainNr: 0, + expErr: errors.New("uninitialized fault domain tree"), }, "root only": { tree: NewFaultDomainTree(), @@ -1571,9 +1570,8 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { rankDomain("/rack1/pdu3", 4), rankDomain("/rack2/pdu4", 5), ), - inputRanks: []uint32{4, 0, 5, 3, 100}, - expErr: errors.New("rank 100 not found"), - expDomainNr: 0, + inputRanks: []uint32{4, 0, 5, 3, 100}, + expErr: errors.New("rank 100 not found"), }, } { t.Run(name, func(t *testing.T) { From 3a2d4f2513ae895f6a2d5ccc94733f934e74d88b Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Fri, 26 Jun 2026 15:16:14 +0000 Subject: [PATCH 07/20] DAOS-19207 control: cleanup test Signed-off-by: Jan Michalski --- src/control/system/membership_test.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/control/system/membership_test.go b/src/control/system/membership_test.go index 7f284035363..dda91aa254e 100644 --- a/src/control/system/membership_test.go +++ b/src/control/system/membership_test.go @@ -1582,16 +1582,14 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { membership := NewMembership(log, db) result, err := membership.CompressedFaultDomainTree(tc.inputRanks...) - - test.CmpErr(t, tc.expErr, err) - - domainNr, err := membership.DomainNr(tc.inputRanks...) test.CmpErr(t, tc.expErr, err) - test.AssertEqual(t, tc.expDomainNr, domainNr, "unexpected domain number") - if diff := cmp.Diff(tc.expResult, result); diff != "" { t.Fatalf("(-want, +got): %s", diff) } + + domainNr, err := membership.DomainNr(log, tc.inputRanks...) + test.CmpErr(t, tc.expErr, err) + test.AssertEqual(t, tc.expDomainNr, domainNr, "unexpected domain number") }) } } From 66110033592943f1ca32fa68eba59b940bbc920c Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Fri, 26 Jun 2026 19:16:01 +0000 Subject: [PATCH 08/20] DAOS-19207 control: fix fault domain calc Signed-off-by: Jan Michalski --- src/control/system/membership.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/control/system/membership.go b/src/control/system/membership.go index 730ec6401bf..4eb8b0f1ead 100644 --- a/src/control/system/membership.go +++ b/src/control/system/membership.go @@ -796,6 +796,21 @@ func (m *Membership) DomainNr(ranks ...uint32) (int, error) { return 0, err } + // TODO DAOS-6353: Properly detect when fault and perf domain are requested. + // Currently any depth greater than the minimum must indicate a performance domain. + minDepth := 2 // domain + rank + if tree.Depth() > minDepth { + // Loop over the children of the root and sum up the number their children. + sum := 0 + for _, child := range subtree.Children { + sum += len(child.Children) + } + return sum, nil + } else { + // There are no perf domains, so all children of the root are fault domains. + return len(subtree.Children), nil + } + return len(subtree.Domains()), nil } From bbe516920903da33b7f0349e9268aa82624d43a4 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Mon, 29 Jun 2026 18:22:18 +0000 Subject: [PATCH 09/20] DAOS-19207 control: fix tests + small adjustments Signed-off-by: Jan Michalski --- src/control/system/membership.go | 6 ++-- src/control/system/membership_test.go | 46 ++------------------------- 2 files changed, 5 insertions(+), 47 deletions(-) diff --git a/src/control/system/membership.go b/src/control/system/membership.go index 4eb8b0f1ead..01b221307f0 100644 --- a/src/control/system/membership.go +++ b/src/control/system/membership.go @@ -799,7 +799,7 @@ func (m *Membership) DomainNr(ranks ...uint32) (int, error) { // TODO DAOS-6353: Properly detect when fault and perf domain are requested. // Currently any depth greater than the minimum must indicate a performance domain. minDepth := 2 // domain + rank - if tree.Depth() > minDepth { + if subtree.Depth() > minDepth { // Loop over the children of the root and sum up the number their children. sum := 0 for _, child := range subtree.Children { @@ -807,11 +807,9 @@ func (m *Membership) DomainNr(ranks ...uint32) (int, error) { } return sum, nil } else { - // There are no perf domains, so all children of the root are fault domains. + // There are no perf domains, so the children of the root are fault domains. return len(subtree.Children), nil } - - return len(subtree.Domains()), nil } const ( diff --git a/src/control/system/membership_test.go b/src/control/system/membership_test.go index dda91aa254e..9a1c907c436 100644 --- a/src/control/system/membership_test.go +++ b/src/control/system/membership_test.go @@ -1339,46 +1339,6 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { }, expDomainNr: 1, }, - "multi branch, no rank leaves": { - tree: NewFaultDomainTree( - MustCreateFaultDomainFromString("/rack0/pdu0"), - MustCreateFaultDomainFromString("/rack0/pdu1"), - MustCreateFaultDomainFromString("/rack1/pdu2"), - MustCreateFaultDomainFromString("/rack1/pdu3"), - MustCreateFaultDomainFromString("/rack2/pdu4"), - ), - expResult: []uint32{ - 0, // metadata - 2, // root - ExpFaultDomainID(0), - 3, - 1, // rack0 - ExpFaultDomainID(1), - 2, - 1, // rack1 - ExpFaultDomainID(4), - 2, - 1, // rack2 - ExpFaultDomainID(7), - 1, - 0, // pdu0 - ExpFaultDomainID(2), - 0, - 0, // pdu1 - ExpFaultDomainID(3), - 0, - 0, // pdu2 - ExpFaultDomainID(5), - 0, - 0, // pdu3 - ExpFaultDomainID(6), - 0, - 0, // pdu4 - ExpFaultDomainID(8), - 0, - }, - expDomainNr: 5, - }, "single branch with rank leaves": { tree: NewFaultDomainTree( rankDomain("/one/two/three", 5), @@ -1447,7 +1407,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { 4, 5, }, - expDomainNr: 6, + expDomainNr: 5, }, "intermediate domain has name like rank": { tree: NewFaultDomainTree( @@ -1559,7 +1519,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { 4, 5, }, - expDomainNr: 4, + expDomainNr: 3, }, "request nonexistent rank": { tree: NewFaultDomainTree( @@ -1587,7 +1547,7 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { t.Fatalf("(-want, +got): %s", diff) } - domainNr, err := membership.DomainNr(log, tc.inputRanks...) + domainNr, err := membership.DomainNr(tc.inputRanks...) test.CmpErr(t, tc.expErr, err) test.AssertEqual(t, tc.expDomainNr, domainNr, "unexpected domain number") }) From a1931215563e7a6c0ed34a8bddfcf3a3b8544b6f Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Mon, 29 Jun 2026 18:37:19 +0000 Subject: [PATCH 10/20] DAOS-19207 control: fix TestServer_MgmtSvc_PoolCreate Signed-off-by: Jan Michalski --- src/control/server/mgmt_pool_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control/server/mgmt_pool_test.go b/src/control/server/mgmt_pool_test.go index 13035d89ba5..fca3d07a4f9 100644 --- a/src/control/server/mgmt_pool_test.go +++ b/src/control/server/mgmt_pool_test.go @@ -660,7 +660,7 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { Ranks: []uint32{0}, Properties: testPoolRedunFacProp(), }, - expErr: FaultPoolTooFewFaultDomains(1, 2), + expErr: FaultPoolTooFewFaultDomains(1, 1), }, "successful creation with rd_fac": { targetCount: 2, From 98271876d9dd5aa4373ded49f7b88bde83c1f7eb Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Mon, 29 Jun 2026 18:52:44 +0000 Subject: [PATCH 11/20] DAOS-19207 control: add DomainNr comment Signed-off-by: Jan Michalski --- src/control/system/membership.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/control/system/membership.go b/src/control/system/membership.go index 01b221307f0..e65834fe2b0 100644 --- a/src/control/system/membership.go +++ b/src/control/system/membership.go @@ -785,6 +785,11 @@ func (m *Membership) CompressedFaultDomainTree(ranks ...uint32) ([]uint32, error return append([]uint32{md}, compressTree(subtree)...), nil } +// DomainNr returns the number of fault domains in the subtree of the domain +// tree specified by the given ranks. +// If no ranks are provided, the entire tree is considered. +// +// Note: Do not confuse fault domains with the FaultDomain struct. func (m *Membership) DomainNr(ranks ...uint32) (int, error) { tree := m.db.FaultDomainTree() if tree == nil { From d01d07d5fc024f4bed8942f593e5c3bf2e8de0c9 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Fri, 3 Jul 2026 16:59:27 +0000 Subject: [PATCH 12/20] DAOS-19207 control: introduce the FaultDomainLevel method... ... clean up unit tests. Signed-off-by: Jan Michalski --- src/control/server/mgmt_pool.go | 6 +- src/control/system/membership.go | 59 ++++--- src/control/system/membership_test.go | 216 ++++++++++++++++++++++---- 3 files changed, 236 insertions(+), 45 deletions(-) diff --git a/src/control/server/mgmt_pool.go b/src/control/server/mgmt_pool.go index ad38742f5c6..ca71b96766d 100644 --- a/src/control/server/mgmt_pool.go +++ b/src/control/server/mgmt_pool.go @@ -415,7 +415,11 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq } // Check if the requested redundancy factor can be met with the number of supplied fault domains. - domainNr, err := svc.membership.DomainNr(req.Ranks...) + level, err := svc.membership.FaultDomainLevel() + if err != nil { + return nil, err + } + domainNr, err := svc.membership.DomainNr(level, req.Ranks...) if err != nil { return nil, err } diff --git a/src/control/system/membership.go b/src/control/system/membership.go index 5dbf3ef49ff..ef75e7dd679 100644 --- a/src/control/system/membership.go +++ b/src/control/system/membership.go @@ -785,12 +785,44 @@ func (m *Membership) CompressedFaultDomainTree(ranks ...uint32) ([]uint32, error return append([]uint32{md}, compressTree(subtree)...), nil } -// DomainNr returns the number of fault domains in the subtree of the domain -// tree specified by the given ranks. +// FaultDomainLevel returns the fault domain level of the domain tree. +// It assumes the tree is balanced and that the rank level is present. +// So, the fault domain level is the second to last level of the tree. +func (m *Membership) FaultDomainLevel() (int, error) { + tree := m.db.FaultDomainTree() + if tree == nil { + return 0, errors.New("uninitialized fault domain tree") + } + + depth := tree.Depth() + if depth < 2 { + return 0, errors.New("domain tree has no fault domain level") + } + + return depth - 2, nil +} + +// domainNrAtLevel returns the number of domains in the tree at the given level. +// It traverses the tree recursively to reach the specified level. +func domainNrAtLevel(tree *FaultDomainTree, level int) int { + if tree == nil { + return 0 + } + if level == 0 { + return len(tree.Children) + } + + count := 0 + for _, child := range tree.Children { + count += domainNrAtLevel(child, level-1) + } + return count +} + +// DomainNr returns the number of domains in the subtree of the domain tree +// specified by the given ranks at the given level. // If no ranks are provided, the entire tree is considered. -// -// Note: Do not confuse fault domains with the FaultDomain struct. -func (m *Membership) DomainNr(ranks ...uint32) (int, error) { +func (m *Membership) DomainNr(level int, ranks ...uint32) (int, error) { tree := m.db.FaultDomainTree() if tree == nil { return 0, errors.New("uninitialized fault domain tree") @@ -801,20 +833,11 @@ func (m *Membership) DomainNr(ranks ...uint32) (int, error) { return 0, err } - // TODO DAOS-6353: Properly detect when fault and perf domain are requested. - // Currently any depth greater than the minimum must indicate a performance domain. - minDepth := 2 // domain + rank - if subtree.Depth() > minDepth { - // Loop over the children of the root and sum up the number their children. - sum := 0 - for _, child := range subtree.Children { - sum += len(child.Children) - } - return sum, nil - } else { - // There are no perf domains, so the children of the root are fault domains. - return len(subtree.Children), nil + if level >= subtree.Depth() { + return 0, errors.Errorf("level %d >= subtree depth %d", level, subtree.Depth()) } + + return domainNrAtLevel(subtree, level), nil } const ( diff --git a/src/control/system/membership_test.go b/src/control/system/membership_test.go index 9a1c907c436..05bf463307a 100644 --- a/src/control/system/membership_test.go +++ b/src/control/system/membership_test.go @@ -1284,26 +1284,21 @@ func TestSystem_Membership_MarkDead(t *testing.T) { } } -func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { - testMemberWithFaultDomain := func(rank Rank, faultDomain *FaultDomain) *Member { - return &Member{ - Rank: rank, - FaultDomain: faultDomain, - } - } - - rankDomain := func(parent string, rank uint32) *FaultDomain { - parentFd := MustCreateFaultDomainFromString(parent) - member := testMemberWithFaultDomain(Rank(rank), parentFd) - return MemberFaultDomain(member) +func rankDomain(parent string, rank uint32) *FaultDomain { + parentFd := MustCreateFaultDomainFromString(parent) + member := &Member{ + Rank: Rank(rank), + FaultDomain: parentFd, } + return MemberFaultDomain(member) +} +func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { for name, tc := range map[string]struct { - tree *FaultDomainTree - inputRanks []uint32 - expResult []uint32 - expErr error - expDomainNr int + tree *FaultDomainTree + inputRanks []uint32 + expResult []uint32 + expErr error }{ "nil tree": { expErr: errors.New("uninitialized fault domain tree"), @@ -1316,7 +1311,6 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { ExpFaultDomainID(0), 0, }, - expDomainNr: 0, }, "single branch, no rank leaves": { tree: NewFaultDomainTree( @@ -1337,7 +1331,45 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { ExpFaultDomainID(3), 0, }, - expDomainNr: 1, + }, + "multi branch, no rank leaves": { + tree: NewFaultDomainTree( + MustCreateFaultDomainFromString("/rack0/pdu0"), + MustCreateFaultDomainFromString("/rack0/pdu1"), + MustCreateFaultDomainFromString("/rack1/pdu2"), + MustCreateFaultDomainFromString("/rack1/pdu3"), + MustCreateFaultDomainFromString("/rack2/pdu4"), + ), + expResult: []uint32{ + 0, // metadata + 2, // root + ExpFaultDomainID(0), + 3, + 1, // rack0 + ExpFaultDomainID(1), + 2, + 1, // rack1 + ExpFaultDomainID(4), + 2, + 1, // rack2 + ExpFaultDomainID(7), + 1, + 0, // pdu0 + ExpFaultDomainID(2), + 0, + 0, // pdu1 + ExpFaultDomainID(3), + 0, + 0, // pdu2 + ExpFaultDomainID(5), + 0, + 0, // pdu3 + ExpFaultDomainID(6), + 0, + 0, // pdu4 + ExpFaultDomainID(8), + 0, + }, }, "single branch with rank leaves": { tree: NewFaultDomainTree( @@ -1359,7 +1391,6 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { 1, 5, }, - expDomainNr: 1, }, "multi branch with rank leaves": { tree: NewFaultDomainTree( @@ -1407,7 +1438,6 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { 4, 5, }, - expDomainNr: 5, }, "intermediate domain has name like rank": { tree: NewFaultDomainTree( @@ -1429,7 +1459,6 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { 1, 1, // rank }, - expDomainNr: 1, }, "request one rank with node only": { tree: NewFaultDomainTree( @@ -1452,7 +1481,6 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { // ranks 4, }, - expDomainNr: 1, }, "request one rank": { tree: NewFaultDomainTree( @@ -1478,7 +1506,6 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { // ranks 4, }, - expDomainNr: 1, }, "request multiple ranks": { tree: NewFaultDomainTree( @@ -1519,7 +1546,6 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { 4, 5, }, - expDomainNr: 3, }, "request nonexistent rank": { tree: NewFaultDomainTree( @@ -1542,12 +1568,150 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { membership := NewMembership(log, db) result, err := membership.CompressedFaultDomainTree(tc.inputRanks...) + test.CmpErr(t, tc.expErr, err) + if diff := cmp.Diff(tc.expResult, result); diff != "" { t.Fatalf("(-want, +got): %s", diff) } + }) + } +} + +func TestSystem_Membership_FaultDomainLevel(t *testing.T) { + for name, tc := range map[string]struct { + tree *FaultDomainTree + expErr error + expLevel int + }{ + "nil tree": { + expErr: errors.New("uninitialized fault domain tree"), + }, + "root only": { + tree: NewFaultDomainTree(), + expErr: errors.New("domain tree has no fault domain level"), + }, + "rank only": { + tree: NewFaultDomainTree( + rankDomain("/", 0), + ), + expErr: errors.New("domain tree has no fault domain level"), + }, + "no extra domain levels": { + tree: NewFaultDomainTree( + rankDomain("/rack0", 0), + rankDomain("/rack0", 1), + rankDomain("/rack1", 2), + rankDomain("/rack1", 3), + rankDomain("/rack1", 4), + rankDomain("/rack2", 5), + ), + expLevel: 0, + }, + "two domain levels": { + tree: NewFaultDomainTree( + rankDomain("/rack0/pdu0", 0), + rankDomain("/rack0/pdu1", 1), + rankDomain("/rack1/pdu2", 2), + rankDomain("/rack1/pdu3", 3), + rankDomain("/rack1/pdu3", 4), + rankDomain("/rack2/pdu4", 5), + ), + expLevel: 1, + }, + "three domain levels": { + tree: NewFaultDomainTree( + rankDomain("/geo0/rack0/pdu0", 0), + rankDomain("/geo0/rack0/pdu1", 1), + rankDomain("/geo1/rack1/pdu2", 2), + rankDomain("/geo1/rack1/pdu3", 3), + rankDomain("/geo1/rack1/pdu3", 4), + rankDomain("/geo2/rack2/pdu4", 5), + ), + expLevel: 2, + }, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + // fmt.Printf(tc.tree.String()) + // fmt.Printf("Depth: %d\n", tc.tree.Depth()) + + db := raft.MockDatabaseWithFaultDomainTree(t, log, tc.tree) + membership := NewMembership(log, db) + + level, err := membership.FaultDomainLevel() + test.CmpErr(t, tc.expErr, err) + test.AssertEqual(t, tc.expLevel, level, "unexpected fault domain level") + }) + } +} + +func TestSystem_Membership_DomainNr(t *testing.T) { + bigDomainTree := NewFaultDomainTree( + rankDomain("/geo0/rack0/pdu0", 0), + rankDomain("/geo0/rack0/pdu1", 1), + rankDomain("/geo0/rack1/pdu2", 2), + rankDomain("/geo0/rack1/pdu3", 3), + rankDomain("/geo0/rack1/pdu4", 4), + rankDomain("/geo1/rack2/pdu5", 5), + ) + ranksSubset := []uint32{4, 0, 5, 3} + geoLevel := 0 + rackLevel := 1 + pduLevel := 2 + + for name, tc := range map[string]struct { + tree *FaultDomainTree + inputRanks []uint32 + inputLevel int + expDomainNr int + expErr error + }{ + "nil tree": { + expErr: errors.New("uninitialized fault domain tree"), + }, + "no extra domain levels": { + tree: NewFaultDomainTree( + rankDomain("/rack0", 0), + rankDomain("/rack0", 1), + rankDomain("/rack1", 2), + rankDomain("/rack1", 3), + rankDomain("/rack1", 4), + rankDomain("/rack2", 5), + ), + inputRanks: []uint32{5, 1, 2, 3}, + inputLevel: 0, + expDomainNr: 3, + }, + "big domain tree: geo level": { + tree: bigDomainTree, + inputRanks: ranksSubset, + inputLevel: geoLevel, + expDomainNr: 2, + }, + "big domain tree: rack level": { + tree: bigDomainTree, + inputRanks: ranksSubset, + inputLevel: rackLevel, + expDomainNr: 3, + }, + "big domain tree: pdu level": { + tree: bigDomainTree, + inputRanks: ranksSubset, + inputLevel: pduLevel, + expDomainNr: 4, + }, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + db := raft.MockDatabaseWithFaultDomainTree(t, log, tc.tree) + membership := NewMembership(log, db) - domainNr, err := membership.DomainNr(tc.inputRanks...) + domainNr, err := membership.DomainNr(tc.inputLevel, tc.inputRanks...) test.CmpErr(t, tc.expErr, err) test.AssertEqual(t, tc.expDomainNr, domainNr, "unexpected domain number") }) From 1f528f38ea89c43bdd2d63ffc02dc4b9594a2716 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Mon, 6 Jul 2026 17:32:55 +0000 Subject: [PATCH 13/20] DAOS-19207 control: remove redundant comment Doc-only: true Signed-off-by: Jan Michalski --- src/control/system/membership_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/control/system/membership_test.go b/src/control/system/membership_test.go index 05bf463307a..56b52675231 100644 --- a/src/control/system/membership_test.go +++ b/src/control/system/membership_test.go @@ -1635,9 +1635,6 @@ func TestSystem_Membership_FaultDomainLevel(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) defer test.ShowBufferOnFailure(t, buf) - // fmt.Printf(tc.tree.String()) - // fmt.Printf("Depth: %d\n", tc.tree.Depth()) - db := raft.MockDatabaseWithFaultDomainTree(t, log, tc.tree) membership := NewMembership(log, db) From 3baa4de6339307b5eef4cd58c1390c5ac988afd6 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Mon, 6 Jul 2026 18:02:36 +0000 Subject: [PATCH 14/20] DAOS-19207 control: align with the already established level... ... numbering scheme. Signed-off-by: Jan Michalski --- src/control/system/membership.go | 8 +++++--- src/control/system/membership_test.go | 14 +++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/control/system/membership.go b/src/control/system/membership.go index ef75e7dd679..af96b1b0c72 100644 --- a/src/control/system/membership.go +++ b/src/control/system/membership.go @@ -795,11 +795,13 @@ func (m *Membership) FaultDomainLevel() (int, error) { } depth := tree.Depth() - if depth < 2 { + // The depth includes the rank level but it does NOT include the root level. + // So, the tree needs to have more than one level to have a fault domain level. + if depth <= 1 { return 0, errors.New("domain tree has no fault domain level") } - return depth - 2, nil + return depth - 1, nil } // domainNrAtLevel returns the number of domains in the tree at the given level. @@ -809,7 +811,7 @@ func domainNrAtLevel(tree *FaultDomainTree, level int) int { return 0 } if level == 0 { - return len(tree.Children) + return 1 } count := 0 diff --git a/src/control/system/membership_test.go b/src/control/system/membership_test.go index 56b52675231..50111bcf71d 100644 --- a/src/control/system/membership_test.go +++ b/src/control/system/membership_test.go @@ -1606,7 +1606,7 @@ func TestSystem_Membership_FaultDomainLevel(t *testing.T) { rankDomain("/rack1", 4), rankDomain("/rack2", 5), ), - expLevel: 0, + expLevel: 1, }, "two domain levels": { tree: NewFaultDomainTree( @@ -1617,7 +1617,7 @@ func TestSystem_Membership_FaultDomainLevel(t *testing.T) { rankDomain("/rack1/pdu3", 4), rankDomain("/rack2/pdu4", 5), ), - expLevel: 1, + expLevel: 2, }, "three domain levels": { tree: NewFaultDomainTree( @@ -1628,7 +1628,7 @@ func TestSystem_Membership_FaultDomainLevel(t *testing.T) { rankDomain("/geo1/rack1/pdu3", 4), rankDomain("/geo2/rack2/pdu4", 5), ), - expLevel: 2, + expLevel: 3, }, } { t.Run(name, func(t *testing.T) { @@ -1655,9 +1655,9 @@ func TestSystem_Membership_DomainNr(t *testing.T) { rankDomain("/geo1/rack2/pdu5", 5), ) ranksSubset := []uint32{4, 0, 5, 3} - geoLevel := 0 - rackLevel := 1 - pduLevel := 2 + geoLevel := 1 + rackLevel := 2 + pduLevel := 3 for name, tc := range map[string]struct { tree *FaultDomainTree @@ -1679,7 +1679,7 @@ func TestSystem_Membership_DomainNr(t *testing.T) { rankDomain("/rack2", 5), ), inputRanks: []uint32{5, 1, 2, 3}, - inputLevel: 0, + inputLevel: 1, expDomainNr: 3, }, "big domain tree: geo level": { From eb2637f6267dfc4ffc0432d5fefb392b92f982ef Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Tue, 7 Jul 2026 10:13:56 +0000 Subject: [PATCH 15/20] DAOS-19207 control: do not expect fault domain when rd_fac is not... ... specified. Signed-off-by: Jan Michalski --- src/control/server/mgmt_pool.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/control/server/mgmt_pool.go b/src/control/server/mgmt_pool.go index ca71b96766d..e4fce5c2db7 100644 --- a/src/control/server/mgmt_pool.go +++ b/src/control/server/mgmt_pool.go @@ -415,17 +415,18 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq } // Check if the requested redundancy factor can be met with the number of supplied fault domains. - level, err := svc.membership.FaultDomainLevel() - if err != nil { - return nil, err - } - domainNr, err := svc.membership.DomainNr(level, req.Ranks...) - if err != nil { - return nil, err - } for _, prop := range req.GetProperties() { if prop.GetNumber() == uint32(daos.PoolPropertyRedunFac) { rdFac := int(prop.GetNumval()) + // Since the redundancy factor is specified, it is assumed that a fault‑domain level must exist. + level, err := svc.membership.FaultDomainLevel() + if err != nil { + return nil, err + } + domainNr, err := svc.membership.DomainNr(level, req.Ranks...) + if err != nil { + return nil, err + } if rdFac+1 > domainNr { return nil, FaultPoolTooFewFaultDomains(rdFac, domainNr) } From b401b8b82a3998f80537279038e4434dde899843 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Tue, 7 Jul 2026 10:15:38 +0000 Subject: [PATCH 16/20] DAOS-19207 control: remove redundant targetCount argument Signed-off-by: Jan Michalski --- src/control/server/mgmt_pool_test.go | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/src/control/server/mgmt_pool_test.go b/src/control/server/mgmt_pool_test.go index fca3d07a4f9..bf3e3647d49 100644 --- a/src/control/server/mgmt_pool_test.go +++ b/src/control/server/mgmt_pool_test.go @@ -434,7 +434,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { for name, tc := range map[string]struct { mgmtSvc *mgmtSvc setupMockDrpc func(_ *mgmtSvc, _ error) - targetCount int memberCount int mdonssdEnabled bool req *mgmtpb.PoolCreateReq @@ -454,8 +453,7 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: FaultWrongSystem("bad", build.DefaultSystemName), }, "missing superblock": { - mgmtSvc: missingSB, - targetCount: 8, + mgmtSvc: missingSB, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 0}, @@ -464,8 +462,7 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: errNotReplica, }, "not MS replica": { - mgmtSvc: notAP, - targetCount: 8, + mgmtSvc: notAP, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 0}, @@ -474,7 +471,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: errNotReplica, }, "dRPC send fails": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 0}, @@ -483,7 +479,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: errors.New("send failure"), }, "zero target count": { - targetCount: 0, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 0}, @@ -492,7 +487,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: errors.New("zero target count"), }, "garbage resp": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, @@ -507,7 +501,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: errors.New("unmarshal"), }, "successful creation": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, @@ -523,7 +516,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { }, }, "create with memory file ratio; mdonssd not enabled": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, @@ -533,7 +525,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: errors.New("MD-on-SSD has not been enabled"), }, "successful creation with memory file ratio": { - targetCount: 8, mdonssdEnabled: true, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), @@ -553,7 +544,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { }, }, "successful creation minimum size": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{ @@ -578,7 +568,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { }, }, "successful creation auto size": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TotalBytes: 100 * humanize.GiByte, @@ -601,7 +590,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { }, }, "failed creation invalid ranks": { - targetCount: 1, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, @@ -611,7 +599,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: FaultPoolInvalidRanks([]ranklist.Rank{11, 40}), }, "failed creation invalid number of ranks": { - targetCount: 1, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, @@ -621,7 +608,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: FaultPoolInvalidNumRanks(3, 2), }, "svc replicas > max": { - targetCount: 1, memberCount: MaxPoolServiceReps + 2, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), @@ -633,7 +619,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: FaultPoolInvalidServiceReps(uint32(MaxPoolServiceReps)), }, "svc replicas > numRanks": { - targetCount: 1, memberCount: MaxPoolServiceReps - 2, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), @@ -645,7 +630,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: FaultPoolInvalidServiceReps(uint32(MaxPoolServiceReps - 2)), }, "no label": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, @@ -653,7 +637,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: FaultPoolNoLabel, }, "failed creation too few fault domains": { - targetCount: 1, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, @@ -663,7 +646,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: FaultPoolTooFewFaultDomains(1, 1), }, "successful creation with rd_fac": { - targetCount: 2, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, From d423047410bb14009d0f94c0f72ed6e9723c318c Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Tue, 7 Jul 2026 10:16:35 +0000 Subject: [PATCH 17/20] DAOS-19207 control: add a fault domain mock Signed-off-by: Jan Michalski --- src/control/system/mocks.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/control/system/mocks.go b/src/control/system/mocks.go index eef4cf6d5ad..0487166a89e 100644 --- a/src/control/system/mocks.go +++ b/src/control/system/mocks.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2022 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -48,7 +48,7 @@ func MockMemberFullSpec(t *testing.T, rank Rank, uuidStr string, uri string, add PrimaryFabricURI: uri, Addr: addr, State: state, - FaultDomain: MustCreateFaultDomain(), + FaultDomain: MustCreateFaultDomain(fmt.Sprintf("mock_domain%d", rank.Uint32())), LastUpdate: time.Now(), } } From 8a929985b0ac5f41ce63cd957b4b8f3707ffb9d6 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Tue, 7 Jul 2026 12:16:43 +0000 Subject: [PATCH 18/20] Revert "DAOS-19207 control: add a fault domain mock" This reverts commit d423047410bb14009d0f94c0f72ed6e9723c318c. Signed-off-by: Jan Michalski --- src/control/system/mocks.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/control/system/mocks.go b/src/control/system/mocks.go index 0487166a89e..eef4cf6d5ad 100644 --- a/src/control/system/mocks.go +++ b/src/control/system/mocks.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2022 Intel Corporation. -// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -48,7 +48,7 @@ func MockMemberFullSpec(t *testing.T, rank Rank, uuidStr string, uri string, add PrimaryFabricURI: uri, Addr: addr, State: state, - FaultDomain: MustCreateFaultDomain(fmt.Sprintf("mock_domain%d", rank.Uint32())), + FaultDomain: MustCreateFaultDomain(), LastUpdate: time.Now(), } } From b4ef189b68849d73466adf5741499ea0016167bc Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Tue, 7 Jul 2026 12:17:17 +0000 Subject: [PATCH 19/20] DAOS-19207 control: add fault domain to... ... the TestServer_MgmtSvc_PoolCreate test. Signed-off-by: Jan Michalski --- src/control/server/mgmt_pool_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/control/server/mgmt_pool_test.go b/src/control/server/mgmt_pool_test.go index bf3e3647d49..675c78dcf36 100644 --- a/src/control/server/mgmt_pool_test.go +++ b/src/control/server/mgmt_pool_test.go @@ -686,7 +686,8 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { numMembers = 2 } for i := 0; i < numMembers; i++ { - mm := system.MockMember(t, uint32(i), system.MemberStateJoined) + faultDomain := system.MustCreateFaultDomain(fmt.Sprintf("mock_domain%d", i)) + mm := system.MockMember(t, uint32(i), system.MemberStateJoined).WithFaultDomain(faultDomain) if _, err := tc.mgmtSvc.membership.Add(mm); err != nil { t.Fatal(err) } From a65865d457b383245cbaa59b5c78395b27338c37 Mon Sep 17 00:00:00 2001 From: Jan Michalski Date: Fri, 10 Jul 2026 14:30:49 +0000 Subject: [PATCH 20/20] DAOS-19207 control: remove redundant condition Signed-off-by: Jan Michalski --- src/control/system/membership.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/control/system/membership.go b/src/control/system/membership.go index af96b1b0c72..b196bc399c0 100644 --- a/src/control/system/membership.go +++ b/src/control/system/membership.go @@ -807,9 +807,6 @@ func (m *Membership) FaultDomainLevel() (int, error) { // domainNrAtLevel returns the number of domains in the tree at the given level. // It traverses the tree recursively to reach the specified level. func domainNrAtLevel(tree *FaultDomainTree, level int) int { - if tree == nil { - return 0 - } if level == 0 { return 1 }