diff --git a/src/control/fault/code/codes.go b/src/control/fault/code/codes.go index bb07f0b89ed..f6e7800a49e 100644 --- a/src/control/fault/code/codes.go +++ b/src/control/fault/code/codes.go @@ -161,6 +161,7 @@ const ( ServerPoolHasContainers ServerPoolMemRatioNoRoles ServerBadFaultDomainLabels + ServerPoolTooFewFaultDomains ServerJoinReplaceEnabledPoolRank ServerRankAdminExcluded ServerTransparentHugepageEnabled diff --git a/src/control/server/faults.go b/src/control/server/faults.go index a2e51760026..4359d06f1a6 100644 --- a/src/control/server/faults.go +++ b/src/control/server/faults.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -174,6 +174,15 @@ var FaultPoolMemRatioNoRoles = serverFault( "either remove MD-on-SSD-specific options from the command request or set bdev_roles in "+ "server config file to enable MD-on-SSD") +func FaultPoolTooFewFaultDomains(rdFac int, numDomains int) *fault.Fault { + return serverFault( + code.ServerPoolTooFewFaultDomains, + fmt.Sprintf("pool redundancy factor %d requires at least %d fault domains but only %d are available", + rdFac, rdFac+1, numDomains), + "retry the request with a lower redundancy factor or add more fault domains", + ) +} + func FaultBadFaultDomainLabels(faultPath, addr string, reqLabels, systemLabels []string) *fault.Fault { return serverFault( code.ServerBadFaultDomainLabels, diff --git a/src/control/server/mgmt_pool.go b/src/control/server/mgmt_pool.go index dbe5f0d5ae9..e4fce5c2db7 100644 --- a/src/control/server/mgmt_pool.go +++ b/src/control/server/mgmt_pool.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -414,6 +414,26 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq return nil, FaultPoolInvalidServiceReps(maxSvcReps) } + // Check if the requested redundancy factor can be met with the number of supplied fault domains. + for _, prop := range req.GetProperties() { + if prop.GetNumber() == uint32(daos.PoolPropertyRedunFac) { + rdFac := int(prop.GetNumval()) + // Since the redundancy factor is specified, it is assumed that a fault‑domain level must exist. + level, err := svc.membership.FaultDomainLevel() + if err != nil { + return nil, err + } + domainNr, err := svc.membership.DomainNr(level, req.Ranks...) + if err != nil { + return nil, err + } + if rdFac+1 > domainNr { + return nil, FaultPoolTooFewFaultDomains(rdFac, domainNr) + } + break + } + } + // IO engine needs the fault domain tree for placement purposes req.FaultDomains, err = svc.membership.CompressedFaultDomainTree(req.Ranks...) if err != nil { diff --git a/src/control/server/mgmt_pool_test.go b/src/control/server/mgmt_pool_test.go index 51a3bdc2813..675c78dcf36 100644 --- a/src/control/server/mgmt_pool_test.go +++ b/src/control/server/mgmt_pool_test.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -118,6 +118,23 @@ func testPoolLabelProp() []*mgmtpb.PoolProperty { } } +func testPoolRedunFacProp() []*mgmtpb.PoolProperty { + return []*mgmtpb.PoolProperty{ + { + Number: daos.PoolPropertyLabel, + Value: &mgmtpb.PoolProperty_Strval{ + Strval: "test", + }, + }, + { + Number: daos.PoolPropertyRedunFac, + Value: &mgmtpb.PoolProperty_Numval{ + Numval: 1, + }, + }, + } +} + func TestServer_MgmtSvc_PoolCreateAlreadyExists(t *testing.T) { for name, tc := range map[string]struct { state system.PoolServiceState @@ -417,7 +434,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { for name, tc := range map[string]struct { mgmtSvc *mgmtSvc setupMockDrpc func(_ *mgmtSvc, _ error) - targetCount int memberCount int mdonssdEnabled bool req *mgmtpb.PoolCreateReq @@ -437,8 +453,7 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: FaultWrongSystem("bad", build.DefaultSystemName), }, "missing superblock": { - mgmtSvc: missingSB, - targetCount: 8, + mgmtSvc: missingSB, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 0}, @@ -447,8 +462,7 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: errNotReplica, }, "not MS replica": { - mgmtSvc: notAP, - targetCount: 8, + mgmtSvc: notAP, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 0}, @@ -457,7 +471,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: errNotReplica, }, "dRPC send fails": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 0}, @@ -466,7 +479,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: errors.New("send failure"), }, "zero target count": { - targetCount: 0, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 0}, @@ -475,7 +487,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: errors.New("zero target count"), }, "garbage resp": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, @@ -490,7 +501,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: errors.New("unmarshal"), }, "successful creation": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, @@ -506,7 +516,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { }, }, "create with memory file ratio; mdonssd not enabled": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, @@ -516,7 +525,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: errors.New("MD-on-SSD has not been enabled"), }, "successful creation with memory file ratio": { - targetCount: 8, mdonssdEnabled: true, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), @@ -536,7 +544,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { }, }, "successful creation minimum size": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{ @@ -561,7 +568,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { }, }, "successful creation auto size": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TotalBytes: 100 * humanize.GiByte, @@ -584,7 +590,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { }, }, "failed creation invalid ranks": { - targetCount: 1, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, @@ -594,7 +599,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: FaultPoolInvalidRanks([]ranklist.Rank{11, 40}), }, "failed creation invalid number of ranks": { - targetCount: 1, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, @@ -604,7 +608,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: FaultPoolInvalidNumRanks(3, 2), }, "svc replicas > max": { - targetCount: 1, memberCount: MaxPoolServiceReps + 2, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), @@ -616,7 +619,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: FaultPoolInvalidServiceReps(uint32(MaxPoolServiceReps)), }, "svc replicas > numRanks": { - targetCount: 1, memberCount: MaxPoolServiceReps - 2, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), @@ -628,13 +630,37 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { expErr: FaultPoolInvalidServiceReps(uint32(MaxPoolServiceReps - 2)), }, "no label": { - targetCount: 8, req: &mgmtpb.PoolCreateReq{ Uuid: test.MockUUID(1), TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, }, expErr: FaultPoolNoLabel, }, + "failed creation too few fault domains": { + req: &mgmtpb.PoolCreateReq{ + Uuid: test.MockUUID(1), + TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, + Ranks: []uint32{0}, + Properties: testPoolRedunFacProp(), + }, + expErr: FaultPoolTooFewFaultDomains(1, 1), + }, + "successful creation with rd_fac": { + req: &mgmtpb.PoolCreateReq{ + Uuid: test.MockUUID(1), + TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, + Ranks: []uint32{0, 1}, + Properties: testPoolRedunFacProp(), + }, + drpcRet: &mgmtpb.PoolCreateResp{ + TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, + TgtRanks: []uint32{0, 1}, + }, + expResp: &mgmtpb.PoolCreateResp{ + TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, + TgtRanks: []uint32{0, 1}, + }, + }, } { t.Run(name, func(t *testing.T) { buf.Reset() @@ -660,7 +686,8 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { numMembers = 2 } for i := 0; i < numMembers; i++ { - mm := system.MockMember(t, uint32(i), system.MemberStateJoined) + faultDomain := system.MustCreateFaultDomain(fmt.Sprintf("mock_domain%d", i)) + mm := system.MockMember(t, uint32(i), system.MemberStateJoined).WithFaultDomain(faultDomain) if _, err := tc.mgmtSvc.membership.Add(mm); err != nil { t.Fatal(err) } diff --git a/src/control/system/membership.go b/src/control/system/membership.go index 3218c5249b9..b196bc399c0 100644 --- a/src/control/system/membership.go +++ b/src/control/system/membership.go @@ -785,6 +785,60 @@ func (m *Membership) CompressedFaultDomainTree(ranks ...uint32) ([]uint32, error return append([]uint32{md}, compressTree(subtree)...), nil } +// FaultDomainLevel returns the fault domain level of the domain tree. +// It assumes the tree is balanced and that the rank level is present. +// So, the fault domain level is the second to last level of the tree. +func (m *Membership) FaultDomainLevel() (int, error) { + tree := m.db.FaultDomainTree() + if tree == nil { + return 0, errors.New("uninitialized fault domain tree") + } + + depth := tree.Depth() + // The depth includes the rank level but it does NOT include the root level. + // So, the tree needs to have more than one level to have a fault domain level. + if depth <= 1 { + return 0, errors.New("domain tree has no fault domain level") + } + + return depth - 1, nil +} + +// domainNrAtLevel returns the number of domains in the tree at the given level. +// It traverses the tree recursively to reach the specified level. +func domainNrAtLevel(tree *FaultDomainTree, level int) int { + if level == 0 { + return 1 + } + + count := 0 + for _, child := range tree.Children { + count += domainNrAtLevel(child, level-1) + } + return count +} + +// DomainNr returns the number of domains in the subtree of the domain tree +// specified by the given ranks at the given level. +// If no ranks are provided, the entire tree is considered. +func (m *Membership) DomainNr(level int, ranks ...uint32) (int, error) { + tree := m.db.FaultDomainTree() + if tree == nil { + return 0, errors.New("uninitialized fault domain tree") + } + + subtree, err := getFaultDomainSubtree(tree, ranks...) + if err != nil { + return 0, err + } + + if level >= subtree.Depth() { + return 0, errors.Errorf("level %d >= subtree depth %d", level, subtree.Depth()) + } + + return domainNrAtLevel(subtree, level), nil +} + const ( DomTreeMetadataHasFaultDom uint32 = (1 << iota) DomTreeMetadataHasPerfDom diff --git a/src/control/system/membership_test.go b/src/control/system/membership_test.go index 7d3e2db8473..50111bcf71d 100644 --- a/src/control/system/membership_test.go +++ b/src/control/system/membership_test.go @@ -1284,20 +1284,16 @@ func TestSystem_Membership_MarkDead(t *testing.T) { } } -func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { - testMemberWithFaultDomain := func(rank Rank, faultDomain *FaultDomain) *Member { - return &Member{ - Rank: rank, - FaultDomain: faultDomain, - } - } - - rankDomain := func(parent string, rank uint32) *FaultDomain { - parentFd := MustCreateFaultDomainFromString(parent) - member := testMemberWithFaultDomain(Rank(rank), parentFd) - return MemberFaultDomain(member) +func rankDomain(parent string, rank uint32) *FaultDomain { + parentFd := MustCreateFaultDomainFromString(parent) + member := &Member{ + Rank: Rank(rank), + FaultDomain: parentFd, } + return MemberFaultDomain(member) +} +func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { for name, tc := range map[string]struct { tree *FaultDomainTree inputRanks []uint32 @@ -1581,3 +1577,140 @@ func TestSystem_Membership_CompressedFaultDomainTree(t *testing.T) { }) } } + +func TestSystem_Membership_FaultDomainLevel(t *testing.T) { + for name, tc := range map[string]struct { + tree *FaultDomainTree + expErr error + expLevel int + }{ + "nil tree": { + expErr: errors.New("uninitialized fault domain tree"), + }, + "root only": { + tree: NewFaultDomainTree(), + expErr: errors.New("domain tree has no fault domain level"), + }, + "rank only": { + tree: NewFaultDomainTree( + rankDomain("/", 0), + ), + expErr: errors.New("domain tree has no fault domain level"), + }, + "no extra domain levels": { + tree: NewFaultDomainTree( + rankDomain("/rack0", 0), + rankDomain("/rack0", 1), + rankDomain("/rack1", 2), + rankDomain("/rack1", 3), + rankDomain("/rack1", 4), + rankDomain("/rack2", 5), + ), + expLevel: 1, + }, + "two domain levels": { + tree: NewFaultDomainTree( + rankDomain("/rack0/pdu0", 0), + rankDomain("/rack0/pdu1", 1), + rankDomain("/rack1/pdu2", 2), + rankDomain("/rack1/pdu3", 3), + rankDomain("/rack1/pdu3", 4), + rankDomain("/rack2/pdu4", 5), + ), + expLevel: 2, + }, + "three domain levels": { + tree: NewFaultDomainTree( + rankDomain("/geo0/rack0/pdu0", 0), + rankDomain("/geo0/rack0/pdu1", 1), + rankDomain("/geo1/rack1/pdu2", 2), + rankDomain("/geo1/rack1/pdu3", 3), + rankDomain("/geo1/rack1/pdu3", 4), + rankDomain("/geo2/rack2/pdu4", 5), + ), + expLevel: 3, + }, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + db := raft.MockDatabaseWithFaultDomainTree(t, log, tc.tree) + membership := NewMembership(log, db) + + level, err := membership.FaultDomainLevel() + test.CmpErr(t, tc.expErr, err) + test.AssertEqual(t, tc.expLevel, level, "unexpected fault domain level") + }) + } +} + +func TestSystem_Membership_DomainNr(t *testing.T) { + bigDomainTree := NewFaultDomainTree( + rankDomain("/geo0/rack0/pdu0", 0), + rankDomain("/geo0/rack0/pdu1", 1), + rankDomain("/geo0/rack1/pdu2", 2), + rankDomain("/geo0/rack1/pdu3", 3), + rankDomain("/geo0/rack1/pdu4", 4), + rankDomain("/geo1/rack2/pdu5", 5), + ) + ranksSubset := []uint32{4, 0, 5, 3} + geoLevel := 1 + rackLevel := 2 + pduLevel := 3 + + for name, tc := range map[string]struct { + tree *FaultDomainTree + inputRanks []uint32 + inputLevel int + expDomainNr int + expErr error + }{ + "nil tree": { + expErr: errors.New("uninitialized fault domain tree"), + }, + "no extra domain levels": { + tree: NewFaultDomainTree( + rankDomain("/rack0", 0), + rankDomain("/rack0", 1), + rankDomain("/rack1", 2), + rankDomain("/rack1", 3), + rankDomain("/rack1", 4), + rankDomain("/rack2", 5), + ), + inputRanks: []uint32{5, 1, 2, 3}, + inputLevel: 1, + expDomainNr: 3, + }, + "big domain tree: geo level": { + tree: bigDomainTree, + inputRanks: ranksSubset, + inputLevel: geoLevel, + expDomainNr: 2, + }, + "big domain tree: rack level": { + tree: bigDomainTree, + inputRanks: ranksSubset, + inputLevel: rackLevel, + expDomainNr: 3, + }, + "big domain tree: pdu level": { + tree: bigDomainTree, + inputRanks: ranksSubset, + inputLevel: pduLevel, + expDomainNr: 4, + }, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + db := raft.MockDatabaseWithFaultDomainTree(t, log, tc.tree) + membership := NewMembership(log, db) + + domainNr, err := membership.DomainNr(tc.inputLevel, tc.inputRanks...) + test.CmpErr(t, tc.expErr, err) + test.AssertEqual(t, tc.expDomainNr, domainNr, "unexpected domain number") + }) + } +}