DAOS-19207 control: check redundancy factor early

src/control/fault/code/codes.go

@@ -161,6 +161,7 @@ const (
 	ServerPoolHasContainers
 	ServerPoolMemRatioNoRoles
 	ServerBadFaultDomainLabels
+	ServerPoolTooFewFaultDomains
 	ServerJoinReplaceEnabledPoolRank
 	ServerRankAdminExcluded
 	ServerTransparentHugepageEnabled

src/control/server/faults.go

@@ -1,6 +1,6 @@
 //
 // (C) Copyright 2020-2024 Intel Corporation.
-// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -174,6 +174,15 @@ var FaultPoolMemRatioNoRoles = serverFault(
 	"either remove MD-on-SSD-specific options from the command request or set bdev_roles in "+
 		"server config file to enable MD-on-SSD")
 
+func FaultPoolTooFewFaultDomains(rdFac int, numDomains int) *fault.Fault {
+	return serverFault(
+		code.ServerPoolTooFewFaultDomains,
+		fmt.Sprintf("pool redundancy factor %d requires at least %d fault domains but only %d are available",
+			rdFac, rdFac+1, numDomains),
+		"retry the request with a lower redundancy factor or add more fault domains",
+	)
+}
+
 func FaultBadFaultDomainLabels(faultPath, addr string, reqLabels, systemLabels []string) *fault.Fault {
 	return serverFault(
 		code.ServerBadFaultDomainLabels,

src/control/server/mgmt_pool.go

@@ -1,6 +1,6 @@
 //
 // (C) Copyright 2020-2024 Intel Corporation.
-// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -414,6 +414,26 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq
 		return nil, FaultPoolInvalidServiceReps(maxSvcReps)
 	}
 
+	// Check if the requested redundancy factor can be met with the number of supplied fault domains.
+	for _, prop := range req.GetProperties() {
+		if prop.GetNumber() == uint32(daos.PoolPropertyRedunFac) {
+			rdFac := int(prop.GetNumval())
+			// Since the redundancy factor is specified, it is assumed that a fault‑domain level must exist.
+			level, err := svc.membership.FaultDomainLevel()
+			if err != nil {
+				return nil, err
+			}
+			domainNr, err := svc.membership.DomainNr(level, req.Ranks...)
+			if err != nil {
+				return nil, err
+			}
+			if rdFac+1 > domainNr {
+				return nil, FaultPoolTooFewFaultDomains(rdFac, domainNr)
+			}
+			break
+		}
+	}
+
 	// IO engine needs the fault domain tree for placement purposes
 	req.FaultDomains, err = svc.membership.CompressedFaultDomainTree(req.Ranks...)
 	if err != nil {

src/control/server/mgmt_pool_test.go

@@ -1,6 +1,6 @@
 //
 // (C) Copyright 2020-2024 Intel Corporation.
-// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -118,6 +118,23 @@ func testPoolLabelProp() []*mgmtpb.PoolProperty {
 	}
 }
 
+func testPoolRedunFacProp() []*mgmtpb.PoolProperty {
+	return []*mgmtpb.PoolProperty{
+		{
+			Number: daos.PoolPropertyLabel,
+			Value: &mgmtpb.PoolProperty_Strval{
+				Strval: "test",
+			},
+		},
+		{
+			Number: daos.PoolPropertyRedunFac,
+			Value: &mgmtpb.PoolProperty_Numval{
+				Numval: 1,
+			},
+		},
+	}
+}
+
 func TestServer_MgmtSvc_PoolCreateAlreadyExists(t *testing.T) {
 	for name, tc := range map[string]struct {
 		state     system.PoolServiceState
@@ -417,7 +434,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 	for name, tc := range map[string]struct {
 		mgmtSvc        *mgmtSvc
 		setupMockDrpc  func(_ *mgmtSvc, _ error)
-		targetCount    int
 		memberCount    int
 		mdonssdEnabled bool
 		req            *mgmtpb.PoolCreateReq
@@ -437,8 +453,7 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			expErr: FaultWrongSystem("bad", build.DefaultSystemName),
 		},
 		"missing superblock": {
-			mgmtSvc:     missingSB,
-			targetCount: 8,
+			mgmtSvc: missingSB,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
 				TierBytes:  []uint64{100 * humanize.GiByte, 0},
@@ -447,8 +462,7 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			expErr: errNotReplica,
 		},
 		"not MS replica": {
-			mgmtSvc:     notAP,
-			targetCount: 8,
+			mgmtSvc: notAP,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
 				TierBytes:  []uint64{100 * humanize.GiByte, 0},
@@ -457,7 +471,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			expErr: errNotReplica,
 		},
 		"dRPC send fails": {
-			targetCount: 8,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
 				TierBytes:  []uint64{100 * humanize.GiByte, 0},
@@ -466,7 +479,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			expErr: errors.New("send failure"),
 		},
 		"zero target count": {
-			targetCount: 0,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
 				TierBytes:  []uint64{100 * humanize.GiByte, 0},
@@ -475,7 +487,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			expErr: errors.New("zero target count"),
 		},
 		"garbage resp": {
-			targetCount: 8,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
 				TierBytes:  []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
@@ -490,7 +501,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			expErr: errors.New("unmarshal"),
 		},
 		"successful creation": {
-			targetCount: 8,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
 				TierBytes:  []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
@@ -506,7 +516,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			},
 		},
 		"create with memory file ratio; mdonssd not enabled": {
-			targetCount: 8,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
 				TierBytes:  []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
@@ -516,7 +525,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			expErr: errors.New("MD-on-SSD has not been enabled"),
 		},
 		"successful creation with memory file ratio": {
-			targetCount:    8,
 			mdonssdEnabled: true,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
@@ -536,7 +544,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			},
 		},
 		"successful creation minimum size": {
-			targetCount: 8,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid: test.MockUUID(1),
 				TierBytes: []uint64{
@@ -561,7 +568,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			},
 		},
 		"successful creation auto size": {
-			targetCount: 8,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
 				TotalBytes: 100 * humanize.GiByte,
@@ -584,7 +590,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			},
 		},
 		"failed creation invalid ranks": {
-			targetCount: 1,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
 				TierBytes:  []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
@@ -594,7 +599,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			expErr: FaultPoolInvalidRanks([]ranklist.Rank{11, 40}),
 		},
 		"failed creation invalid number of ranks": {
-			targetCount: 1,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
 				TierBytes:  []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
@@ -604,7 +608,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			expErr: FaultPoolInvalidNumRanks(3, 2),
 		},
 		"svc replicas > max": {
-			targetCount: 1,
 			memberCount: MaxPoolServiceReps + 2,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
@@ -616,7 +619,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			expErr: FaultPoolInvalidServiceReps(uint32(MaxPoolServiceReps)),
 		},
 		"svc replicas > numRanks": {
-			targetCount: 1,
 			memberCount: MaxPoolServiceReps - 2,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:       test.MockUUID(1),
@@ -628,13 +630,37 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			expErr: FaultPoolInvalidServiceReps(uint32(MaxPoolServiceReps - 2)),
 		},
 		"no label": {
-			targetCount: 8,
 			req: &mgmtpb.PoolCreateReq{
 				Uuid:      test.MockUUID(1),
 				TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
 			},
 			expErr: FaultPoolNoLabel,
 		},
+		"failed creation too few fault domains": {
+			req: &mgmtpb.PoolCreateReq{
+				Uuid:       test.MockUUID(1),
+				TierBytes:  []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
+				Ranks:      []uint32{0},
+				Properties: testPoolRedunFacProp(),
+			},
+			expErr: FaultPoolTooFewFaultDomains(1, 1),
+		},
+		"successful creation with rd_fac": {
+			req: &mgmtpb.PoolCreateReq{
+				Uuid:       test.MockUUID(1),
+				TierBytes:  []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
+				Ranks:      []uint32{0, 1},
+				Properties: testPoolRedunFacProp(),
+			},
+			drpcRet: &mgmtpb.PoolCreateResp{
+				TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
+				TgtRanks:  []uint32{0, 1},
+			},
+			expResp: &mgmtpb.PoolCreateResp{
+				TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
+				TgtRanks:  []uint32{0, 1},
+			},
+		},
 	} {
 		t.Run(name, func(t *testing.T) {
 			buf.Reset()
@@ -660,7 +686,8 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 				numMembers = 2
 			}
 			for i := 0; i < numMembers; i++ {
-				mm := system.MockMember(t, uint32(i), system.MemberStateJoined)
+				faultDomain := system.MustCreateFaultDomain(fmt.Sprintf("mock_domain%d", i))
+				mm := system.MockMember(t, uint32(i), system.MemberStateJoined).WithFaultDomain(faultDomain)
 				if _, err := tc.mgmtSvc.membership.Add(mm); err != nil {
 					t.Fatal(err)
 				}

src/control/system/membership.go

@@ -785,6 +785,60 @@ func (m *Membership) CompressedFaultDomainTree(ranks ...uint32) ([]uint32, error
 	return append([]uint32{md}, compressTree(subtree)...), nil
 }
 
+// FaultDomainLevel returns the fault domain level of the domain tree.
+// It assumes the tree is balanced and that the rank level is present.
+// So, the fault domain level is the second to last level of the tree.
+func (m *Membership) FaultDomainLevel() (int, error) {
+	tree := m.db.FaultDomainTree()
+	if tree == nil {
+		return 0, errors.New("uninitialized fault domain tree")
+	}
+
+	depth := tree.Depth()
+	// The depth includes the rank level but it does NOT include the root level.
+	// So, the tree needs to have more than one level to have a fault domain level.
+	if depth <= 1 {
+		return 0, errors.New("domain tree has no fault domain level")
+	}
+
+	return depth - 1, nil
+}
+
+// domainNrAtLevel returns the number of domains in the tree at the given level.
+// It traverses the tree recursively to reach the specified level.
+func domainNrAtLevel(tree *FaultDomainTree, level int) int {
+	if level == 0 {
+		return 1
+	}
+
+	count := 0
+	for _, child := range tree.Children {
+		count += domainNrAtLevel(child, level-1)
+	}
+	return count
+}
+
+// DomainNr returns the number of domains in the subtree of the domain tree
+// specified by the given ranks at the given level.
+// If no ranks are provided, the entire tree is considered.
+func (m *Membership) DomainNr(level int, ranks ...uint32) (int, error) {
+	tree := m.db.FaultDomainTree()
+	if tree == nil {
+		return 0, errors.New("uninitialized fault domain tree")
+	}
+
+	subtree, err := getFaultDomainSubtree(tree, ranks...)
+	if err != nil {
+		return 0, err
+	}
+
+	if level >= subtree.Depth() {
+		return 0, errors.Errorf("level %d >= subtree depth %d", level, subtree.Depth())
+	}
+
+	return domainNrAtLevel(subtree, level), nil
+}
+
 const (
 	DomTreeMetadataHasFaultDom uint32 = (1 << iota)
 	DomTreeMetadataHasPerfDom