From 6555882275d6535029229fbb51b2a64f71b3ae4c Mon Sep 17 00:00:00 2001 From: Jason Price Date: Wed, 13 Nov 2019 11:08:46 -0500 Subject: [PATCH 01/10] refresh some checked fields in addPods --- pkg/rc/replication_controller.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/rc/replication_controller.go b/pkg/rc/replication_controller.go index 3d8c147b7..f00c54a13 100644 --- a/pkg/rc/replication_controller.go +++ b/pkg/rc/replication_controller.go @@ -330,7 +330,14 @@ func (rc *replicationController) addPods(rcFields fields.RC, current types.PodLo cancelFunc() txn, cancelFunc = rc.newAuditingTransaction(context.Background(), rcFields, txn.Nodes()) } - if len(possibleSorted) < i+1 { + + // since significant time may have passed since these values were instantiated, + // get updated values each iteration, and leverage those + tmpEligible, err := rc.eligibleNodes(rcFields) + // TODO: check err + tmpPossible := types.NewNodeSet(tmpEligible...).Difference(types.NewNodeSet(currentNodes...)) + tmpPossibleSorted := tmpPossible.ListNodes() + if len(tmpPossibleSorted) < i+1 { errMsg := fmt.Sprintf( "Not enough nodes to meet desire: %d replicas desired, %d currentNodes, %d eligible. Scheduled on %d nodes instead.", rcFields.ReplicasDesired, len(currentNodes), len(eligible), i, From e1b83c916c88577cbb5de60ebaa32e52a761939c Mon Sep 17 00:00:00 2001 From: Jason Price Date: Thu, 14 Nov 2019 13:48:57 -0500 Subject: [PATCH 02/10] not sure how this worked before... --- pkg/rc/replication_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rc/replication_controller.go b/pkg/rc/replication_controller.go index f00c54a13..028c0e5b6 100644 --- a/pkg/rc/replication_controller.go +++ b/pkg/rc/replication_controller.go @@ -360,7 +360,7 @@ func (rc *replicationController) addPods(rcFields fields.RC, current types.PodLo } scheduleOn := possibleSorted[i] - err := rc.schedule(txn, rcFields, scheduleOn) + err = rc.schedule(txn, rcFields, scheduleOn) if err != nil { return err } From 548ca31e2ef9993aa6e1dc101e430b0ffa688b21 Mon Sep 17 00:00:00 2001 From: Jason Price Date: Thu, 14 Nov 2019 15:55:09 -0500 Subject: [PATCH 03/10] ignore new err, restore old version of existing err --- pkg/rc/replication_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/rc/replication_controller.go b/pkg/rc/replication_controller.go index 028c0e5b6..3d8be0069 100644 --- a/pkg/rc/replication_controller.go +++ b/pkg/rc/replication_controller.go @@ -333,7 +333,7 @@ func (rc *replicationController) addPods(rcFields fields.RC, current types.PodLo // since significant time may have passed since these values were instantiated, // get updated values each iteration, and leverage those - tmpEligible, err := rc.eligibleNodes(rcFields) + tmpEligible, _ := rc.eligibleNodes(rcFields) // TODO: check err tmpPossible := types.NewNodeSet(tmpEligible...).Difference(types.NewNodeSet(currentNodes...)) tmpPossibleSorted := tmpPossible.ListNodes() @@ -360,7 +360,7 @@ func (rc *replicationController) addPods(rcFields fields.RC, current types.PodLo } scheduleOn := possibleSorted[i] - err = rc.schedule(txn, rcFields, scheduleOn) + err := rc.schedule(txn, rcFields, scheduleOn) if err != nil { return err } From 9d5abbbb70518446fc2023b1d6f2956829c0ccc7 Mon Sep 17 00:00:00 2001 From: Jason Price Date: Fri, 15 Nov 2019 08:34:40 -0500 Subject: [PATCH 04/10] rebuild test From d84deeeb9b6dde4db2c8f2c76ba416d336e6f2dd Mon Sep 17 00:00:00 2001 From: Jason Price Date: Fri, 15 Nov 2019 16:30:17 -0500 Subject: [PATCH 05/10] now with error handling --- pkg/rc/replication_controller.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/rc/replication_controller.go b/pkg/rc/replication_controller.go index 3d8be0069..dd21a55b2 100644 --- a/pkg/rc/replication_controller.go +++ b/pkg/rc/replication_controller.go @@ -333,8 +333,10 @@ func (rc *replicationController) addPods(rcFields fields.RC, current types.PodLo // since significant time may have passed since these values were instantiated, // get updated values each iteration, and leverage those - tmpEligible, _ := rc.eligibleNodes(rcFields) - // TODO: check err + tmpEligible, terr := rc.eligibleNodes(rcFields) + if terr != nil { + return terr + } tmpPossible := types.NewNodeSet(tmpEligible...).Difference(types.NewNodeSet(currentNodes...)) tmpPossibleSorted := tmpPossible.ListNodes() if len(tmpPossibleSorted) < i+1 { From 27c8725d78516439d3b87f738e2af2086af6f90c Mon Sep 17 00:00:00 2001 From: Jason Price Date: Fri, 15 Nov 2019 17:25:22 -0500 Subject: [PATCH 06/10] now with same variables --- pkg/rc/replication_controller.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/rc/replication_controller.go b/pkg/rc/replication_controller.go index dd21a55b2..5221586ae 100644 --- a/pkg/rc/replication_controller.go +++ b/pkg/rc/replication_controller.go @@ -333,13 +333,13 @@ func (rc *replicationController) addPods(rcFields fields.RC, current types.PodLo // since significant time may have passed since these values were instantiated, // get updated values each iteration, and leverage those - tmpEligible, terr := rc.eligibleNodes(rcFields) - if terr != nil { - return terr + eligible, err := rc.eligibleNodes(rcFields) + if err != nil { + return err } - tmpPossible := types.NewNodeSet(tmpEligible...).Difference(types.NewNodeSet(currentNodes...)) - tmpPossibleSorted := tmpPossible.ListNodes() - if len(tmpPossibleSorted) < i+1 { + possible = types.NewNodeSet(eligible...).Difference(types.NewNodeSet(currentNodes...)) + possibleSorted = possible.ListNodes() + if len(possibleSorted) < i+1 { errMsg := fmt.Sprintf( "Not enough nodes to meet desire: %d replicas desired, %d currentNodes, %d eligible. Scheduled on %d nodes instead.", rcFields.ReplicasDesired, len(currentNodes), len(eligible), i, From 077e6d858e91ce5eb7d706380c2c71365a7dd7bb Mon Sep 17 00:00:00 2001 From: Jason Price Date: Fri, 15 Nov 2019 17:37:45 -0500 Subject: [PATCH 07/10] rebuild test From 0d25c260d446ba4e385a16b060a67dd6944687c6 Mon Sep 17 00:00:00 2001 From: Jason Price Date: Fri, 15 Nov 2019 18:38:00 -0500 Subject: [PATCH 08/10] rebuild test From 0c8cc6078b3d9b10ca69122870e3e87dc3b2a9cb Mon Sep 17 00:00:00 2001 From: Jason Price Date: Fri, 15 Nov 2019 18:50:09 -0500 Subject: [PATCH 09/10] back to non-colon equal --- pkg/rc/replication_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rc/replication_controller.go b/pkg/rc/replication_controller.go index 5221586ae..bb2b2e8aa 100644 --- a/pkg/rc/replication_controller.go +++ b/pkg/rc/replication_controller.go @@ -362,7 +362,7 @@ func (rc *replicationController) addPods(rcFields fields.RC, current types.PodLo } scheduleOn := possibleSorted[i] - err := rc.schedule(txn, rcFields, scheduleOn) + err = rc.schedule(txn, rcFields, scheduleOn) if err != nil { return err } From b20857e444a031112d2bed3f6b3abd25c21f7b96 Mon Sep 17 00:00:00 2001 From: Jason Price Date: Mon, 18 Nov 2019 07:57:46 -0500 Subject: [PATCH 10/10] test had insufficient scaffolding --- pkg/rc/replication_controller_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/rc/replication_controller_test.go b/pkg/rc/replication_controller_test.go index f2ef462c3..06341b442 100644 --- a/pkg/rc/replication_controller_test.go +++ b/pkg/rc/replication_controller_test.go @@ -867,6 +867,7 @@ func TestAddPods(t *testing.T) { ID: rc.rcID, ReplicasDesired: 5, Manifest: testManifest(), + NodeSelector: klabels.Everything().Add("nodeQuality", klabels.EqualsOperator, []string{"good"}), } // empty