diff --git a/pkg/controller/statusmanager/pod_status.go b/pkg/controller/statusmanager/pod_status.go index 6e83c910f9..f8797c399c 100644 --- a/pkg/controller/statusmanager/pod_status.go +++ b/pkg/controller/statusmanager/pod_status.go @@ -112,7 +112,11 @@ func (status *StatusManager) SetFromPods() { if !isNonCritical(ds) { clbo = append(clbo, status.CheckCrashLoopBackOffPods(dsName, ds.Spec.Selector.MatchLabels, "DaemonSet")...) } - } else if ds.Status.NumberAvailable == 0 && dsRolloutActive { + } else if ds.Status.NumberAvailable == 0 && (ds.Status.DesiredNumberScheduled > 0 || ds.Status.ObservedGeneration < ds.Generation) && dsRolloutActive { + // A DaemonSet whose observed status desires zero pods (e.g. its + // node selector matches no nodes) is fully rolled out. Only treat + // NumberAvailable == 0 as "unscheduled" if pods are desired or the + // DaemonSet controller has not yet observed the current generation. progressing = append(progressing, fmt.Sprintf("DaemonSet %q is not yet scheduled on any nodes", dsName.String())) dsProgressing = true } diff --git a/pkg/controller/statusmanager/status_manager_test.go b/pkg/controller/statusmanager/status_manager_test.go index 6911e0c0e5..564c06e7c6 100644 --- a/pkg/controller/statusmanager/status_manager_test.go +++ b/pkg/controller/statusmanager/status_manager_test.go @@ -1618,6 +1618,140 @@ func TestStatusManagerIgnoresDaemonSetNodeScaleChurnAfterInstall(t *testing.T) { } } +func TestStatusManagerDaemonSetMatchingZeroNodes(t *testing.T) { + // The availability level check compares the release annotation against + // RELEASE_VERSION; pin it so an inherited environment cannot skew the + // Available assertions below. + t.Setenv("RELEASE_VERSION", "") + client := fake.NewFakeClient() + status := New(client, "testing", names.StandAloneClusterName) + status.clock = testingclock.NewFakeClock(time.Now()) + setFakeListers(status) + no := &operv1.Network{ObjectMeta: metav1.ObjectMeta{Name: names.OPERATOR_CONFIG}} + setOC(t, client, no) + + // A DaemonSet whose scheduling constraints match zero nodes (e.g. + // frr-k8s on a compact cluster, where the control plane nodes run a + // static pod instead) is fully rolled out. It must not be reported as + // Progressing or Degraded, even during install. + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "one", Name: "alpha", Generation: 1, Labels: sl}, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "alpha"}, + }, + }, + Status: appsv1.DaemonSetStatus{ + CurrentNumberScheduled: 0, + DesiredNumberScheduled: 0, + UpdatedNumberScheduled: 0, + NumberAvailable: 0, + NumberReady: 0, + ObservedGeneration: 0, + }, + } + // While the controller has not observed the generation yet, zero desired + // pods proves nothing: the DaemonSet must still report Progressing. + set(t, client, ds) + status.SetFromPods() + + _, oc, err := getStatuses(client, "testing") + if err != nil { + t.Fatalf("error getting ClusterOperator: %v", err) + } + if !conditionsInclude(oc.Status.Conditions, []operv1.OperatorCondition{ + { + Type: operv1.OperatorStatusTypeProgressing, + Status: operv1.ConditionTrue, + }, + }) { + t.Fatalf("unexpected Status.Conditions: %#v", oc.Status.Conditions) + } + + // Once observed with zero desired pods, it is fully rolled out. + ds.Status.ObservedGeneration = 1 + setStatus(t, client, ds) + status.SetFromPods() + + _, oc, err = getStatuses(client, "testing") + if err != nil { + t.Fatalf("error getting ClusterOperator: %v", err) + } + if !conditionsInclude(oc.Status.Conditions, []operv1.OperatorCondition{ + { + Type: operv1.OperatorStatusTypeDegraded, + Status: operv1.ConditionFalse, + }, + { + Type: operv1.OperatorStatusTypeProgressing, + Status: operv1.ConditionFalse, + }, + { + Type: operv1.OperatorStatusTypeUpgradeable, + Status: operv1.ConditionTrue, + }, + { + Type: operv1.OperatorStatusTypeAvailable, + Status: operv1.ConditionTrue, + }, + }) { + t.Fatalf("unexpected Status.Conditions: %#v", oc.Status.Conditions) + } + + // The DaemonSet must not be tracked as an in-flight rollout, so it can + // never feed the hung-rollout degraded path. + ps := getLastPodState(t, client, "testing") + if len(ps.DaemonsetStates) != 0 { + t.Fatalf("expected no daemonset rollout state for a daemonset matching zero nodes: %#v", ps.DaemonsetStates) + } + + // Even with stale last-seen state (as if it had been tracked as + // progressing for over ProgressTimeout), it must not be reported hung. + ps.DaemonsetStates = []daemonsetState{{ + ClusteredName: ClusteredName{Namespace: "one", Name: "alpha"}, + LastSeenStatus: ds.Status, + LastChangeTime: time.Now().Add(-time.Hour), + }} + ps.InstallComplete = false + setLastPodState(t, client, "testing", ps) + status.SetFromPods() + // RolloutHung is debounced for 2 min, so advance clock and call again + status.clock.(*testingclock.FakeClock).Step(3 * time.Minute) + status.SetFromPods() + + _, oc, err = getStatuses(client, "testing") + if err != nil { + t.Fatalf("error getting ClusterOperator: %v", err) + } + if !conditionsInclude(oc.Status.Conditions, []operv1.OperatorCondition{ + { + Type: operv1.OperatorStatusTypeDegraded, + Status: operv1.ConditionFalse, + }, + { + Type: operv1.OperatorStatusTypeProgressing, + Status: operv1.ConditionFalse, + }, + }) { + t.Fatalf("unexpected Status.Conditions: %#v", oc.Status.Conditions) + } + + // The stale state must have been pruned, not marked hung. + ps = getLastPodState(t, client, "testing") + if len(ps.DaemonsetStates) != 0 { + t.Fatalf("expected stale daemonset rollout state to be pruned: %#v", ps.DaemonsetStates) + } + + err = client.ClientFor("").CRClient().Get(t.Context(), types.NamespacedName{Namespace: "one", Name: "alpha"}, ds) + if err != nil { + t.Fatalf("error getting DaemonSet: %v", err) + } + if val, set := ds.GetAnnotations()[names.RolloutHungAnnotation]; set { + t.Fatalf("Expected no rollout-hung annotation, but was present %s", val) + } + +} + func TestStatusManagerSetFromDeployments(t *testing.T) { client := fake.NewFakeClient() status := New(client, "testing", names.StandAloneClusterName)