OPNET-783: statusmanager: a DaemonSet matching zero nodes is not progressing

pkg/controller/statusmanager/pod_status.go

@@ -112,7 +112,11 @@ func (status *StatusManager) SetFromPods() {
 			if !isNonCritical(ds) {
 				clbo = append(clbo, status.CheckCrashLoopBackOffPods(dsName, ds.Spec.Selector.MatchLabels, "DaemonSet")...)
 			}
-		} else if ds.Status.NumberAvailable == 0 && dsRolloutActive {
+		} else if ds.Status.NumberAvailable == 0 && (ds.Status.DesiredNumberScheduled > 0 || ds.Status.ObservedGeneration < ds.Generation) && dsRolloutActive {
+			// A DaemonSet whose observed status desires zero pods (e.g. its
+			// node selector matches no nodes) is fully rolled out. Only treat
+			// NumberAvailable == 0 as "unscheduled" if pods are desired or the
+			// DaemonSet controller has not yet observed the current generation.
 			progressing = append(progressing, fmt.Sprintf("DaemonSet %q is not yet scheduled on any nodes", dsName.String()))
 			dsProgressing = true
 		}

pkg/controller/statusmanager/status_manager_test.go

@@ -1618,6 +1618,140 @@ func TestStatusManagerIgnoresDaemonSetNodeScaleChurnAfterInstall(t *testing.T) {
 	}
 }
 
+func TestStatusManagerDaemonSetMatchingZeroNodes(t *testing.T) {
+	// The availability level check compares the release annotation against
+	// RELEASE_VERSION; pin it so an inherited environment cannot skew the
+	// Available assertions below.
+	t.Setenv("RELEASE_VERSION", "")
+	client := fake.NewFakeClient()
+	status := New(client, "testing", names.StandAloneClusterName)
+	status.clock = testingclock.NewFakeClock(time.Now())
+	setFakeListers(status)
+	no := &operv1.Network{ObjectMeta: metav1.ObjectMeta{Name: names.OPERATOR_CONFIG}}
+	setOC(t, client, no)
+
+	// A DaemonSet whose scheduling constraints match zero nodes (e.g.
+	// frr-k8s on a compact cluster, where the control plane nodes run a
+	// static pod instead) is fully rolled out. It must not be reported as
+	// Progressing or Degraded, even during install.
+	ds := &appsv1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "one", Name: "alpha", Generation: 1, Labels: sl},
+		Spec: appsv1.DaemonSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"app": "alpha"},
+			},
+		},
+		Status: appsv1.DaemonSetStatus{
+			CurrentNumberScheduled: 0,
+			DesiredNumberScheduled: 0,
+			UpdatedNumberScheduled: 0,
+			NumberAvailable:        0,
+			NumberReady:            0,
+			ObservedGeneration:     0,
+		},
+	}
+	// While the controller has not observed the generation yet, zero desired
+	// pods proves nothing: the DaemonSet must still report Progressing.
+	set(t, client, ds)
+	status.SetFromPods()
+
+	_, oc, err := getStatuses(client, "testing")
+	if err != nil {
+		t.Fatalf("error getting ClusterOperator: %v", err)
+	}
+	if !conditionsInclude(oc.Status.Conditions, []operv1.OperatorCondition{
+		{
+			Type:   operv1.OperatorStatusTypeProgressing,
+			Status: operv1.ConditionTrue,
+		},
+	}) {
+		t.Fatalf("unexpected Status.Conditions: %#v", oc.Status.Conditions)
+	}
+
+	// Once observed with zero desired pods, it is fully rolled out.
+	ds.Status.ObservedGeneration = 1
+	setStatus(t, client, ds)
+	status.SetFromPods()
+
+	_, oc, err = getStatuses(client, "testing")
+	if err != nil {
+		t.Fatalf("error getting ClusterOperator: %v", err)
+	}
+	if !conditionsInclude(oc.Status.Conditions, []operv1.OperatorCondition{
+		{
+			Type:   operv1.OperatorStatusTypeDegraded,
+			Status: operv1.ConditionFalse,
+		},
+		{
+			Type:   operv1.OperatorStatusTypeProgressing,
+			Status: operv1.ConditionFalse,
+		},
+		{
+			Type:   operv1.OperatorStatusTypeUpgradeable,
+			Status: operv1.ConditionTrue,
+		},
+		{
+			Type:   operv1.OperatorStatusTypeAvailable,
+			Status: operv1.ConditionTrue,
+		},
+	}) {
+		t.Fatalf("unexpected Status.Conditions: %#v", oc.Status.Conditions)
+	}
+
+	// The DaemonSet must not be tracked as an in-flight rollout, so it can
+	// never feed the hung-rollout degraded path.
+	ps := getLastPodState(t, client, "testing")
+	if len(ps.DaemonsetStates) != 0 {
+		t.Fatalf("expected no daemonset rollout state for a daemonset matching zero nodes: %#v", ps.DaemonsetStates)
+	}
+
+	// Even with stale last-seen state (as if it had been tracked as
+	// progressing for over ProgressTimeout), it must not be reported hung.
+	ps.DaemonsetStates = []daemonsetState{{
+		ClusteredName:  ClusteredName{Namespace: "one", Name: "alpha"},
+		LastSeenStatus: ds.Status,
+		LastChangeTime: time.Now().Add(-time.Hour),
+	}}
+	ps.InstallComplete = false
+	setLastPodState(t, client, "testing", ps)
+	status.SetFromPods()
+	// RolloutHung is debounced for 2 min, so advance clock and call again
+	status.clock.(*testingclock.FakeClock).Step(3 * time.Minute)
+	status.SetFromPods()
+
+	_, oc, err = getStatuses(client, "testing")
+	if err != nil {
+		t.Fatalf("error getting ClusterOperator: %v", err)
+	}
+	if !conditionsInclude(oc.Status.Conditions, []operv1.OperatorCondition{
+		{
+			Type:   operv1.OperatorStatusTypeDegraded,
+			Status: operv1.ConditionFalse,
+		},
+		{
+			Type:   operv1.OperatorStatusTypeProgressing,
+			Status: operv1.ConditionFalse,
+		},
+	}) {
+		t.Fatalf("unexpected Status.Conditions: %#v", oc.Status.Conditions)
+	}
+
+	// The stale state must have been pruned, not marked hung.
+	ps = getLastPodState(t, client, "testing")
+	if len(ps.DaemonsetStates) != 0 {
+		t.Fatalf("expected stale daemonset rollout state to be pruned: %#v", ps.DaemonsetStates)
+	}
+
+	err = client.ClientFor("").CRClient().Get(t.Context(), types.NamespacedName{Namespace: "one", Name: "alpha"}, ds)
+	if err != nil {
+		t.Fatalf("error getting DaemonSet: %v", err)
+	}
+	if val, set := ds.GetAnnotations()[names.RolloutHungAnnotation]; set {
+		t.Fatalf("Expected no rollout-hung annotation, but was present %s", val)
+	}
+
+}
+
 func TestStatusManagerSetFromDeployments(t *testing.T) {
 	client := fake.NewFakeClient()
 	status := New(client, "testing", names.StandAloneClusterName)