Merge pull request #8136 from omerap12/add-metric-for-failed-in-place

Add metric for failed in-place update attempt

Author: k8s-ci-robot

vertical-pod-autoscaler/docs/features.md

@@ -126,4 +126,5 @@ VPA provides metrics to track in-place update operations:
 * `vpa_in_place_updatable_pods_total`: Number of pods matching in-place update criteria
 * `vpa_in_place_updated_pods_total`: Number of pods successfully updated in-place
 * `vpa_vpas_with_in_place_updatable_pods_total`: Number of VPAs with pods eligible for in-place updates
-* `vpa_vpas_with_in_place_updated_pods_total`: Number of VPAs with successfully in-place updated pods
+* `vpa_vpas_with_in_place_updated_pods_total`: Number of VPAs with successfully in-place updated pods
+* `vpa_updater_failed_in_place_update_attempts_total`: Number of failed attempts to update pods in-place.

vertical-pod-autoscaler/pkg/updater/logic/updater.go

@@ -284,6 +284,7 @@ func (u *updater) RunOnce(ctx context.Context) {
 			err := inPlaceLimiter.InPlaceUpdate(pod, vpa, u.eventRecorder)
 			if err != nil {
 				klog.V(0).InfoS("In-place update failed", "error", err, "pod", klog.KObj(pod))
+				metrics_updater.RecordFailedInPlaceUpdate(vpaSize, "InPlaceUpdateError")
 				continue
 			}
 			withInPlaceUpdated = true

vertical-pod-autoscaler/pkg/utils/metrics/updater/updater.go

@@ -108,15 +108,21 @@ var (
 		}, []string{"vpa_size_log2"},
 	)
 
-	// TODO: Add metrics for failed in-place update attempts
+	failedInPlaceUpdateAttempts = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Name:      "failed_in_place_update_attempts_total",
+			Help:      "Number of failed attempts to update Pods in-place.",
+		}, []string{"vpa_size_log2", "reason"},
+	)
 
 	functionLatency = metrics.CreateExecutionTimeMetric(metricsNamespace,
 		"Time spent in various parts of VPA Updater main loop.")
 )
 
 // Register initializes all metrics for VPA Updater
 func Register() {
-	prometheus.MustRegister(controlledCount, evictableCount, evictedCount, vpasWithEvictablePodsCount, vpasWithEvictedPodsCount, inPlaceUpdatableCount, inPlaceUpdatedCount, vpasWithInPlaceUpdatablePodsCount, vpasWithInPlaceUpdatedPodsCount, functionLatency)
+	prometheus.MustRegister(controlledCount, evictableCount, evictedCount, vpasWithEvictablePodsCount, vpasWithEvictedPodsCount, inPlaceUpdatableCount, inPlaceUpdatedCount, vpasWithInPlaceUpdatablePodsCount, vpasWithInPlaceUpdatedPodsCount, failedInPlaceUpdateAttempts, functionLatency)
 }
 
 // NewExecutionTimer provides a timer for Updater's RunOnce execution
@@ -179,6 +185,12 @@ func AddInPlaceUpdatedPod(vpaSize int) {
 	inPlaceUpdatedCount.WithLabelValues(strconv.Itoa(log2)).Inc()
 }
 
+// RecordFailedInPlaceUpdate increases the counter of failed in-place update attempts by given VPA size and reason
+func RecordFailedInPlaceUpdate(vpaSize int, reason string) {
+	log2 := metrics.GetVpaSizeLog2(vpaSize)
+	failedInPlaceUpdateAttempts.WithLabelValues(strconv.Itoa(log2), reason).Inc()
+}
+
 // Add increases the counter for the given VPA size
 func (g *SizeBasedGauge) Add(vpaSize int, value int) {
 	log2 := metrics.GetVpaSizeLog2(vpaSize)