Enhance Kubernetes resource pressure testing and metrics collection

jobs/competitive-test.yml

@@ -58,6 +58,11 @@ jobs:
   timeoutInMinutes: ${{ parameters.timeout_in_minutes }}
   condition: or(eq(variables['Build.Reason'], 'Manual'), and(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Build.SourceBranchName'], 'main')))
   steps:
+  - script: |
+      sed -i "/name[[:space:]]*=[[:space:]]*\"os-sku\"/,+2s/value[[:space:]]*=[[:space:]]*\"[^\"]*/value = \"$OS_SKU/g" $(Pipeline.Workspace)/s/scenarios/perf-eval/k8s-node-stress/terraform-inputs/azure.tfvars
+      sed -i "/name[[:space:]]*=[[:space:]]*\"userpool1\"/,+2s/vm_size[[:space:]]*=[[:space:]]*\"[^\"]*/vm_size = \"$VM_SIZE/g" $(Pipeline.Workspace)/s/scenarios/perf-eval/k8s-node-stress/terraform-inputs/azure.tfvars
+      cat $(pwd)/scenarios/perf-eval/k8s-node-stress/terraform-inputs/azure.tfvars
+    displayName: "Set TF Overrides"
   - template: /steps/setup-tests.yml
     parameters:
       cloud: ${{ parameters.cloud }}

modules/python/clusterloader2/cri/config/config.yaml

@@ -3,12 +3,14 @@ name: resource-consumer
 {{$deploymentSize := DefaultParam .CL2_DEPLOYMENT_SIZE 10}}
 {{$memory := DefaultParam .CL2_RESOURCE_CONSUME_MEMORY "100"}}
 {{$memoryKi := DefaultParam .CL2_RESOURCE_CONSUME_MEMORY_KI "100"}}
+{{$testMemoryKi := DefaultParam .CL2_TEST_RESOURCE_CONSUME_MEMORY_KI $memoryKi}}
 {{$cpu := DefaultParam .CL2_RESOURCE_CONSUME_CPU 100}}
 {{$repeats := DefaultParam .CL2_REPEATS 1}}
 
 {{$steps := DefaultParam .CL2_STEPS 1}}
 {{$nodePerStep := DefaultParam .CL2_NODE_PER_STEP 1}}
 {{$totalNodes := MultiplyInt $nodePerStep $steps}}
+{{$nodeToMeasure := DefaultParam .CL2_NODE_TO_MEASURE ""}}
 {{$replicas := MultiplyInt $deploymentSize $totalNodes}}
 {{$scaleReplicas := MultiplyInt $deploymentSize $nodePerStep}}
 {{$scaleEnabled := DefaultParam .CL2_SCALE_ENABLED false}}
@@ -39,26 +41,31 @@ tuningSets:
 steps:
   - name: Start measurements
     measurements:
+      - Identifier: TestMetrics
+        Method: TestMetrics
+        Params:
+          action: start
+          systemPodMetricsEnabled: true
+          clusterOOMsTrackerEnabled: true
+          clusterOOMsIgnoredProcesses: ""
+          enableRestartCountCheck: true
+          labelSelector: group = resource-consumer
       - Identifier: PodStartupLatency
         Method: PodStartupLatency
         Params:
           action: start
           labelSelector: group = resource-consumer
-          threshold: {{$podStartupLatencyThreshold}}
+          threshold: "60s"
       - Identifier: ResourceUsageSummary
         Method: ResourceUsageSummary
         Params:
           action: start
           labelSelector: group = resource-consumer
-      - Identifier: WaitForRunningLatencyDeployments
-        Method: WaitForControlledPodsRunning
+      - Identifier: SchedulingThroughput
+        Method: SchedulingThroughput
         Params:
           action: start
-          checkIfPodsAreUpdated: true
-          apiVersion: apps/v1
-          kind: Deployment
           labelSelector: group = resource-consumer
-          operationTimeout: {{$operationTimeout}}
 
 {{range $i := Loop $repeats}}
 
@@ -76,6 +83,12 @@ steps:
         action: start
   {{end}}
 
+  - module:
+      path: /node-measurements.yaml
+      params:
+        action: start
+        node: {{$nodeToMeasure}}
+
   {{range $j := Loop $steps}}
   - name: Create deployment {{$j}}
     phases:
@@ -101,7 +114,7 @@ steps:
           {{if eq $osType "windows"}}
           Memory: {{$memory}}
           {{else}}
-          Memory: {{$memoryKi}}K
+          Memory: {{$testMemoryKi}}K
           {{end}}
           CPU: --millicores={{$cpu}}
           MemoryRequest: {{$memoryKi}}
@@ -112,36 +125,56 @@ steps:
           OSType: {{$osType}}
           HostNetwork: {{$hostNetwork}}
 
-  - name: Waiting for latency pods to be running
-    measurements:
-      - Identifier: WaitForRunningLatencyDeployments
-        Method: WaitForControlledPodsRunning
-        Params:
-          action: gather
-
   - name: Wait for resource consumption
     measurements:
       - Identifier: Sleep
         Method: Sleep
         Params:
           duration: 1m
+  {{end}}
 
-  - name: Wait for nodes to be ready
-    measurements:
-      - Identifier: ConfirmNodeCount
-        Method: WaitForNodes
-        Params:
-          action: start
-        {{if $scaleEnabled}}
-          minDesiredNodeCount: {{MultiplyInt (AddInt (MultiplyInt $nodePerStep (AddInt $j 1)) 1) 0.8}}
-          maxDesiredNodeCount: {{AddInt $totalNodes 1}}
+  {{range $j := Loop $steps}}
+  - name: Create deployment {{$j}}
+    phases:
+    - namespaceRange:
+          min: 1
+          max: 1
+      replicasPerNamespace: 1
+      tuningSet: Uniform1qps
+      objectBundle:
+      - basename: resource-consumer-{{$j}}
+        objectTemplatePath: deployment_template.yaml
+        templateFillMap:
+      {{if $scaleEnabled}}
+        {{if eq $j 0}}
+          Replicas: {{AddInt $scaleReplicas $deploymentSize}}
         {{else}}
-          minDesiredNodeCount: {{MultiplyInt $totalNodes 0.8}}
-          maxDesiredNodeCount: {{$totalNodes}}
+          Replicas: {{$scaleReplicas}}
         {{end}}
-          labelSelector: cri-resource-consume = true
-          timeout: 1m
-          refreshInterval: 5s
+      {{else}}
+          Replicas: {{$replicas}}
+      {{end}}
+          Group: resource-consumer
+          {{if eq $osType "windows"}}
+          Memory: {{$memory}}
+          {{else}}
+          Memory: {{$testMemoryKi}}K
+          {{end}}
+          CPU: --millicores={{$cpu}}
+          MemoryRequest: {{$memoryKi}}
+          CPURequest: {{$cpu}}m
+          LoadType: {{$loadType}}
+          Provider: {{$provider}}
+          RegistryEndpoint: {{$registryEndpoint}}
+          OSType: {{$osType}}
+          HostNetwork: {{$hostNetwork}}
+
+  - name: Wait for resource consumption
+    measurements:
+      - Identifier: Sleep
+        Method: Sleep
+        Params:
+          duration: 30s
   {{end}}
 
   {{if $scrapeKubelets}}
@@ -158,6 +191,12 @@ steps:
         action: gather
   {{end}}
 
+  - module:
+      path: /node-measurements.yaml
+      params:
+        action: gather
+        node: {{$nodeToMeasure}}
+
   {{range $j := Loop $steps}}
   - name: Deleting deployments {{$j}}
     phases:
@@ -169,18 +208,18 @@ steps:
         objectBundle:
           - basename: resource-consumer-{{$j}}
             objectTemplatePath: deployment_template.yaml
-
-  - name: Waiting for latency pods to be deleted
-    measurements:
-      - Identifier: WaitForRunningLatencyDeployments
-        Method: WaitForControlledPodsRunning
-        Params:
-          action: gather
   {{end}}
 {{end}}
 
   - name: Collect measurements
     measurements:
+      - Identifier: TestMetrics
+        Method: TestMetrics
+        Params:
+          action: gather
+          systemPodMetricsEnabled: true
+          clusterOOMsTrackerEnabled: true
+          enableRestartCountCheck: true
       - Identifier: ResourceUsageSummary
         Method: ResourceUsageSummary
         Params:
@@ -189,3 +228,7 @@ steps:
         Method: PodStartupLatency
         Params:
           action: gather
+      - Identifier: SchedulingThroughput
+        Method: SchedulingThroughput
+        Params:
+          action: gather

modules/python/clusterloader2/cri/config/deployment_template.yaml

@@ -55,11 +55,11 @@ spec:
           - stress
         args:
           - --vm
-          - "1"
+          - "3"
           - --vm-bytes
           - {{$Memory}}
           - --vm-hang
-          - "0"
+          - "30"
           - --timeout
           - "3600"
         {{end}}

modules/python/clusterloader2/cri/config/node-measurements.yaml

@@ -0,0 +1,45 @@
+{{$action := .action}} # start, gather
+{{$node := .node}}
+
+steps:
+  - name: {{$action}} Node Resource Measurements
+    measurements:
+    - Identifier: ResourceMetrics
+      Method: GenericPrometheusQuery
+      Params:
+        action: {{$action}}
+        metricName: NodeResourceMetrics
+        metricVersion: v1
+        unit: mixed
+        queries:
+        # Node Level Summary
+        - name: NodeInfo
+          query: kube_node_info{node = "{{$node}}"}
+        - name: NodeMemoryAllocatableGiB
+          query: sum(kube_node_status_allocatable{resource="memory", node = "{{$node}}"}) / 1073741824
+        - name: NodeMemoryCapacityGiB
+          query: sum(kube_node_status_capacity{resource="memory", node = "{{$node}}"}) / 1073741824
+        - name: NodesReady
+          query: changes(kube_node_status_condition{condition="Ready",status="true", node = "{{$node}}"}[5m])
+        - name: NodePressureStatus
+          query: changes(kube_node_status_condition{condition="MemoryPressure", status="true", node = "{{$node}}"}[5m])
+        # Node Memory Usage Stats (from kubelet/cAdvisor - container metrics aggregated by node)
+        - name: NodeMemoryUsageTotalGiB
+          query: sum(container_memory_working_set_bytes{instance="{{$node}}"}) / 1073741824
+        - name: NodeMemoryRequestsTotalGiB
+          query: sum(kube_pod_container_resource_requests{resource="memory", node = "{{$node}}"}) / 1073741824
+        - name: NodeMemoryCommitmentGiB
+          query: (sum(kube_pod_container_resource_limits{resource="memory", node = "{{$node}}"}) - sum(kube_node_status_allocatable{resource="memory", node = "{{$node}}"})) / 1073741824
+        - name: NodeMemoryRequestToAllocatableRatio
+          query: sum(kube_pod_container_resource_requests{resource="memory", node = "{{$node}}"}) / sum(kube_node_status_allocatable{resource="memory", node = "{{$node}}"})
+        - name: NodeMemoryUsageToAllocatableRatio
+          query: sum(container_memory_working_set_bytes{instance="{{$node}}"}) / sum(kube_node_status_allocatable{resource="memory", node = "{{$node}}"})
+        - name: NodeMemoryUsageToLimitsRatio
+          query: sum(container_memory_working_set_bytes{instance="{{$node}}"}) / sum(kube_pod_container_resource_limits{resource="memory", node = "{{$node}}"})
+        # Container Level Summary
+        - name: ContainerRuntimes
+          query: count(container_runtime_version{node = "{{$node}}"})
+        - name: ContainerMemoryFailures
+          query: increase(container_memory_failcnt{node = "{{$node}}"}[5m])
+        - name: ContainersNearMemoryLimit
+          query: count((container_memory_working_set_bytes{instance="{{$node}}"} / container_spec_memory_limit_bytes{instance="{{$node}}"} > 0.8))

modules/python/clusterloader2/cri/cri.py

@@ -12,14 +12,13 @@
 setup_logging()
 logger = get_logger(__name__)
 
-MEMORY_SCALE_FACTOR = 0.95 # 95% of the total allocatable memory to account for error margin
-
 # TODO: Refactor to use a config dataclass to reduce number of arguments
 # Reference: modules/python/clusterloader2/job_controller/job_controller.py
 def override_config_clusterloader2(
-    node_count, node_per_step, max_pods, repeats, operation_timeout,
+    node_count, node_to_measure, node_per_step, max_pods, repeats, operation_timeout,
     load_type, scale_enabled, pod_startup_latency_threshold, provider,
-    registry_endpoint, os_type, scrape_kubelets, scrape_containerd, containerd_scrape_interval, host_network, override_file):
+    registry_endpoint, os_type, scrape_kubelets, scrape_containerd, containerd_scrape_interval, host_network, override_file, use_custom_kubelet = False):
+    MEMORY_SCALE_FACTOR = 1.0
     client = KubernetesClient(os.path.expanduser("~/.kube/config"))
     nodes = client.get_nodes(label_selector="cri-resource-consume=true")
     if len(nodes) == 0:
@@ -51,7 +50,7 @@ def override_config_clusterloader2(
     # Calculate request cpu and memory for each pod
     daemonset_count = client.get_daemonsets_pods_count("kube-system", node.metadata.name)
     logger.info(f"Node {node.metadata.name} has {daemonset_count} daemonset pods")
-    pod_count = max_pods - daemonset_count
+    pod_count = max_pods
     cpu_request = cpu_value // pod_count
     memory_request_in_ki = math.ceil(memory_value * MEMORY_SCALE_FACTOR // pod_count)
     memory_request_in_k = int(memory_request_in_ki // 1.024)
@@ -75,11 +74,13 @@ def override_config_clusterloader2(
         file.write(f"CL2_DEPLOYMENT_SIZE: {pod_count}\n")
         file.write(f"CL2_RESOURCE_CONSUME_MEMORY: {memory_request}\n")
         file.write(f"CL2_RESOURCE_CONSUME_MEMORY_KI: {memory_request_in_ki}Ki\n")
+        file.write(f"CL2_TEST_RESOURCE_CONSUME_MEMORY_KI: {5*memory_request_in_ki}Ki\n")
         file.write(f"CL2_RESOURCE_CONSUME_CPU: {cpu_request}\n")
         file.write(f"CL2_REPEATS: {repeats}\n")
         file.write(f"CL2_NODE_COUNT: {node_count}\n")
         file.write(f"CL2_NODE_PER_STEP: {node_per_step}\n")
         file.write(f"CL2_STEPS: {steps}\n")
+        file.write(f"CL2_NODE_TO_MEASURE: {node_to_measure}\n")
         file.write(f"CL2_OPERATION_TIMEOUT: {operation_timeout}\n")
         file.write(f"CL2_LOAD_TYPE: {load_type}\n")
         file.write(f"CL2_SCALE_ENABLED: {str(scale_enabled).lower()}\n")
@@ -139,6 +140,17 @@ def verify_measurement():
         except k8s_client.ApiException as e:
             logger.error(f"Error fetching metrics: {e}")
 
+def parse_node_reader_report(file_path, template):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        logger.info(f"Processing node_reader metrics report")
+
+        template["group"] = "self"
+        template["measurement"] = "NodeReader"
+
+        node_reader_data = json.loads(file.read())
+        template["data"] = node_reader_data
+        return template
+
 def collect_clusterloader2(
     node_count,
     max_pods,
@@ -190,6 +202,9 @@ def collect_clusterloader2(
     for f in os.listdir(cl2_report_dir):
         file_path = os.path.join(cl2_report_dir, f)
         with open(file_path, 'r', encoding='utf-8') as file:
+            if file_path.endswith("node_reader.json"):
+                content += json.dumps(parse_node_reader_report(file_path, template)) + "\n"
+                continue
             measurement, group_name = get_measurement(file_path)
             if not measurement:
                 continue
@@ -216,6 +231,12 @@ def collect_clusterloader2(
                     template["percentile"] = "dataItems"
                     template["data"] = item
                     content += json.dumps(template) + "\n"
+            else:
+                result = template.copy()
+                result["group"] = group_name
+                result["measurement"] = measurement
+                result["data"] = data
+                content += json.dumps(result) + "\n"
 
     os.makedirs(os.path.dirname(result_file), exist_ok=True)
     with open(result_file, 'w', encoding='utf-8') as file:
@@ -228,6 +249,9 @@ def main():
     # Sub-command for override_config_clusterloader2
     parser_override = subparsers.add_parser("override", help="Override CL2 config file")
     parser_override.add_argument("--node_count", type=int, help="Number of nodes")
+    parser_override.add_argument(
+        "--node_to_measure", type=str, default="", help="Name of the node to gather detailed measurements from"
+    )
     parser_override.add_argument(
         "--node_per_step", type=int, help="Number of nodes to scale per step"
     )
@@ -382,6 +406,7 @@ def main():
     if args.command == "override":
         override_config_clusterloader2(
             args.node_count,
+            args.node_to_measure,
             args.node_per_step,
             args.max_pods,
             args.repeats,