diff --git a/modules/python/clusterloader2/autoscale/autoscale.py b/modules/python/clusterloader2/autoscale/autoscale.py index 188ca5e4c3..9e91efe68d 100644 --- a/modules/python/clusterloader2/autoscale/autoscale.py +++ b/modules/python/clusterloader2/autoscale/autoscale.py @@ -5,7 +5,7 @@ import subprocess from datetime import datetime, timezone -from clusterloader2.utils import parse_xml_to_json, run_cl2_command +from clusterloader2.utils import parse_xml_to_json, run_cl2_command, process_cl2_reports from clients.kubernetes_client import KubernetesClient from utils.logger_config import get_logger, setup_logging @@ -60,38 +60,125 @@ def calculate_cpu_request_for_clusterloader2(node_label_selector, node_count, po cpu_request = int(cpu_request * 0.95) return cpu_request -def override_config_clusterloader2(cpu_per_node, node_count, pod_count, scale_up_timeout, scale_down_timeout, loop_count, node_label_selector, node_selector, override_file, warmup_deployment, cl2_config_dir, os_type="linux", warmup_deployment_template="", deployment_template=""): - logger.info(f"CPU per node: {cpu_per_node}") +def override_config_clusterloader2(cpu_per_node, node_count, pod_count, scale_up_timeout, scale_down_timeout, loop_count, node_label_selector, node_selector, override_file, warmup_deployment, cl2_config_dir, os_type="linux", warmup_deployment_template="", deployment_template="", pod_cpu_request=0, pod_memory_request="", cl2_config_file="config.yaml"): desired_node_count = 1 if warmup_deployment in ["true", "True"]: warmup_deployment_for_karpeneter(cl2_config_dir, warmup_deployment_template) desired_node_count = 0 - cpu_request = calculate_cpu_request_for_clusterloader2(node_label_selector, node_count, pod_count, warmup_deployment, cl2_config_dir, warmup_deployment_template) - - logger.info(f"Total number of nodes: {node_count}, total number of pods: {pod_count}") - logger.info(f"CPU request for each pod: {cpu_request}m") - - # assuming the number of surge nodes is no more than 10 + + logger.info(f"Overriding CL2 config file at: {cl2_config_file}") + is_complex = cl2_config_file == "ms_complex_config.yaml" + if not is_complex: + pod_cpu_request = calculate_cpu_request_for_clusterloader2(node_label_selector, node_count, pod_count, warmup_deployment, cl2_config_dir, warmup_deployment_template) + with open(override_file, 'w', encoding='utf-8') as file: - file.write(f"CL2_DEPLOYMENT_CPU: {cpu_request}m\n") - file.write(f"CL2_MIN_NODE_COUNT: {node_count}\n") - file.write(f"CL2_MAX_NODE_COUNT: {node_count + 10}\n") - file.write(f"CL2_DESIRED_NODE_COUNT: {desired_node_count}\n") + file.write(f"CL2_DEPLOYMENT_CPU: {pod_cpu_request}m\n") + file.write(f"CL2_DEPLOYMENT_MEMORY: {pod_memory_request}\n") file.write(f"CL2_DEPLOYMENT_SIZE: {pod_count}\n") file.write(f"CL2_SCALE_UP_TIMEOUT: {scale_up_timeout}\n") file.write(f"CL2_SCALE_DOWN_TIMEOUT: {scale_down_timeout}\n") file.write(f"CL2_LOOP_COUNT: {loop_count}\n") - file.write(f"CL2_NODE_LABEL_SELECTOR: {node_label_selector}\n") + + if not is_complex: + file.write(f"CL2_MIN_NODE_COUNT: {node_count}\n") + file.write(f"CL2_MAX_NODE_COUNT: {node_count + 10}\n") + file.write(f"CL2_DESIRED_NODE_COUNT: {desired_node_count}\n") + file.write(f"CL2_NODE_LABEL_SELECTOR: {node_label_selector}\n") + file.write(f"CL2_NODE_SELECTOR: \"{node_selector}\"\n") file.write(f"CL2_OS_TYPE: {os_type}\n") - if deployment_template !='': + + if deployment_template: file.write(f"CL2_DEPLOYMENT_TEMPLATE_PATH: {deployment_template}\n") - file.close() -def execute_clusterloader2(cl2_image, cl2_config_dir, cl2_report_dir, kubeconfig, provider): - run_cl2_command(kubeconfig, cl2_image, cl2_config_dir, cl2_report_dir, provider, overrides=True) +def execute_clusterloader2(cl2_image, cl2_config_dir, cl2_report_dir, kubeconfig, provider, cl2_config_file="config.yaml"): + run_cl2_command(kubeconfig, cl2_image, cl2_config_dir, cl2_report_dir, provider, cl2_config_file, overrides=True) + +def _build_report_template(capacity_type, pod_count, cloud_info, run_id, run_url, autoscale_type="up", cpu_per_node=None, node_count=None, data=None, is_complex=False, pod_cpu_request=0, pod_memory_request=""): + """Build CL2 measurement template""" + result = { + "timestamp": datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), + "autoscale_type": autoscale_type, + "capacity_type": capacity_type, + "pod_count": pod_count, + "data": data if data is not None else {}, + "cloud_info": cloud_info, + "run_id": run_id, + "run_url": run_url + } + + # Only add node-specific info for non-complex configs + if not is_complex: + result["cpu_per_node"] = cpu_per_node + result["node_count"] = node_count + if is_complex: # cl2 measurement + result["group"] = None + result["measurement"] = None + result["result"] = None + result["pod_memory"] = pod_memory_request + result["pod_cpu"] = pod_cpu_request + + return result + +def _process_test_results(testsuites, index_pattern, metric_mappings, cpu_per_node, capacity_type, node_count, pod_count, cloud_info, run_id, run_url, is_complex_config=False, pod_cpu_request=0, pod_memory_request=""): + """Process test results and generate JSON content""" + summary = {} + + # Define which metrics to include in data based on config type + if is_complex_config: + data_metrics = ["wait_for_pods_seconds"] + else: + data_metrics = [ + "wait_for_nodes_seconds", + "wait_for_50Perc_nodes_seconds", + "wait_for_70Perc_nodes_seconds", + "wait_for_90Perc_nodes_seconds", + "wait_for_99Perc_nodes_seconds", + "wait_for_pods_seconds" + ] + + # Process each loop + for testcase in testsuites[0]["testcases"]: + name = testcase["name"] + index = -1 + match = index_pattern.search(name) + if match: + index = match.group() + if index not in summary: + summary[index] = { + "up": { "failures": 0 }, + "down": { "failures": 0 } + } + else: + continue + + failure = testcase["failure"] + for test_key, (category, summary_key) in metric_mappings.items(): + if test_key in name: + summary[index][category][summary_key] = -1 if failure else testcase["time"] + summary[index][category]["failures"] += 1 if failure else 0 + break # Exit loop once matched + + content = "" + for index, inner_dict in summary.items(): + for key, value in inner_dict.items(): + # Build data dict dynamically based on available metrics + data = {metric: value.get(metric) for metric in data_metrics} + data["autoscale_result"] = "success" if value["failures"] == 0 else "failure" + + # For complex configs, don't include cpu_per_node and node_count + result = _build_report_template( + capacity_type, pod_count, cloud_info, run_id, run_url, + autoscale_type=key, + cpu_per_node=cpu_per_node, + node_count=node_count, + data=data, is_complex=is_complex_config, pod_cpu_request=pod_cpu_request, pod_memory_request=pod_memory_request + ) + content += json.dumps(result) + "\n" + + return content def collect_clusterloader2( cpu_per_node, @@ -102,83 +189,53 @@ def collect_clusterloader2( cloud_info, run_id, run_url, - result_file + result_file, + cl2_config_file, + pod_cpu_request, + pod_memory_request ): index_pattern = re.compile(r'(\d+)$') raw_data = parse_xml_to_json(os.path.join(cl2_report_dir, "junit.xml"), indent = 2) json_data = json.loads(raw_data) testsuites = json_data["testsuites"] - summary = {} - metric_mappings = { - "WaitForRunningPodsUp": ("up", "wait_for_pods_seconds"), - "WaitForNodesUpPerc50": ("up", "wait_for_50Perc_nodes_seconds"), - "WaitForNodesUpPerc70": ("up", "wait_for_70Perc_nodes_seconds"), - "WaitForNodesUpPerc90": ("up", "wait_for_90Perc_nodes_seconds"), - "WaitForNodesUpPerc99": ("up", "wait_for_99Perc_nodes_seconds"), - "WaitForNodesUpPerc100": ("up", "wait_for_nodes_seconds"), - "WaitForRunningPodsDown": ("down", "wait_for_pods_seconds"), - "WaitForNodesDownPerc50": ("down", "wait_for_50Perc_nodes_seconds"), - "WaitForNodesDownPerc70": ("down", "wait_for_70Perc_nodes_seconds"), - "WaitForNodesDownPerc90": ("down", "wait_for_90Perc_nodes_seconds"), - "WaitForNodesDownPerc99": ("down", "wait_for_99Perc_nodes_seconds"), - "WaitForNodesDownPerc100": ("down", "wait_for_nodes_seconds"), - } + + # Different metric mappings based on config file type + is_complex_config = "ms_complex_config.yaml" == cl2_config_file + + if is_complex_config: + # Metric mappings for complex config + metric_mappings = { + "WaitForRunningPodsUp": ("up", "wait_for_pods_seconds"), + "WaitForRunningPodsDown": ("down", "wait_for_pods_seconds"), + } + else: + # Metric mappings for standard config + metric_mappings = { + "WaitForRunningPodsUp": ("up", "wait_for_pods_seconds"), + "WaitForNodesUpPerc50": ("up", "wait_for_50Perc_nodes_seconds"), + "WaitForNodesUpPerc70": ("up", "wait_for_70Perc_nodes_seconds"), + "WaitForNodesUpPerc90": ("up", "wait_for_90Perc_nodes_seconds"), + "WaitForNodesUpPerc99": ("up", "wait_for_99Perc_nodes_seconds"), + "WaitForNodesUpPerc100": ("up", "wait_for_nodes_seconds"), + "WaitForRunningPodsDown": ("down", "wait_for_pods_seconds"), + "WaitForNodesDownPerc50": ("down", "wait_for_50Perc_nodes_seconds"), + "WaitForNodesDownPerc70": ("down", "wait_for_70Perc_nodes_seconds"), + "WaitForNodesDownPerc90": ("down", "wait_for_90Perc_nodes_seconds"), + "WaitForNodesDownPerc99": ("down", "wait_for_99Perc_nodes_seconds"), + "WaitForNodesDownPerc100": ("down", "wait_for_nodes_seconds"), + } if testsuites: - # Process each loop - for testcase in testsuites[0]["testcases"]: - name = testcase["name"] - index = -1 - match = index_pattern.search(name) - if match: - index = match.group() - if index not in summary: - summary[index] = { - "up": { "failures": 0 }, - "down": { "failures": 0 } - } - else: - continue - - failure = testcase["failure"] - for test_key, (category, summary_key) in metric_mappings.items(): - if test_key in name: - summary[index][category][summary_key] = -1 if failure else testcase["time"] - summary[index][category]["failures"] += 1 if failure else 0 - break # Exit loop once matched - - content = "" - for index, inner_dict in summary.items(): - for key, value in inner_dict.items(): - data = { - "wait_for_nodes_seconds": value["wait_for_nodes_seconds"], - "wait_for_50Perc_nodes_seconds": value["wait_for_50Perc_nodes_seconds"], - "wait_for_70Perc_nodes_seconds": value["wait_for_70Perc_nodes_seconds"], - "wait_for_90Perc_nodes_seconds": value["wait_for_90Perc_nodes_seconds"], - "wait_for_99Perc_nodes_seconds": value["wait_for_99Perc_nodes_seconds"], - "wait_for_pods_seconds": value["wait_for_pods_seconds"], - "autoscale_result": "success" if value["failures"] == 0 else "failure" - } - # TODO: Expose optional parameter to include test details - result = { - "timestamp": datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), - "autoscale_type": key, - "cpu_per_node": cpu_per_node, - "capacity_type": capacity_type, - "node_count": node_count, - "pod_count": pod_count, - "data": data, - # "raw_data": raw_data, - "cloud_info": cloud_info, - "run_id": run_id, - "run_url": run_url - } - content += json.dumps(result) + "\n" + content = _process_test_results(testsuites, index_pattern, metric_mappings, cpu_per_node, capacity_type, node_count, pod_count, cloud_info, run_id, run_url, is_complex_config, pod_cpu_request, pod_memory_request) else: raise Exception(f"No testsuites found in the report! Raw data: {raw_data}") - + if is_complex_config: + cl2_measurement = _build_report_template(capacity_type, pod_count, cloud_info, run_id, run_url, data={}, is_complex=is_complex_config, pod_cpu_request=pod_cpu_request, pod_memory_request=pod_memory_request) + cl2_result = process_cl2_reports(cl2_report_dir, cl2_measurement) + logger.info(f"Result, category up: {cl2_result}") + content += cl2_result os.makedirs(os.path.dirname(result_file), exist_ok=True) with open(result_file, 'w', encoding='utf-8') as file: file.write(content) @@ -203,7 +260,9 @@ def main(): parser_override.add_argument("--os_type", type=str, choices=["linux", "windows"], default="linux", help="Operating system type for the node pools") parser_override.add_argument("--warmup_deployment_template", type=str, default="", help="Path to the CL2 warm up deployment file") parser_override.add_argument("--deployment_template", type=str, default="", help="Path to the CL2 deployment file") - + parser_override.add_argument("--pod_cpu_request", type=int, help="CPU request for each pod") + parser_override.add_argument("--pod_memory_request", type=str, help="Memory request for each pod") + parser_override.add_argument("--cl2_config_file", default="config.yaml", type=str, default="config.yaml", help="name of CL2 config file") # Sub-command for execute_clusterloader2 parser_execute = subparsers.add_parser("execute", help="Execute scale up operation") parser_execute.add_argument("cl2_image", type=str, help="Name of the CL2 image") @@ -211,7 +270,7 @@ def main(): parser_execute.add_argument("cl2_report_dir", type=str, help="Path to the CL2 report directory") parser_execute.add_argument("kubeconfig", type=str, help="Path to the kubeconfig file") parser_execute.add_argument("provider", type=str, help="Cloud provider name") - + parser_execute.add_argument("--cl2_config_file", default="config.yaml", type=str, default="config.yaml", help="Path to the CL2 config file") # Sub-command for collect_clusterloader2 parser_collect = subparsers.add_parser("collect", help="Collect scale up data") parser_collect.add_argument("cpu_per_node", type=int, help="Name of cpu cores per node") @@ -223,15 +282,18 @@ def main(): parser_collect.add_argument("run_id", type=str, help="Run ID") parser_collect.add_argument("run_url", type=str, help="Run URL") parser_collect.add_argument("result_file", type=str, help="Path to the result file") - + parser_collect.add_argument("--cl2_config_file", default="config.yaml", type=str, help="Path to the CL2 config file") + parser_collect.add_argument("--pod_cpu_request", type=int, help="CPU request for each pod") + parser_collect.add_argument("--pod_memory_request", type=str, help="Memory request for each pod") + args = parser.parse_args() if args.command == "override": - override_config_clusterloader2(args.cpu_per_node, args.node_count, args.pod_count, args.scale_up_timeout, args.scale_down_timeout, args.loop_count, args.node_label_selector, args.node_selector, args.cl2_override_file, args.warmup_deployment, args.cl2_config_dir, args.os_type, args.warmup_deployment_template, args.deployment_template) + override_config_clusterloader2(args.cpu_per_node, args.node_count, args.pod_count, args.scale_up_timeout, args.scale_down_timeout, args.loop_count, args.node_label_selector, args.node_selector, args.cl2_override_file, args.warmup_deployment, args.cl2_config_dir, args.os_type, args.warmup_deployment_template, args.deployment_template,pod_cpu_request=args.pod_cpu_request, pod_memory_request=args.pod_memory_request, cl2_config_file=args.cl2_config_file) elif args.command == "execute": - execute_clusterloader2(args.cl2_image, args.cl2_config_dir, args.cl2_report_dir, args.kubeconfig, args.provider) + execute_clusterloader2(args.cl2_image, args.cl2_config_dir, args.cl2_report_dir, args.kubeconfig, args.provider, args.cl2_config_file) elif args.command == "collect": - collect_clusterloader2(args.cpu_per_node, args.capacity_type, args.node_count, args.pod_count, args.cl2_report_dir, args.cloud_info, args.run_id, args.run_url, args.result_file) + collect_clusterloader2(args.cpu_per_node, args.capacity_type, args.node_count, args.pod_count, args.cl2_report_dir, args.cloud_info, args.run_id, args.run_url, args.result_file, args.cl2_config_file, args.pod_cpu_request, args.pod_memory_request) if __name__ == "__main__": main() diff --git a/modules/python/clusterloader2/autoscale/config/ms_complex_config.yaml b/modules/python/clusterloader2/autoscale/config/ms_complex_config.yaml new file mode 100644 index 0000000000..56e6d15dcf --- /dev/null +++ b/modules/python/clusterloader2/autoscale/config/ms_complex_config.yaml @@ -0,0 +1,165 @@ +{{$deploymentTemplatePath := DefaultParam .CL2_DEPLOYMENT_TEMPLATE_PATH "deployment_template.yaml"}} +{{$deploymentSize := DefaultParam .CL2_DEPLOYMENT_SIZE 100}} +{{$deploymentCpu := DefaultParam .CL2_DEPLOYMENT_CPU "346m"}} +{{$deploymentMemory := DefaultParam .CL2_DEPLOYMENT_MEMORY "100Mi"}} +{{$nodeSelector := DefaultParam .CL2_NODE_SELECTOR "{karpenter.sh/nodepool: default}"}} +{{$podLabelSelector := DefaultParam .CL2_POD_LABEL_SELECTOR "app = inflate"}} +{{$scaleUpTimeout := DefaultParam .CL2_SCALE_UP_TIMEOUT "30m"}} +{{$scaleDownTimeout := DefaultParam .CL2_SCALE_DOWN_TIMEOUT "10m"}} +{{$refreshInterval := DefaultParam .CL2_REFRESH_INTERVAL "5s"}} +{{$loopCount := DefaultParam .CL2_LOOP_COUNT 1}} +{{$coolDownTime := DefaultParam .CL2_COOLDOWN_TIME "120s"}} +{{$osType := DefaultParam .CL2_OS_TYPE "linux"}} +{{$countErrorMargin := MultiplyInt .CL2_DEPLOYMENT_SIZE 0.01}} + +name: autoscale +namespace: + number: 1 + prefix: autoscale + deleteStaleNamespaces: true + deleteAutomanagedNamespaces: true + enableExistingNamespaces: true + +tuningSets: +- name: Uniform1qps + qpsLoad: + qps: 20 + +steps: +{{range $i := Loop $loopCount}} +- name: Start Measurements {{$i}} + measurements: + - Identifier: ResourceUsageSummary + Method: ResourceUsageSummary + Params: + action: start + - Identifier: PodStartupLatency + Method: PodStartupLatency + Params: + action: start + labelSelector: {{$podLabelSelector}} + threshold: {{$scaleUpTimeout}} + - Identifier: SchedulingThroughput + Method: SchedulingThroughput + Params: + action: start + labelSelector: {{$podLabelSelector}} +- name: Create deployment {{$i}} + phases: + - namespaceRange: + min: 1 + max: 1 + replicasPerNamespace: 1 + tuningSet: Uniform1qps + objectBundle: + - basename: inflate + objectTemplatePath: {{$deploymentTemplatePath}} + templateFillMap: + Replicas: {{$deploymentSize}} + CPUperJob: {{$deploymentCpu}} + MemoryRequest: {{$deploymentMemory}} + NodeSelector: {{ (StructuralData $nodeSelector) }} + OSType: {{$osType}} +- name: Measure nodes and pods scale up {{$i}} + measurements: + - Identifier: WaitForRunningPodsUp {{$i}} + Method: WaitForRunningPods + Params: + action: start + desiredPodCount: {{$deploymentSize}} + countErrorMargin: {{$countErrorMargin}} + labelSelector: {{$podLabelSelector}} + timeout: {{$scaleUpTimeout}} + refreshInterval: {{$refreshInterval}} +- name: Capture Metrics After Scale Up {{$i}} + measurements: + - Identifier: ResourceMetrics{{$i}} + Method: GenericPrometheusQuery + Params: + action: start + metricName: Resource Metrics Summary + metricVersion: v1 + unit: mixed + queries: + # Node Level Summary + - name: TotalNodes + query: count(kube_node_status_allocatable{resource="cpu"}) + - name: NodeCPUAllocatable + query: sum(kube_node_status_allocatable{resource="cpu"}) + - name: NodeMemoryAllocatable + query: sum(kube_node_status_allocatable{resource="memory"}) + # Node CPU Usage Stats (from kubelet/cAdvisor - container metrics aggregated by node) + - name: NodeCPUUsageAvg + query: avg(sum by (instance) (rate(container_cpu_usage_seconds_total{id="/"}[2m]))) + - name: NodeCPUUsageMax + query: max(sum by (instance) (rate(container_cpu_usage_seconds_total{id="/"}[2m]))) + - name: NodeCPUUsageMin + query: min(sum by (instance) (rate(container_cpu_usage_seconds_total{id="/"}[2m]))) + # Node Memory Usage Stats (from kubelet/cAdvisor - container metrics aggregated by node) + - name: NodeMemoryUsageAvg + query: avg(sum by (instance) (container_memory_working_set_bytes{id="/"})) + - name: NodeMemoryUsageMax + query: max(sum by (instance) (container_memory_working_set_bytes{id="/"})) + - name: NodeMemoryUsageMin + query: min(sum by (instance) (container_memory_working_set_bytes{id="/"})) + # Pod Level Summary + - name: TotalPods + query: count(kube_pod_status_phase{phase="Running"}) + # Pod Distribution Summary + - name: PodsPerNodeAvg + query: avg(count by (node) (kube_pod_info{node!=""})) + - name: PodsPerNodeMax + query: max(count by (node) (kube_pod_info{node!=""})) + - name: PodsPerNodeMin + query: min(count by (node) (kube_pod_info{node!=""})) +- name: Gather Measurements {{$i}} + measurements: + - Identifier: PodStartupLatency + Method: PodStartupLatency + Params: + action: gather + - Identifier: SchedulingThroughput + Method: SchedulingThroughput + Params: + action: gather + - Identifier: ResourceUsageSummary + Method: ResourceUsageSummary + Params: + action: gather + - Identifier: ResourceMetrics{{$i}} + Method: GenericPrometheusQuery + Params: + action: gather +- name: WaitBeforeDelete + measurements: + - Identifier: WaitBeforeDelete + Method: Sleep + Params: + action: start + duration: {{$coolDownTime}} +- name: Delete deployment {{$i}} + phases: + - namespaceRange: + min: 1 + max: 1 + replicasPerNamespace: 0 + tuningSet: Uniform1qps + objectBundle: + - basename: inflate + objectTemplatePath: {{$deploymentTemplatePath}} + templateFillMap: + Replicas: {{$deploymentSize}} + CPUperJob: {{$deploymentCpu}} + MemoryRequest: {{$deploymentMemory}} + OSType: {{$osType}} +- name: Measure nodes and pods scale down {{$i}} + measurements: + - Identifier: WaitForRunningPodsDown {{$i}} + Method: WaitForRunningPods + Params: + action: start + desiredPodCount: 0 + labelSelector: {{$podLabelSelector}} + timeout: {{$scaleDownTimeout}} + refreshInterval: {{$refreshInterval}} +{{end}} \ No newline at end of file diff --git a/pipelines/perf-eval/Autoscale Benchmark/node-auto-provisioning-benchmark-complex.yml b/pipelines/perf-eval/Autoscale Benchmark/node-auto-provisioning-benchmark-complex.yml index 75286b09ba..e9cbe5cf73 100644 --- a/pipelines/perf-eval/Autoscale Benchmark/node-auto-provisioning-benchmark-complex.yml +++ b/pipelines/perf-eval/Autoscale Benchmark/node-auto-provisioning-benchmark-complex.yml @@ -30,19 +30,19 @@ stages: topology: karpenter matrix: complex-nap: - cpu_per_node: 2 - node_count: 5 - pod_count: 5 + pod_count: 5000 + pod_cpu_request: 16 + pod_memory_request: "60Gi" scale_up_timeout: "15m" scale_down_timeout: "15m" - node_label_selector: "karpenter.sh/nodepool = default" node_selector: "{karpenter.sh/nodepool: default}" loop_count: 1 warmup_deployment: true warmup_deployment_template: warmup_deployment.yaml - vm_size: Standard_D2s_v4 capacity_type: on-demand + cl2_config_file: "ms_complex_config.yaml" + karpenter_nodepool_file: "karpenter_complex_nodepool.azure.yml" max_parallel: 1 - timeout_in_minutes: 60 + timeout_in_minutes: 120 credential_type: service_connection ssh_key_enabled: false diff --git a/scenarios/perf-eval/nap/kubernetes/karpenter_complex_nodepool.azure.yml b/scenarios/perf-eval/nap/kubernetes/karpenter_complex_nodepool.azure.yml new file mode 100644 index 0000000000..a2758a6b6b --- /dev/null +++ b/scenarios/perf-eval/nap/kubernetes/karpenter_complex_nodepool.azure.yml @@ -0,0 +1,137 @@ +# Shared AKSNodeClass (common for both Spot and On-Demand) +--- +apiVersion: karpenter.azure.com/v1alpha2 +kind: AKSNodeClass +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose AKSNodeClass for running Ubuntu2204 nodes" +spec: + imageFamily: Ubuntu2204 + +# On-Demand NodePool (default) +--- +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose On-Demand NodePool" +spec: + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 1m + budgets: + - nodes: "100%" + template: + spec: + nodeClassRef: + group: karpenter.azure.com + kind: AKSNodeClass + name: default + expireAfter: Never + requirements: + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.azure.com/sku-name + operator: In + values: + - "Standard_D96ds_v5" # 55k DDSv5 + - "Standard_D96d_v5" + - "Standard_D96_v5" # 100k Dv5 + - "Standard_D96s_v5" + - key: topology.kubernetes.io/zone + operator: In + values: + - eastus2-1 + - eastus2-2 + - eastus2-3 + +# Spot NodePool +--- +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: spot + annotations: + kubernetes.io/description: "Spot NodePool for burstable cost-efficient workloads" +spec: + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 1s + budgets: + - nodes: "100%" + template: + spec: + nodeClassRef: + group: karpenter.azure.com + kind: AKSNodeClass + name: default + expireAfter: Never + requirements: + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + - key: karpenter.azure.com/sku-name + operator: In + values: [Standard_D2_v5] +# system-surge NodePool +--- +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: system-surge + annotations: + kubernetes.io/description: "Surge capacity pool for system pod pressure" +spec: + disruption: + budgets: + - nodes: "1" + consolidateAfter: 1m + consolidationPolicy: WhenEmpty + template: + metadata: + labels: + kubernetes.azure.com/ebpf-dataplane: "cilium" + kubernetes.azure.com/mode: "system" + spec: + expireAfter: Never + nodeClassRef: + group: karpenter.azure.com + kind: AKSNodeClass + name: default + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.azure.com/sku-name + operator: In + values: + - Standard_D16_v3 + - key: topology.kubernetes.io/zone + operator: In + values: + - eastus2-1 + - eastus2-2 + - eastus2-3 + startupTaints: + - effect: NoExecute + key: node.cilium.io/agent-not-ready + value: "true" + taints: + - effect: NoSchedule + key: CriticalAddonsOnly + value: "true" diff --git a/scenarios/perf-eval/nap/terraform-inputs/azure-complex.tfvars b/scenarios/perf-eval/nap/terraform-inputs/azure-complex.tfvars index 17e75e82ae..d7594125df 100644 --- a/scenarios/perf-eval/nap/terraform-inputs/azure-complex.tfvars +++ b/scenarios/perf-eval/nap/terraform-inputs/azure-complex.tfvars @@ -11,7 +11,6 @@ public_ip_config_list = [ } ] - network_config_list = [ { role = "crud" @@ -118,7 +117,6 @@ route_table_config_list = [ } ] - aks_cli_config_list = [ { role = "nap" diff --git a/steps/engine/clusterloader2/autoscale/collect.yml b/steps/engine/clusterloader2/autoscale/collect.yml index f5b99c4074..4177f1ef4f 100644 --- a/steps/engine/clusterloader2/autoscale/collect.yml +++ b/steps/engine/clusterloader2/autoscale/collect.yml @@ -15,8 +15,8 @@ steps: - script: | set -eo pipefail - PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE collect $CPU_PER_NODE ${CAPACITY_TYPE:-on-demand} $NODE_COUNT $POD_COUNT \ - $CL2_REPORT_DIR "$CLOUD_INFO" $RUN_ID $RUN_URL $TEST_RESULTS_FILE + PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE collect ${CPU_PER_NODE:-0} ${CAPACITY_TYPE:-on-demand} ${NODE_COUNT:-0} ${POD_COUNT:-0} \ + $CL2_REPORT_DIR "$CLOUD_INFO" $RUN_ID $RUN_URL $TEST_RESULTS_FILE --cl2_config_file ${CL2_CONFIG_FILE} --pod_cpu_request ${POD_CPU_REQUEST:-0} --pod_memory_request ${POD_MEMORY_REQUEST:-""} workingDirectory: modules/python env: CLOUD: ${{ parameters.cloud }} diff --git a/steps/engine/clusterloader2/autoscale/execute.yml b/steps/engine/clusterloader2/autoscale/execute.yml index 8e878f33a7..4ecc9afd75 100644 --- a/steps/engine/clusterloader2/autoscale/execute.yml +++ b/steps/engine/clusterloader2/autoscale/execute.yml @@ -13,11 +13,12 @@ steps: set -eo pipefail PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE override \ - $CPU_PER_NODE $NODE_COUNT $POD_COUNT \ + ${CPU_PER_NODE:-0} ${NODE_COUNT:-0} ${POD_COUNT:-0} \ $SCALE_UP_TIMEOUT $SCALE_DOWN_TIMEOUT \ - $LOOP_COUNT "$NODE_LABEL_SELECTOR" "$NODE_SELECTOR" ${CL2_CONFIG_DIR}/overrides.yaml ${WARMUP_DEPLOYMENT:-false} ${CL2_CONFIG_DIR} --os_type ${OS_TYPE:-linux} --warmup_deployment_template ${WARMUP_DEPLOYMENT_TEMPLATE:-""} --deployment_template ${DEPLOYMENT_TEMPLATE:-""} + $LOOP_COUNT "${NODE_LABEL_SELECTOR:-""}" "$NODE_SELECTOR" ${CL2_CONFIG_DIR}/overrides.yaml ${WARMUP_DEPLOYMENT:-false} ${CL2_CONFIG_DIR} --os_type ${OS_TYPE:-linux} --warmup_deployment_template ${WARMUP_DEPLOYMENT_TEMPLATE:-""} --deployment_template ${DEPLOYMENT_TEMPLATE:-""} \ + --pod_cpu_request ${POD_CPU_REQUEST:-0} --pod_memory_request ${POD_MEMORY_REQUEST:-""} --cl2_config_file ${CL2_CONFIG_FILE:-config.yaml} PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE execute \ - ${CL2_IMAGE} ${CL2_CONFIG_DIR} $CL2_REPORT_DIR ${HOME}/.kube/config $CLOUD + ${CL2_IMAGE} ${CL2_CONFIG_DIR} $CL2_REPORT_DIR ${HOME}/.kube/config $CLOUD --cl2_config_file ${CL2_CONFIG_FILE:-config.yaml} workingDirectory: modules/python env: ${{ if eq(parameters.cloud, 'azure') }}: diff --git a/steps/topology/karpenter/validate-resources.yml b/steps/topology/karpenter/validate-resources.yml index 85b669aade..8c1039cfe4 100644 --- a/steps/topology/karpenter/validate-resources.yml +++ b/steps/topology/karpenter/validate-resources.yml @@ -5,6 +5,9 @@ parameters: type: string - name: regions type: object +- name: karpenter_nodepool_file + type: string + default: '' steps: - template: /steps/cloud/${{ parameters.cloud }}/update-kubeconfig.yml @@ -21,17 +24,28 @@ steps: - bash: | set -euo pipefail set -x - kubectl apply -f $KARPENTER_NODEPOOL_FILE + + if [ -z "$KARPENTER_NODEPOOL_FILE" ]; then + echo "karpenter_nodepool_file parameter is null or empty. Using default." + KARPENTER_NODEPOOL_FILE=$DEFAULT_KARPENTER_FILE + fi + echo "Using karpenter_nodepool_file: $KARPENTER_NODEPOOL_FILE" + + kubectl apply -f $KARPENTER_DIR/$KARPENTER_NODEPOOL_FILE - # Patch the On-Demand NodePool - kubectl patch nodepool default --type='json' -p="[{'op': 'replace', 'path': '/spec/template/spec/requirements/2/values', 'value': ['$VM_SIZE']}]" + if [ -n "$VM_SIZE" ]; then + # Patch the On-Demand NodePool + kubectl patch nodepool default --type='json' -p="[{'op': 'replace', 'path': '/spec/template/spec/requirements/2/values', 'value': ['$VM_SIZE']}]" - # Patch the Spot NodePool - kubectl patch nodepool spot --type='json' -p="[{'op': 'replace', 'path': '/spec/template/spec/requirements/2/values', 'value': ['$VM_SIZE']}]" + # Patch the Spot NodePool + kubectl patch nodepool spot --type='json' -p="[{'op': 'replace', 'path': '/spec/template/spec/requirements/2/values', 'value': ['$VM_SIZE']}]" + fi kubectl get nodepool default -o yaml kubectl get nodepool spot -o yaml env: CLOUD: ${{ parameters.cloud }} - KARPENTER_NODEPOOL_FILE: $(Pipeline.Workspace)/s/scenarios/$(SCENARIO_TYPE)/$(SCENARIO_NAME)/kubernetes/karpenter_nodepool.${{ parameters.cloud }}.yml + KARPENTER_DIR: $(Pipeline.Workspace)/s/scenarios/$(SCENARIO_TYPE)/$(SCENARIO_NAME)/kubernetes + KARPENTER_NODEPOOL_FILE: ${{ parameters.karpenter_nodepool_file }} + DEFAULT_KARPENTER_FILE: karpenter_nodepool.${{ parameters.cloud }}.yml displayName: "Validate Karpenter setup"