From e6198a54e24b9c820b976addca00bfe3a2188cb5 Mon Sep 17 00:00:00 2001 From: Sumanth Reddy Chinna Pullaiah Date: Mon, 1 Dec 2025 17:00:53 +0000 Subject: [PATCH 1/3] update --- .../azure/{network => }/firewall/README.md | 0 .../azure/{network => }/firewall/main.tf | 0 .../azure/{network => }/firewall/output.tf | 0 .../azure/{network => }/firewall/variables.tf | 0 modules/terraform/azure/main.tf | 20 ++- modules/terraform/azure/network/main.tf | 21 +-- modules/terraform/azure/network/variables.tf | 67 +--------- modules/terraform/azure/public-ip/output.tf | 15 ++- modules/terraform/azure/variables.tf | 122 +++++++++--------- 9 files changed, 95 insertions(+), 150 deletions(-) rename modules/terraform/azure/{network => }/firewall/README.md (100%) rename modules/terraform/azure/{network => }/firewall/main.tf (100%) rename modules/terraform/azure/{network => }/firewall/output.tf (100%) rename modules/terraform/azure/{network => }/firewall/variables.tf (100%) diff --git a/modules/terraform/azure/network/firewall/README.md b/modules/terraform/azure/firewall/README.md similarity index 100% rename from modules/terraform/azure/network/firewall/README.md rename to modules/terraform/azure/firewall/README.md diff --git a/modules/terraform/azure/network/firewall/main.tf b/modules/terraform/azure/firewall/main.tf similarity index 100% rename from modules/terraform/azure/network/firewall/main.tf rename to modules/terraform/azure/firewall/main.tf diff --git a/modules/terraform/azure/network/firewall/output.tf b/modules/terraform/azure/firewall/output.tf similarity index 100% rename from modules/terraform/azure/network/firewall/output.tf rename to modules/terraform/azure/firewall/output.tf diff --git a/modules/terraform/azure/network/firewall/variables.tf b/modules/terraform/azure/firewall/variables.tf similarity index 100% rename from modules/terraform/azure/network/firewall/variables.tf rename to modules/terraform/azure/firewall/variables.tf diff --git a/modules/terraform/azure/main.tf b/modules/terraform/azure/main.tf index 8de91ae1f9..c2737dd6dd 100644 --- a/modules/terraform/azure/main.tf +++ b/modules/terraform/azure/main.tf @@ -27,6 +27,7 @@ locals { aks_cli_custom_config_path = "${path.cwd}/../../../scenarios/${var.scenario_type}/${var.scenario_name}/config/aks_custom_config.json" all_subnets = merge([for network in var.network_config_list : module.virtual_network[network.role].subnets]...) + firewall_config_map = { for fw in var.firewall_config_list : fw.name => fw } updated_aks_config_list = length(var.aks_config_list) > 0 ? [ for aks in var.aks_config_list : merge( aks, @@ -75,8 +76,7 @@ module "virtual_network" { network_config = each.value resource_group_name = local.run_id location = local.region - public_ips = module.public_ips.pip_ids - public_ip_addresses = module.public_ips.pip_addresses + public_ips = module.public_ips.public_ips tags = local.tags } @@ -88,6 +88,22 @@ module "dns_zones" { tags = local.tags } +module "firewall" { + for_each = local.firewall_config_map + + source = "./firewall" + resource_group_name = local.run_id + location = local.region + tags = local.tags + + firewall_config = merge(each.value, { + subnet_id = local.all_subnets[each.value.subnet_name] + public_ip_address_id = module.public_ips.public_ips[each.value.public_ip_name].id + }) + + depends_on = [module.virtual_network] +} + module "aks" { for_each = local.aks_config_map diff --git a/modules/terraform/azure/network/main.tf b/modules/terraform/azure/network/main.tf index 7ec5188c1d..a6bcb82a14 100644 --- a/modules/terraform/azure/network/main.tf +++ b/modules/terraform/azure/network/main.tf @@ -1,4 +1,6 @@ locals { + public_ip_ids = { for name, ip in var.public_ips : name => ip.id } + public_ip_addresses = { for name, ip in var.public_ips : name => ip.ip_address } nsr_rules_map = { for rule in var.network_config.nsr_rules : rule.name => rule } nat_gateway_associations_map = var.network_config.nat_gateway_associations == null ? {} : { for nat in var.network_config.nat_gateway_associations : nat.nat_gateway_name => nat } input_route_tables_map = var.network_config.route_tables == null ? {} : { for rt in var.network_config.route_tables : rt.name => rt } @@ -8,7 +10,6 @@ locals { for subnet in azurerm_virtual_network.vnet.subnet : split("/", subnet.id)[length(split("/", subnet.id)) - 1] => subnet } - firewalls_input = var.network_config.firewalls == null ? {} : { for fw in var.network_config.firewalls : fw.name => fw } network_security_group_name = var.network_config.network_security_group_name expanded_nic_association_map = flatten([ for nic in var.network_config.nic_public_ip_associations : [ @@ -79,7 +80,7 @@ resource "azurerm_network_interface" "nic" { name = each.value.ip_configuration_name subnet_id = local.subnets_map[each.value.subnet_name].id private_ip_address_allocation = "Dynamic" - public_ip_address_id = each.value.public_ip_name != null ? var.public_ips[each.value.public_ip_name] : null + public_ip_address_id = each.value.public_ip_name != null ? local.public_ip_ids[each.value.public_ip_name] : null } } @@ -113,22 +114,6 @@ module "nat_gateway" { tags = local.tags } -module "firewall" { - source = "./firewall" - for_each = local.firewalls_input - - firewall_config = merge(each.value, { - subnet_id = local.subnets_map[each.value.subnet_name].id - public_ip_address_id = var.public_ips[each.value.public_ip_name] - }) - - resource_group_name = var.resource_group_name - location = var.location - tags = local.tags - - depends_on = [azurerm_virtual_network.vnet] -} - module "route_table" { source = "./route-table" for_each = local.input_route_tables_map diff --git a/modules/terraform/azure/network/variables.tf b/modules/terraform/azure/network/variables.tf index b59246245f..ffdd1327fb 100644 --- a/modules/terraform/azure/network/variables.tf +++ b/modules/terraform/azure/network/variables.tf @@ -9,13 +9,8 @@ variable "location" { } variable "public_ips" { - description = "Map of public IP names to IDs" - type = map(string) -} - -variable "public_ip_addresses" { - description = "Map of public IP names to IP addresses" - type = map(string) + description = "Map of public IP names to their objects containing id and ip_address" + type = map(any) } variable "accelerated_networking" { @@ -64,64 +59,6 @@ variable "network_config" { public_ip_names = list(string) subnet_names = list(string) }))) - firewalls = optional(list(object({ - name = string - sku_name = optional(string, "AZFW_VNet") - sku_tier = optional(string, "Standard") - firewall_policy_id = optional(string, null) - threat_intel_mode = optional(string, "Alert") - dns_proxy_enabled = optional(bool, false) - dns_servers = optional(list(string), null) - subnet_name = string - public_ip_name = string - ip_configuration_name = optional(string, "firewall-ipconfig") - nat_rule_collections = optional(list(object({ - name = string - priority = number - action = optional(string, "Dnat") - rules = list(object({ - name = string - source_addresses = optional(list(string), []) - source_ip_groups = optional(list(string), []) - destination_ports = list(string) - destination_addresses = list(string) - translated_address = string - translated_port = string - protocols = list(string) - })) - })), []) - network_rule_collections = optional(list(object({ - name = string - priority = number - action = string # "Allow" or "Deny" - rules = list(object({ - name = string - source_addresses = optional(list(string), []) - source_ip_groups = optional(list(string), []) - destination_ports = list(string) - destination_addresses = optional(list(string), []) - destination_fqdns = optional(list(string), []) - destination_ip_groups = optional(list(string), []) - protocols = list(string) # "TCP", "UDP", "ICMP", "Any" - })) - })), []) - application_rule_collections = optional(list(object({ - name = string - priority = number - action = string # "Allow" or "Deny" - rules = list(object({ - name = string - source_addresses = optional(list(string), []) - source_ip_groups = optional(list(string), []) - target_fqdns = optional(list(string), []) - fqdn_tags = optional(list(string), []) - protocols = optional(list(object({ - port = string - type = string # "Http" or "Https" - })), []) - })) - })), []) - })), []) route_tables = optional(list(object({ name = string bgp_route_propagation_enabled = optional(bool, true) diff --git a/modules/terraform/azure/public-ip/output.tf b/modules/terraform/azure/public-ip/output.tf index 635a0c7f9e..165b9cd4a5 100644 --- a/modules/terraform/azure/public-ip/output.tf +++ b/modules/terraform/azure/public-ip/output.tf @@ -1,8 +1,9 @@ -output "pip_ids" { - value = { for ip in azurerm_public_ip.pip : ip.name => ip.id } -} - -output "pip_addresses" { - description = "Map of public IP names to their IP addresses" - value = { for ip in azurerm_public_ip.pip : ip.name => ip.ip_address } +output "public_ips" { + description = "Map of public IP names to their objects containing id and ip_address" + value = { + for ip in azurerm_public_ip.pip : ip.name => { + id = ip.id + ip_address = ip.ip_address + } + } } diff --git a/modules/terraform/azure/variables.tf b/modules/terraform/azure/variables.tf index 88e4737e18..672fdf7058 100644 --- a/modules/terraform/azure/variables.tf +++ b/modules/terraform/azure/variables.tf @@ -125,64 +125,6 @@ variable "network_config_list" { public_ip_names = list(string) subnet_names = list(string) }))) - firewalls = optional(list(object({ - name = string - sku_name = optional(string, "AZFW_VNet") - sku_tier = optional(string, "Standard") - firewall_policy_id = optional(string, null) - threat_intel_mode = optional(string, "Alert") - dns_proxy_enabled = optional(bool, false) - dns_servers = optional(list(string), null) - subnet_name = string - public_ip_name = string - ip_configuration_name = optional(string, "firewall-ipconfig") - nat_rule_collections = optional(list(object({ - name = string - priority = number - action = optional(string, "Dnat") - rules = list(object({ - name = string - source_addresses = optional(list(string), []) - source_ip_groups = optional(list(string), []) - destination_ports = list(string) - destination_addresses = list(string) - translated_address = string - translated_port = string - protocols = list(string) - })) - })), []) - network_rule_collections = optional(list(object({ - name = string - priority = number - action = string - rules = list(object({ - name = string - source_addresses = optional(list(string), []) - source_ip_groups = optional(list(string), []) - destination_ports = list(string) - destination_addresses = optional(list(string), []) - destination_fqdns = optional(list(string), []) - destination_ip_groups = optional(list(string), []) - protocols = list(string) - })) - })), []) - application_rule_collections = optional(list(object({ - name = string - priority = number - action = string - rules = list(object({ - name = string - source_addresses = optional(list(string), []) - source_ip_groups = optional(list(string), []) - target_fqdns = optional(list(string), []) - fqdn_tags = optional(list(string), []) - protocols = optional(list(object({ - port = string - type = string - })), []) - })) - })), []) - })),[]) route_tables = optional(list(object({ name = string bgp_route_propagation_enabled = optional(bool, true) @@ -200,6 +142,70 @@ variable "network_config_list" { default = [] } +variable "firewall_config_list" { + description = "List of Azure Firewall configurations" + type = list(object({ + name = string + network_role = string # Which network this firewall belongs to + sku_name = optional(string, "AZFW_VNet") + sku_tier = optional(string, "Standard") + firewall_policy_id = optional(string, null) + threat_intel_mode = optional(string, "Alert") + dns_proxy_enabled = optional(bool, false) + dns_servers = optional(list(string), null) + subnet_name = string + public_ip_name = string + ip_configuration_name = optional(string, "firewall-ipconfig") + nat_rule_collections = optional(list(object({ + name = string + priority = number + action = optional(string, "Dnat") + rules = list(object({ + name = string + source_addresses = optional(list(string), []) + source_ip_groups = optional(list(string), []) + destination_ports = list(string) + destination_addresses = list(string) + translated_address = string + translated_port = string + protocols = list(string) + })) + })), []) + network_rule_collections = optional(list(object({ + name = string + priority = number + action = string + rules = list(object({ + name = string + source_addresses = optional(list(string), []) + source_ip_groups = optional(list(string), []) + destination_ports = list(string) + destination_addresses = optional(list(string), []) + destination_fqdns = optional(list(string), []) + destination_ip_groups = optional(list(string), []) + protocols = list(string) + })) + })), []) + application_rule_collections = optional(list(object({ + name = string + priority = number + action = string + rules = list(object({ + name = string + source_addresses = optional(list(string), []) + source_ip_groups = optional(list(string), []) + target_fqdns = optional(list(string), []) + fqdn_tags = optional(list(string), []) + protocols = optional(list(object({ + port = string + type = string + })), []) + })) + })), []) + })) + default = [] +} + variable "dns_zones" { description = "List of DNS zones to create" type = list(object({ From fc02b806e303b54005738f0c8ee7ce8184176b81 Mon Sep 17 00:00:00 2001 From: xinWeiWei24 Date: Thu, 4 Dec 2025 16:12:16 +1100 Subject: [PATCH 2/3] Support AAD in aks-cli config (#954) Add a variable to control AAD enablement in the aks-cli module (aligned with the aks module) and include install-kubelogin steps for AAD support. --- modules/terraform/azure/aks-cli/main.tf | 13 ++ modules/terraform/azure/aks-cli/variables.tf | 8 +- modules/terraform/azure/main.tf | 3 +- .../azure/tests/test_aks_aad.tftest.hcl | 150 ++++++++++++++++++ .../topology/karpenter/validate-resources.yml | 6 + 5 files changed, 178 insertions(+), 2 deletions(-) diff --git a/modules/terraform/azure/aks-cli/main.tf b/modules/terraform/azure/aks-cli/main.tf index 238810f7b9..7dc7fa6f25 100644 --- a/modules/terraform/azure/aks-cli/main.tf +++ b/modules/terraform/azure/aks-cli/main.tf @@ -73,6 +73,16 @@ locals { "" ) + aad_parameter = ( + var.aks_aad_enabled == true ? + format( + "--enable-aad --enable-azure-rbac --aad-admin-group-object-ids %s --aad-tenant-id %s", + data.azurerm_client_config.current.object_id, + data.azurerm_client_config.current.tenant_id + ) + : "" + ) + custom_configurations = ( var.aks_cli_config.use_custom_configurations && var.aks_cli_custom_config_path != null ? format( @@ -108,6 +118,7 @@ locals { local.subnet_id_parameter, local.managed_identity_parameter, local.api_server_vnet_integration_parameter, + local.aad_parameter, ], local.default_node_pool_parameters)) aks_cli_destroy_command = join(" ", [ @@ -120,6 +131,8 @@ locals { ]) } +data "azurerm_client_config" "current" {} + resource "azurerm_user_assigned_identity" "userassignedidentity" { count = var.aks_cli_config.managed_identity_name == null ? 0 : 1 location = var.location diff --git a/modules/terraform/azure/aks-cli/variables.tf b/modules/terraform/azure/aks-cli/variables.tf index 724c1cc194..20342d5f68 100644 --- a/modules/terraform/azure/aks-cli/variables.tf +++ b/modules/terraform/azure/aks-cli/variables.tf @@ -28,6 +28,12 @@ variable "aks_cli_custom_config_path" { default = null } +variable "aks_aad_enabled" { + description = "Indicates whether Azure Active Directory integration is enabled for AKS" + type = bool + default = false +} + variable "aks_cli_config" { type = object({ role = string @@ -59,7 +65,7 @@ variable "aks_cli_config" { value = string })), []) })), []) - optional_parameters = optional(list(object({ # Refer to https://learn.microsoft.com/en-us/cli/azure/aks?view=azure-cli-latest#az-aks-create(aks-preview) for available parameters + optional_parameters = optional(list(object({ # Refer to https://learn.microsoft.com/en-us/cli/azure/aks?view=azure-cli-latest#az-aks-create(aks-preview) for available parameters name = string value = string })), []) diff --git a/modules/terraform/azure/main.tf b/modules/terraform/azure/main.tf index 17c4ea1a1e..cb8a2f0fb0 100644 --- a/modules/terraform/azure/main.tf +++ b/modules/terraform/azure/main.tf @@ -10,7 +10,7 @@ locals { aks_custom_headers = lookup(var.json_input, "aks_custom_headers", []) k8s_machine_type = lookup(var.json_input, "k8s_machine_type", null) k8s_os_disk_type = lookup(var.json_input, "k8s_os_disk_type", null) - aks_aad_enabled = lookup(var.json_input, "aks_aad_enabled", "false") + aks_aad_enabled = lookup(var.json_input, "aks_aad_enabled", false) enable_apiserver_vnet_integration = lookup(var.json_input, "enable_apiserver_vnet_integration", false) tags = { @@ -116,4 +116,5 @@ module "aks-cli" { tags = local.tags subnets_map = local.all_subnets aks_cli_custom_config_path = local.aks_cli_custom_config_path + aks_aad_enabled = local.aks_aad_enabled } diff --git a/modules/terraform/azure/tests/test_aks_aad.tftest.hcl b/modules/terraform/azure/tests/test_aks_aad.tftest.hcl index 9c3d546151..c8a3bbc24b 100644 --- a/modules/terraform/azure/tests/test_aks_aad.tftest.hcl +++ b/modules/terraform/azure/tests/test_aks_aad.tftest.hcl @@ -119,3 +119,153 @@ run "valid_aad_explicitly_disabled" { error_message = "Expected: AAD block should not exist when explicitly disabled" } } + +# ============================================================================ +# AAD Integration Tests for aks-cli module +# ============================================================================ + +# Test case 4: Verify AAD parameters are correctly generated in aks-cli when enabled +run "valid_aks_cli_aad_enabled" { + + command = apply + + variables { + json_input = { + "run_id" : "123456789", + "region" : "eastus", + "public_key_path" : "public_key_path", + "aks_aad_enabled" : true + } + + aks_config_list = [] + + aks_cli_config_list = [ + { + role = "client" + aks_name = "test-aad-cli" + sku_tier = "Standard" + use_aks_preview_cli_extension = true + default_node_pool = { + name = "default" + node_count = 2 + vm_size = "Standard_D2s_v3" + } + optional_parameters = [] + dry_run = true + } + ] + } + + # Verify the CLI command includes --enable-aad flag + assert { + condition = can(regex("--enable-aad", module.aks-cli["client"].aks_cli_command)) + error_message = "Expected: CLI command should include --enable-aad flag\nActual: ${module.aks-cli["client"].aks_cli_command}" + } + + # Verify the CLI command includes --enable-azure-rbac flag + assert { + condition = can(regex("--enable-azure-rbac", module.aks-cli["client"].aks_cli_command)) + error_message = "Expected: CLI command should include --enable-azure-rbac flag\nActual: ${module.aks-cli["client"].aks_cli_command}" + } + + # Verify the CLI command includes --aad-admin-group-object-ids with the mocked object_id + assert { + condition = can(regex("--aad-admin-group-object-ids 12345678-1234-5678-9abc-def012345678", module.aks-cli["client"].aks_cli_command)) + error_message = "Expected: CLI command should include --aad-admin-group-object-ids with correct object_id\nActual: ${module.aks-cli["client"].aks_cli_command}" + } + + # Verify the CLI command includes --aad-tenant-id with the mocked tenant_id + assert { + condition = can(regex("--aad-tenant-id 00000000-0000-0000-0000-000000000000", module.aks-cli["client"].aks_cli_command)) + error_message = "Expected: CLI command should include --aad-tenant-id with correct tenant_id\nActual: ${module.aks-cli["client"].aks_cli_command}" + } +} + +# Test case 5: Verify AAD parameters are omitted in aks-cli when disabled +run "valid_aks_cli_aad_disabled" { + + command = apply + + variables { + json_input = { + "run_id" : "123456789", + "region" : "eastus", + "public_key_path" : "public_key_path", + "aks_aad_enabled" : false + } + + aks_config_list = [] + + aks_cli_config_list = [ + { + role = "client" + aks_name = "test-no-aad-cli" + sku_tier = "Standard" + use_aks_preview_cli_extension = true + default_node_pool = { + name = "default" + node_count = 2 + vm_size = "Standard_D2s_v3" + } + optional_parameters = [] + dry_run = true + } + ] + } + + # Verify the CLI command does NOT include --enable-aad flag + assert { + condition = !can(regex("--enable-aad", module.aks-cli["client"].aks_cli_command)) + error_message = "Expected: CLI command should NOT include --enable-aad flag when disabled\nActual: ${module.aks-cli["client"].aks_cli_command}" + } + + # Verify the CLI command does NOT include any AAD-related flags + assert { + condition = !can(regex("--aad-", module.aks-cli["client"].aks_cli_command)) + error_message = "Expected: CLI command should NOT include any --aad-* flags when disabled\nActual: ${module.aks-cli["client"].aks_cli_command}" + } +} + +# Test case 6: Verify AAD parameters are omitted in aks-cli when not set (default behavior) +run "valid_aks_cli_aad_not_set" { + + command = apply + + variables { + json_input = { + "run_id" : "123456789", + "region" : "eastus", + "public_key_path" : "public_key_path" + } + + aks_config_list = [] + + aks_cli_config_list = [ + { + role = "client" + aks_name = "test-default-cli" + sku_tier = "Standard" + use_aks_preview_cli_extension = true + default_node_pool = { + name = "default" + node_count = 2 + vm_size = "Standard_D2s_v3" + } + optional_parameters = [] + dry_run = true + } + ] + } + + # Verify the CLI command does NOT include --enable-aad flag when not set + assert { + condition = !can(regex("--enable-aad", module.aks-cli["client"].aks_cli_command)) + error_message = "Expected: CLI command should NOT include --enable-aad flag when not set\nActual: ${module.aks-cli["client"].aks_cli_command}" + } + + # Verify the CLI command does NOT include any AAD-related flags when not set + assert { + condition = !can(regex("--aad-", module.aks-cli["client"].aks_cli_command)) + error_message = "Expected: CLI command should NOT include any --aad-* flags when not set\nActual: ${module.aks-cli["client"].aks_cli_command}" + } +} diff --git a/steps/topology/karpenter/validate-resources.yml b/steps/topology/karpenter/validate-resources.yml index c497a370b3..85b669aade 100644 --- a/steps/topology/karpenter/validate-resources.yml +++ b/steps/topology/karpenter/validate-resources.yml @@ -12,6 +12,12 @@ steps: role: nap region: ${{ parameters.regions[0] }} +- ${{ if eq(parameters.cloud, 'azure') }}: + - template: /steps/cloud/azure/install-kubelogin.yml + parameters: + cloud: ${{ parameters.cloud }} + region: ${{ parameters.regions[0] }} + - bash: | set -euo pipefail set -x From ba004c0165b293f6ee53eded6e4474fce2c4c407 Mon Sep 17 00:00:00 2001 From: xinWeiWei24 Date: Thu, 4 Dec 2025 16:12:42 +1100 Subject: [PATCH 3/3] Set the network policy and dataplane to Cilium in complex NAP scenarios (#953) Update the NAP complex scenario and configure the network policy and dataplane to use Cilium. --- .../perf-eval/nap/terraform-inputs/azure-complex.tfvars | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scenarios/perf-eval/nap/terraform-inputs/azure-complex.tfvars b/scenarios/perf-eval/nap/terraform-inputs/azure-complex.tfvars index 4087157a2d..07c45b2ed9 100644 --- a/scenarios/perf-eval/nap/terraform-inputs/azure-complex.tfvars +++ b/scenarios/perf-eval/nap/terraform-inputs/azure-complex.tfvars @@ -75,6 +75,14 @@ aks_cli_config_list = [ { name = "enable-image-cleaner" value = "" + }, + { + name = "network-dataplane" + value = "cilium" + }, + { + name = "network-policy" + value = "cilium" } ] }