diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index 68aa907b0d3..1f3024a1dac 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -23,6 +23,17 @@ import ( "github.com/urfave/cli/v3" ) +func isDeprecatedCSEVar(key string) bool { + switch key { + case "CLOUD_INIT_STATUS_SCRIPT", + "HYPERKUBE_URL", + "MCR_REPOSITORY_BASE", + "BLOCK_OUTBOUND_NETWORK": + return true + } + return false +} + type App struct { // cmdRun is a function that runs the given command. // the goal of this field is to make it easier to test the app by mocking the command runner. @@ -257,14 +268,7 @@ func compareEnvs(ctx context.Context, flags ProvisionFlags, eventLogger *helpers } // Extract CSE-specific env vars from provision config by filtering out unmodified OS env vars. - osEnv := envSliceToMap(os.Environ()) - pcAllEnv := envSliceToMap(provisionConfigCmd.Env) - pcEnv := make(map[string]string, len(pcAllEnv)) - for k, v := range pcAllEnv { - if osVal, inOS := osEnv[k]; !inOS || osVal != v { - pcEnv[k] = v - } - } + pcEnv := extractCSEEnvVars(provisionConfigCmd.Env) // Parse env vars directly from the NBC command file content. nbcCmdContent, err := os.ReadFile(flags.NBCCmd) @@ -274,8 +278,36 @@ func compareEnvs(ctx context.Context, flags ProvisionFlags, eventLogger *helpers } nbcEnv := parseEnvVarsFromNBCCmdContent(string(nbcCmdContent)) - // Collect all keys from both environments. - allKeys := make(map[string]struct{}) + diffs := diffEnvMaps(pcEnv, nbcEnv) + + now := time.Now() + if len(diffs) == 0 { + slog.Info("env compare: no differences found between provision-config and nbc-cmd env vars") + eventLogger.LogEvent("CompareEnvs", "env vars match between provision-config and nbc-cmd", helpers.EventLevelInformational, now, now) + } else { + message := fmt.Sprintf("env var differences (%d): %s", len(diffs), strings.Join(diffs, "; ")) + slog.Info(message) + eventLogger.LogEvent("CompareEnvs", message, helpers.EventLevelInformational, now, now) + } +} + +// extractCSEEnvVars filters a command's env slice to only CSE-specific variables +// by removing entries that match the current OS environment. +func extractCSEEnvVars(cmdEnv []string) map[string]string { + osEnv := envSliceToMap(os.Environ()) + allEnv := envSliceToMap(cmdEnv) + cseEnv := make(map[string]string, len(allEnv)) + for k, v := range allEnv { + if osVal, inOS := osEnv[k]; !inOS || osVal != v { + cseEnv[k] = v + } + } + return cseEnv +} + +// diffEnvMaps compares two environment variable maps and returns a sorted list of human-readable differences. +func diffEnvMaps(pcEnv, nbcEnv map[string]string) []string { + allKeys := make(map[string]struct{}, len(pcEnv)+len(nbcEnv)) for k := range pcEnv { allKeys[k] = struct{}{} } @@ -297,21 +329,29 @@ func compareEnvs(ctx context.Context, flags ProvisionFlags, eventLogger *helpers case inPC && !inNBC: diffs = append(diffs, fmt.Sprintf("only-in-pc: %s", key)) case !inPC && inNBC: - diffs = append(diffs, fmt.Sprintf("only-in-nbc: %s", key)) - case pcVal != nbcVal: + if !isDeprecatedCSEVar(key) { + diffs = append(diffs, fmt.Sprintf("only-in-nbc: %s", key)) + } + case !envValsEqual(pcVal, nbcVal): diffs = append(diffs, fmt.Sprintf("differs: %s", key)) } } + return diffs +} - now := time.Now() - if len(diffs) == 0 { - slog.Info("env compare: no differences found between provision-config and nbc-cmd env vars") - eventLogger.LogEvent("CompareEnvs", "env vars match between provision-config and nbc-cmd", helpers.EventLevelInformational, now, now) - } else { - message := fmt.Sprintf("env var differences (%d): %s", len(diffs), strings.Join(diffs, "; ")) - slog.Info(message) - eventLogger.LogEvent("CompareEnvs", message, helpers.EventLevelInformational, now, now) +// envValsEqual compares two environment variable values, treating them as equal +// if they differ only in the presence of double quotes around substrings. +// This handles cases like PROXY_VARS where the legacy path strips inner quotes +// due to shell quoting collision while the scriptless path preserves them. +func envValsEqual(a, b string) bool { + if a == b { + return true } + return stripDoubleQuotes(a) == stripDoubleQuotes(b) +} + +func stripDoubleQuotes(s string) string { + return strings.ReplaceAll(s, "\"", "") } // parseEnvVarsFromNBCCmdContent extracts environment variable assignments from an NBC command string. @@ -359,14 +399,14 @@ func parseEnvVarsFromNBCCmdContent(content string) map[string]string { } // parseEnvValue parses the value portion of a KEY=VALUE assignment starting at position i. -// It handles concatenated quoted and unquoted segments. Returns the parsed value and the new position. +// It handles concatenated quoted (single or double) and unquoted segments. Returns the parsed value and the new position. func parseEnvValue(content string, i int) (string, int) { n := len(content) var value strings.Builder for i < n { switch { case content[i] == '"': - // Quoted section: read until closing quote. + // Double-quoted section: read until closing double quote. i++ // skip opening quote for i < n && content[i] != '"' { value.WriteByte(content[i]) @@ -375,6 +415,16 @@ func parseEnvValue(content string, i int) (string, int) { if i < n { i++ // skip closing quote } + case content[i] == '\'': + // Single-quoted section: read until closing single quote. + i++ // skip opening quote + for i < n && content[i] != '\'' { + value.WriteByte(content[i]) + i++ + } + if i < n { + i++ // skip closing quote + } case isDelimiter(content[i]): return value.String(), i default: @@ -401,11 +451,12 @@ func isEnvKeyChar(c byte) bool { return (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' } -// skipToken advances past the current non-whitespace token, respecting double-quoted sections. +// skipToken advances past the current non-whitespace token, respecting quoted sections. func skipToken(content string, i int) int { n := len(content) for i < n && content[i] != ' ' && content[i] != '\t' && content[i] != '\n' && content[i] != ';' { - if content[i] == '"' { + switch { + case content[i] == '"': i++ for i < n && content[i] != '"' { i++ @@ -413,7 +464,15 @@ func skipToken(content string, i int) int { if i < n { i++ } - } else { + case content[i] == '\'': + i++ + for i < n && content[i] != '\'' { + i++ + } + if i < n { + i++ + } + default: i++ } } @@ -450,6 +509,7 @@ func (a *App) Provision(ctx context.Context, flags ProvisionFlags) (*ProvisionRe // If both flags are provided, compare environments before proceeding. // This is best-effort and should not block provisioning. if flags.ProvisionConfig != "" && flags.NBCCmd != "" { + slog.Info("ProvisionConfig and NBCCmd both provided, comparing envs") compareEnvs(ctx, flags, a.eventLogger) } diff --git a/aks-node-controller/app_test.go b/aks-node-controller/app_test.go index 337109da386..16e77af6a13 100644 --- a/aks-node-controller/app_test.go +++ b/aks-node-controller/app_test.go @@ -526,6 +526,12 @@ func TestParseEnvVarsFromNBCCmdContent(t *testing.T) { assert.Equal(t, "false", got["GPU_NEEDS_FABRIC_MANAGER"]) assert.Equal(t, "900", got["CSE_TIMEOUT"]) }) + + t.Run("single-quoted values", func(t *testing.T) { + content := `PROXY_VARS='export HTTPS_PROXY="https://proxy:8443"; export http_proxy="http://proxy:8080";'` + got := parseEnvVarsFromNBCCmdContent(content) + assert.Equal(t, `export HTTPS_PROXY="https://proxy:8443"; export http_proxy="http://proxy:8080";`, got["PROXY_VARS"]) + }) } // compareEnvsConfigEnv builds a CSE env map from the test provision config, diff --git a/aks-node-controller/helpers/const.go b/aks-node-controller/helpers/const.go index e51261f25d3..73485491d01 100644 --- a/aks-node-controller/helpers/const.go +++ b/aks-node-controller/helpers/const.go @@ -7,8 +7,8 @@ const ( NetworkPluginKubenet = "kubenet" NetworkPolicyAzure = "azure" NetworkPolicyCalico = "calico" - LoadBalancerBasic = "basic" - LoadBalancerStandard = "standard" + LoadBalancerBasic = "Basic" + LoadBalancerStandard = "Standard" VMSizeStandardDc2s = "Standard_DC2s" VMSizeStandardDc4s = "Standard_DC4s" DefaultLinuxUser = "azureuser" diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index 99c69c7aa4f..179ca0f1736 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -181,6 +181,7 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat return "", fmt.Errorf("AKSNodeConfig is nil") } + // TODO: add containerdv2 support // the containerd config template is different based on whether the node is with GPU or not. _template := containerdConfigTemplate if noGPU { @@ -402,7 +403,7 @@ func getSysctlContent(s *aksnodeconfigv1.SysctlConfig) string { m["vm.vfs_cache_pressure"] = s.GetVmVfsCachePressure() } - return base64.StdEncoding.EncodeToString([]byte(createSortedKeyValuePairs(m, "\n"))) + return base64.StdEncoding.EncodeToString([]byte(createSortedKeyValuePairs(m, "\n") + "\n")) } func getShouldConfigContainerdUlimits(u *aksnodeconfigv1.UlimitConfig) bool { @@ -471,24 +472,18 @@ func getPortRangeEndValue(portRange string) int { // createSortedKeyValuePairs creates a string with key=value pairs, sorted by key, with custom delimiter. func createSortedKeyValuePairs[T any](m map[string]T, delimiter string) string { - keys := []string{} + keys := make([]string, 0, len(m)) for key := range m { keys = append(keys, key) } // we are sorting the keys for deterministic output for readability and testing. sort.Strings(keys) - var buf bytes.Buffer - i := 0 + pairs := make([]string, 0, len(keys)) for _, key := range keys { - i++ - // set the last delimiter to empty string - if i == len(keys) { - delimiter = "" - } - buf.WriteString(fmt.Sprintf("%s=%v%s", key, m[key], delimiter)) + pairs = append(pairs, fmt.Sprintf("%s=%v", key, m[key])) } - return buf.String() + return strings.Join(pairs, delimiter) } func getExcludeMasterFromStandardLB(lb *aksnodeconfigv1.LoadBalancerConfig) bool { @@ -652,7 +647,7 @@ func marshalToJSON(v any) ([]byte, error) { } var rawMessage json.RawMessage = data - jsonByte, err := json.MarshalIndent(rawMessage, "", " ") + jsonByte, err := json.MarshalIndent(rawMessage, "", " ") if err != nil { log.Printf("error marshalling kubelet config file content: %v", err) return nil, err diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 2592d12c01a..0f9bdbb0b06 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -166,7 +166,8 @@ net.ipv4.neigh.default.gc_thresh1=4096 net.ipv4.neigh.default.gc_thresh2=8192 net.ipv4.neigh.default.gc_thresh3=16384 net.ipv4.tcp_max_syn_backlog=16384 -net.ipv4.tcp_retries2=8`)), +net.ipv4.tcp_retries2=8 +`)), }, { name: "SysctlConfig with custom values", @@ -187,7 +188,8 @@ net.ipv4.neigh.default.gc_thresh1=4096 net.ipv4.neigh.default.gc_thresh2=8192 net.ipv4.neigh.default.gc_thresh3=16384 net.ipv4.tcp_max_syn_backlog=9999 -net.ipv4.tcp_retries2=8`)), +net.ipv4.tcp_retries2=8 +`)), }, } for _, tt := range tests { diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index 73c9618bb7f..ec2cc09cec2 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -33,7 +33,7 @@ func executeBootstrapTemplate(inputContract *aksnodeconfigv1.Configuration) (str func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string { cloudProviderSettings := getCloudProviderSettings(config) env := map[string]string{ - "PROVISION_OUTPUT": "/var/log/azure/cluster-provision.log", + "PROVISION_OUTPUT": "/var/log/azure/cluster-provision-cse-output.log", "MOBY_VERSION": "", "CLOUDPROVIDER_BACKOFF": fmt.Sprintf("%v", cloudProviderSettings.backoff), "CLOUDPROVIDER_BACKOFF_MODE": cloudProviderSettings.backoffMode, @@ -47,7 +47,7 @@ func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string { "CLOUDPROVIDER_RATELIMIT_BUCKET": fmt.Sprintf("%v", cloudProviderSettings.rateLimitBucket), "CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE": fmt.Sprintf("%v", cloudProviderSettings.rateLimitBucketWrite), "CLI_TOOL": "ctr", - "NETWORK_MODE": "transparent", + "NETWORK_MODE": "", "ADMINUSER": getLinuxAdminUsername(config.GetLinuxAdminUsername()), "TENANT_ID": config.GetAuthConfig().GetTenantId(), "KUBERNETES_VERSION": config.GetKubernetesVersion(), @@ -194,7 +194,8 @@ func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string { "SERVICE_ACCOUNT_IMAGE_PULL_DEFAULT_TENANT_ID": config.GetServiceAccountImagePullProfile().GetDefaultTenantId(), "IDENTITY_BINDINGS_LOCAL_AUTHORITY_SNI": config.GetServiceAccountImagePullProfile().GetLocalAuthoritySni(), "CSE_TIMEOUT": getCSETimeout(config), - "SKIP_WAAGENT_HOLD": "true", + "SKIP_WAAGENT_HOLD": "false", + "NETWORK_ISOLATED_CLUSTER_TEST_MODE": "false", // temp: needs to be added to config } for i, cert := range config.CustomCaCerts { diff --git a/aks-node-controller/parser/templates/containerd.toml.gtpl b/aks-node-controller/parser/templates/containerd.toml.gtpl index db8d87d130b..6b9d7442fdb 100644 --- a/aks-node-controller/parser/templates/containerd.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd.toml.gtpl @@ -56,8 +56,8 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetEnableArtifactStreaming }} [proxy_plugins] [proxy_plugins.overlaybd] - type = "snapshot" - address = "/run/overlaybd-snapshotter/overlaybd.sock" + type = "snapshot" + address = "/run/overlaybd-snapshotter/overlaybd.sock" {{- end}} {{- if .GetIsKata }} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] diff --git a/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl index eda56f66eca..2eb27cab54c 100644 --- a/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl @@ -40,8 +40,8 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetEnableArtifactStreaming }} [proxy_plugins] [proxy_plugins.overlaybd] - type = "snapshot" - address = "/run/overlaybd-snapshotter/overlaybd.sock" + type = "snapshot" + address = "/run/overlaybd-snapshotter/overlaybd.sock" {{- end}} {{- if .GetIsKata }} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] diff --git a/aks-node-controller/parser/templates/localdns.toml.gtpl b/aks-node-controller/parser/templates/localdns.toml.gtpl index 99584dde2a1..818b23aa421 100644 --- a/aks-node-controller/parser/templates/localdns.toml.gtpl +++ b/aks-node-controller/parser/templates/localdns.toml.gtpl @@ -131,13 +131,11 @@ health-check.localdns.local:53 { template ANY ANY internal.cloudapp.net { match "^(?:[^.]+\.){4,}internal\.cloudapp\.net\.$" rcode NXDOMAIN - fallthrough - } template ANY ANY reddog.microsoft.com { rcode NXDOMAIN } {{- end}} } -{{- end}} \ No newline at end of file +{{- end}} diff --git a/aks-node-controller/pkg/nodeconfigutils/utils.go b/aks-node-controller/pkg/nodeconfigutils/utils.go index 6fe4bed34f0..c506f611616 100644 --- a/aks-node-controller/pkg/nodeconfigutils/utils.go +++ b/aks-node-controller/pkg/nodeconfigutils/utils.go @@ -2,6 +2,7 @@ package nodeconfigutils import ( "bytes" + "compress/gzip" "encoding/base64" "fmt" "mime/multipart" @@ -16,6 +17,8 @@ const ( AKSNodeConfigFilePath = "/opt/azure/containers/aks-node-controller-config.json" + NBCCmdFilePath = "/opt/azure/containers/aks-node-controller-nbc-cmd.sh" + boothookTemplate = `#cloud-boothook #!/bin/bash set -euo pipefail @@ -29,6 +32,28 @@ cat <<'EOF' | base64 -d >%[1]s EOF chmod 0600 %[1]s +logger -t aks-boothook "launching aks-node-controller service $(date -Ins)" +systemctl start --no-block aks-node-controller.service +` + + boothookPhase3Template = `#cloud-boothook +#!/bin/bash +set -euo pipefail + +logger -t aks-boothook "boothook start $(date -Ins)" + +mkdir -p /opt/azure/containers + +cat <<'EOF' | base64 -d | gzip -d >%[1]s +%[2]s +EOF +chmod 0600 %[1]s + +cat <<'EOF' | base64 -d | gzip -d >%[3]s +%[4]s +EOF +chmod 0755 %[3]s + logger -t aks-boothook "launching aks-node-controller service $(date -Ins)" systemctl start --no-block aks-node-controller.service ` @@ -82,6 +107,41 @@ func CustomData(cfg *aksnodeconfigv1.Configuration) (string, error) { return base64.StdEncoding.EncodeToString(customData.Bytes()), nil } +func CustomDataPhase3(cfg *aksnodeconfigv1.Configuration, nbcCSECMD string) (string, error) { + aksNodeConfigJSON, err := MarshalConfigurationV1(cfg) + if err != nil { + return "", fmt.Errorf("failed to marshal nbc, error: %w", err) + } + + encodedAksNodeConfigJSON, err := gzipAndBase64Encode(aksNodeConfigJSON) + if err != nil { + return "", fmt.Errorf("failed to gzip and base64 encode nbc config: %w", err) + } + encodedNBCCSECmd, err := gzipAndBase64Encode([]byte(nbcCSECMD)) + if err != nil { + return "", fmt.Errorf("failed to gzip and base64 encode nbc cse cmd: %w", err) + } + boothook := fmt.Sprintf(boothookPhase3Template, AKSNodeConfigFilePath, encodedAksNodeConfigJSON, NBCCmdFilePath, encodedNBCCSECmd) + + var customData bytes.Buffer + writer := multipart.NewWriter(&customData) + + fmt.Fprintf(&customData, "MIME-Version: 1.0\r\n") + fmt.Fprintf(&customData, "Content-Type: multipart/mixed; boundary=%q\r\n\r\n", writer.Boundary()) + + if err := writeMIMEPart(writer, "text/cloud-boothook", boothook); err != nil { + return "", fmt.Errorf("failed to write boothook part: %w", err) + } + if err := writeMIMEPart(writer, "text/cloud-config", cloudConfigTemplate); err != nil { + return "", fmt.Errorf("failed to write cloud-config part: %w", err) + } + if err := writer.Close(); err != nil { + return "", fmt.Errorf("failed to finalize multipart custom data: %w", err) + } + + return gzipAndBase64Encode(customData.Bytes()) +} + // CustomDataFlatcar builds base64-encoded custom data for Flatcar Container Linux nodes. // Unlike Ubuntu/Azure Linux which use cloud-init and expect MIME multipart custom data, // Flatcar uses Ignition (configured via Butane) to process machine configuration. Ignition @@ -118,6 +178,19 @@ func writeMIMEPart(writer *multipart.Writer, contentType, content string) error return err } +func gzipAndBase64Encode(data []byte) (string, error) { + var gzipped bytes.Buffer + gzipWriter := gzip.NewWriter(&gzipped) + if _, err := gzipWriter.Write(data); err != nil { + return "", fmt.Errorf("failed to gzip custom data: %w", err) + } + if err := gzipWriter.Close(); err != nil { + return "", fmt.Errorf("failed to finalize gzip custom data: %w", err) + } + + return base64.StdEncoding.EncodeToString(gzipped.Bytes()), nil +} + func MarshalConfigurationV1(cfg *aksnodeconfigv1.Configuration) ([]byte, error) { options := protojson.MarshalOptions{ UseEnumNumbers: false, diff --git a/aks-node-controller/pkg/nodeconfigutils/utils_test.go b/aks-node-controller/pkg/nodeconfigutils/utils_test.go index 598ccc3c439..c5751cac5e4 100644 --- a/aks-node-controller/pkg/nodeconfigutils/utils_test.go +++ b/aks-node-controller/pkg/nodeconfigutils/utils_test.go @@ -1,6 +1,8 @@ package nodeconfigutils import ( + "bytes" + "compress/gzip" "encoding/base64" "io" "mime" @@ -271,6 +273,71 @@ func TestCustomDataUsesMultipartBoothookAndCloudConfig(t *testing.T) { require.ErrorIs(t, err, io.EOF) } +func TestCustomDataPhase3UsesGzippedMultipartBoothookAndCloudConfig(t *testing.T) { + cfg := &aksnodeconfigv1.Configuration{ + Version: "v1", + AuthConfig: &aksnodeconfigv1.AuthConfig{ + SubscriptionId: "test-subscription", + }, + ClusterConfig: &aksnodeconfigv1.ClusterConfig{ + ResourceGroup: "test-rg", + Location: "eastus", + }, + ApiServerConfig: &aksnodeconfigv1.ApiServerConfig{ + ApiServerName: "test-api-server", + }, + } + + customData, err := CustomDataPhase3(cfg, "echo test") + require.NoError(t, err) + + decoded, err := base64.StdEncoding.DecodeString(customData) + require.NoError(t, err) + + gzipReader, err := gzip.NewReader(bytes.NewReader(decoded)) + require.NoError(t, err) + defer gzipReader.Close() + + uncompressed, err := io.ReadAll(gzipReader) + require.NoError(t, err) + + sections := strings.SplitN(string(uncompressed), "\r\n\r\n", 2) + require.Len(t, sections, 2) + + message := textproto.MIMEHeader{} + for _, line := range strings.Split(sections[0], "\r\n") { + if line == "" { + continue + } + parts := strings.SplitN(line, ": ", 2) + require.Len(t, parts, 2) + message.Add(parts[0], parts[1]) + } + + mediaType, params, err := mime.ParseMediaType(message.Get("Content-Type")) + require.NoError(t, err) + require.Equal(t, "multipart/mixed", mediaType) + + reader := multipart.NewReader(strings.NewReader(sections[1]), params["boundary"]) + + part, err := reader.NextPart() + require.NoError(t, err) + require.Equal(t, "text/cloud-boothook", part.Header.Get("Content-Type")) + boothook, err := io.ReadAll(part) + require.NoError(t, err) + require.Contains(t, string(boothook), AKSNodeConfigFilePath) + require.Contains(t, string(boothook), NBCCmdFilePath) + + part, err = reader.NextPart() + require.NoError(t, err) + require.Equal(t, "text/cloud-config", part.Header.Get("Content-Type")) + _, err = io.ReadAll(part) + require.NoError(t, err) + + _, err = reader.NextPart() + require.ErrorIs(t, err, io.EOF) +} + func TestMarshalUnmarshalWithPopulatedConfig(t *testing.T) { t.Run("fully populated config marshals to >100 bytes", func(t *testing.T) { cfg := &aksnodeconfigv1.Configuration{} diff --git a/e2e/node_config.go b/e2e/node_config.go index e76537b03b7..c037248f580 100644 --- a/e2e/node_config.go +++ b/e2e/node_config.go @@ -20,7 +20,7 @@ import ( // this is a base kubelet config for Scriptless e2e test func baseKubeletConfig() *aksnodeconfigv1.KubeletConfig { return &aksnodeconfigv1.KubeletConfig{ - EnableKubeletConfigFile: true, + EnableKubeletConfigFile: false, KubeletFlags: map[string]string{ "--cloud-provider": "external", "--kubeconfig": "/var/lib/kubelet/kubeconfig", @@ -58,9 +58,11 @@ func baseKubeletConfig() *aksnodeconfigv1.KubeletConfig { Webhook: &aksnodeconfigv1.KubeletWebhookAuthentication{ Enabled: true, }, + Anonymous: &aksnodeconfigv1.KubeletAnonymousAuthentication{}, }, Authorization: &aksnodeconfigv1.KubeletAuthorization{ - Mode: "Webhook", + Mode: "Webhook", + Webhook: &aksnodeconfigv1.KubeletWebhookAuthorization{}, }, EventRecordQps: to.Ptr(int32(0)), ClusterDomain: "cluster.local", @@ -81,8 +83,9 @@ func baseKubeletConfig() *aksnodeconfigv1.KubeletConfig { "nodefs.inodesFree": "5%", }, ProtectKernelDefaults: true, - FeatureGates: map[string]bool{}, - FailSwapOn: to.Ptr(false), + FeatureGates: map[string]bool{ + "RotateKubeletServerCertificate": true, + }, KubeReserved: map[string]string{ "cpu": "100m", "memory": "1638Mi", @@ -90,10 +93,6 @@ func baseKubeletConfig() *aksnodeconfigv1.KubeletConfig { EnforceNodeAllocatable: []string{ "pods", }, - AllowedUnsafeSysctls: []string{ - "kernel.msg*", - "net.ipv4.route.min_pmtu", - }, }, } } @@ -159,16 +158,59 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod bootstrappingConfig.BootstrappingAuthMethod = aksnodeconfigv1.BootstrappingAuthMethod_BOOTSTRAPPING_AUTH_METHOD_SECURE_TLS_BOOTSTRAPPING } - return &aksnodeconfigv1.Configuration{ + k8sConfig := cs.Properties.OrchestratorProfile.KubernetesConfig + + // Derive UseInstanceMetadata from NBC config. + useInstanceMeta := false + if k8sConfig.UseInstanceMetadata != nil { + useInstanceMeta = *k8sConfig.UseInstanceMetadata + } + + // Base64-encode SP secret and kubelet client key to match what baker.go renders. + spSecret := "" + if cs.Properties.ServicePrincipalProfile != nil && cs.Properties.ServicePrincipalProfile.Secret != "" { + spSecret = base64.StdEncoding.EncodeToString([]byte(cs.Properties.ServicePrincipalProfile.Secret)) + } + kubeletClientKey := "" + if cs.Properties.CertificateProfile != nil && cs.Properties.CertificateProfile.ClientPrivateKey != "" { + kubeletClientKey = base64.StdEncoding.EncodeToString([]byte(cs.Properties.CertificateProfile.ClientPrivateKey)) + } + + // Build HttpProxyConfig with all fields (not just NoProxyEntries) to match NBC. + httpProxyConfig := &aksnodeconfigv1.HttpProxyConfig{} + if nbc.HTTPProxyConfig != nil { + if nbc.HTTPProxyConfig.NoProxy != nil { + httpProxyConfig.NoProxyEntries = *nbc.HTTPProxyConfig.NoProxy + } + if nbc.HTTPProxyConfig.HTTPProxy != nil { + httpProxyConfig.HttpProxy = *nbc.HTTPProxyConfig.HTTPProxy + } + if nbc.HTTPProxyConfig.HTTPSProxy != nil { + httpProxyConfig.HttpsProxy = *nbc.HTTPProxyConfig.HTTPSProxy + } + if nbc.HTTPProxyConfig.TrustedCA != nil { + httpProxyConfig.ProxyTrustedCa = *nbc.HTTPProxyConfig.TrustedCA + } + } + + // Derive EnableUnattendedUpgrade from NBC (baker uses !DisableUnattendedUpgrades). + enableUnattendedUpgrade := !nbc.DisableUnattendedUpgrades + vnetCNIPluginURL := nbc.CloudSpecConfig.KubernetesSpecConfig.VnetCNILinuxPluginsDownloadURL + if nbc.IsARM64 { + vnetCNIPluginURL = nbc.CloudSpecConfig.KubernetesSpecConfig.VnetCNIARM64LinuxPluginsDownloadURL + } + + cfg := &aksnodeconfigv1.Configuration{ Version: "v1", BootstrappingConfig: bootstrappingConfig, DisableCustomData: true, LinuxAdminUsername: "azureuser", VmSize: config.Config.DefaultVMSKU, ClusterConfig: &aksnodeconfigv1.ClusterConfig{ - Location: nbc.ContainerService.Location, - ResourceGroup: nbc.ResourceGroupName, - VmType: aksnodeconfigv1.VmType_VM_TYPE_VMSS, + Location: nbc.ContainerService.Location, + ResourceGroup: nbc.ResourceGroupName, + VmType: aksnodeconfigv1.VmType_VM_TYPE_VMSS, + UseInstanceMetadata: useInstanceMeta, ClusterNetworkConfig: &aksnodeconfigv1.ClusterNetworkConfig{ SecurityGroupName: cs.Properties.GetNSGName(), VnetName: cs.Properties.GetVirtualNetworkName(), @@ -177,26 +219,29 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod RouteTable: cs.Properties.GetRouteTableName(), }, CloudProviderConfig: &aksnodeconfigv1.CloudProviderConfig{ - Backoff: cs.Properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff, - BackoffMode: cs.Properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffMode, - BackoffRetries: to.Ptr(int32(cs.Properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffRetries)), - BackoffExponent: to.Ptr(cs.Properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffExponent), - BackoffDuration: to.Ptr(int32(cs.Properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffDuration)), - BackoffJitter: to.Ptr(cs.Properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffJitter), - RateLimit: cs.Properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimit, - RateLimitQps: to.Ptr(cs.Properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitQPS), - RateLimitQpsWrite: to.Ptr(cs.Properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitQPSWrite), - RateLimitBucket: to.Ptr(int32(cs.Properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitBucket)), - RateLimitBucketWrite: to.Ptr(int32(cs.Properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitBucketWrite)), + Backoff: k8sConfig.CloudProviderBackoff, + BackoffMode: k8sConfig.CloudProviderBackoffMode, + BackoffRetries: to.Ptr(int32(k8sConfig.CloudProviderBackoffRetries)), + BackoffExponent: to.Ptr(k8sConfig.CloudProviderBackoffExponent), + BackoffDuration: to.Ptr(int32(k8sConfig.CloudProviderBackoffDuration)), + BackoffJitter: to.Ptr(k8sConfig.CloudProviderBackoffJitter), + RateLimit: k8sConfig.CloudProviderRateLimit, + RateLimitQps: to.Ptr(k8sConfig.CloudProviderRateLimitQPS), + RateLimitQpsWrite: to.Ptr(k8sConfig.CloudProviderRateLimitQPSWrite), + RateLimitBucket: to.Ptr(int32(k8sConfig.CloudProviderRateLimitBucket)), + RateLimitBucketWrite: to.Ptr(int32(k8sConfig.CloudProviderRateLimitBucketWrite)), }, PrimaryScaleSet: nbc.PrimaryScaleSetName, + LoadBalancerConfig: &aksnodeconfigv1.LoadBalancerConfig{ + LoadBalancerSku: aksnodeconfigv1.LoadBalancerSku_LOAD_BALANCER_SKU_STANDARD, + }, }, ApiServerConfig: &aksnodeconfigv1.ApiServerConfig{ ApiServerName: cs.Properties.HostedMasterProfile.FQDN, }, AuthConfig: &aksnodeconfigv1.AuthConfig{ ServicePrincipalId: cs.Properties.ServicePrincipalProfile.ClientID, - ServicePrincipalSecret: cs.Properties.ServicePrincipalProfile.Secret, + ServicePrincipalSecret: spSecret, TenantId: nbc.TenantID, SubscriptionId: nbc.SubscriptionID, AssignedIdentityId: nbc.UserAssignedIdentityClientID, @@ -204,13 +249,14 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod NetworkConfig: &aksnodeconfigv1.NetworkConfig{ NetworkPlugin: aksnodeconfigv1.NetworkPlugin_NETWORK_PLUGIN_KUBENET, CniPluginsUrl: nbc.CloudSpecConfig.KubernetesSpecConfig.CNIPluginsDownloadURL, - VnetCniPluginsUrl: cs.Properties.OrchestratorProfile.KubernetesConfig.AzureCNIURLLinux, + VnetCniPluginsUrl: vnetCNIPluginURL, }, GpuConfig: &aksnodeconfigv1.GpuConfig{ ConfigGpuDriver: true, GpuDevicePlugin: false, }, - EnableUnattendedUpgrade: true, + EnableUnattendedUpgrade: enableUnattendedUpgrade, + EnableArtifactStreaming: nbc.EnableArtifactStreaming, KubernetesVersion: cs.Properties.OrchestratorProfile.OrchestratorVersion, ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdDownloadUrlBase: nbc.CloudSpecConfig.KubernetesSpecConfig.ContainerdDownloadURLBase, @@ -218,17 +264,15 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod OutboundCommand: helpers.GetDefaultOutboundCommand(), KubernetesCaCert: base64.StdEncoding.EncodeToString([]byte(cs.Properties.CertificateProfile.CaCertificate)), KubeBinaryConfig: &aksnodeconfigv1.KubeBinaryConfig{ - KubeBinaryUrl: cs.Properties.OrchestratorProfile.KubernetesConfig.CustomKubeBinaryURL, + KubeBinaryUrl: k8sConfig.CustomKubeBinaryURL, PodInfraContainerImageUrl: nbc.K8sComponents.PodInfraContainerImageURL, }, - KubeProxyUrl: cs.Properties.OrchestratorProfile.KubernetesConfig.CustomKubeProxyImage, - HttpProxyConfig: &aksnodeconfigv1.HttpProxyConfig{ - NoProxyEntries: *nbc.HTTPProxyConfig.NoProxy, - }, + KubeProxyUrl: k8sConfig.CustomKubeProxyImage, + HttpProxyConfig: httpProxyConfig, LocalDnsProfile: &aksnodeconfigv1.LocalDnsProfile{ EnableLocalDns: true, CpuLimitInMilliCores: to.Ptr(int32(2008)), - MemoryLimitInMb: to.Ptr(int32(256)), + MemoryLimitInMb: to.Ptr(int32(128)), VnetDnsOverrides: map[string]*aksnodeconfigv1.LocalDnsOverrides{ ".": { QueryLogging: "Log", @@ -238,23 +282,23 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod MaxConcurrent: to.Ptr(int32(1000)), CacheDurationInSeconds: to.Ptr(int32(3600)), ServeStaleDurationInSeconds: to.Ptr(int32(3600)), - ServeStale: "Immediate", + ServeStale: "Verify", }, "cluster.local": { QueryLogging: "Error", Protocol: "ForceTCP", ForwardDestination: "ClusterCoreDNS", - ForwardPolicy: "RoundRobin", - MaxConcurrent: to.Ptr(int32(3000)), - CacheDurationInSeconds: to.Ptr(int32(7200)), - ServeStaleDurationInSeconds: to.Ptr(int32(4500)), + ForwardPolicy: "Sequential", + MaxConcurrent: to.Ptr(int32(1000)), + CacheDurationInSeconds: to.Ptr(int32(3600)), + ServeStaleDurationInSeconds: to.Ptr(int32(3600)), ServeStale: "Disable", }, "testdomain456.com": { QueryLogging: "Log", Protocol: "PreferUDP", - ForwardDestination: "VnetDNS", - ForwardPolicy: "Random", + ForwardDestination: "ClusterCoreDNS", + ForwardPolicy: "Sequential", MaxConcurrent: to.Ptr(int32(1000)), CacheDurationInSeconds: to.Ptr(int32(3600)), ServeStaleDurationInSeconds: to.Ptr(int32(3600)), @@ -305,6 +349,20 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod // Therefore, we require client (e.g. AKS-RP) to provide the final kubelet config that is ready to be written to the final kubelet config file on a node. KubeletConfig: baseKubeletConfig(), } + + // Populate KubeletConfig fields from NBC that aren't in the static baseKubeletConfig. + cfg.KubeletConfig.KubeletClientKey = kubeletClientKey + + // Build kubelet flags from the NBC's KubeletConfig map, filtering the same way baker.go does. + if nbc.KubeletConfig != nil { + kubeletFlags := make(map[string]string) + for key, val := range nbc.KubeletConfig { + kubeletFlags[key] = val + } + cfg.KubeletConfig.KubeletFlags = kubeletFlags + } + + return cfg } // this is huge, but accurate, so leave it here. @@ -386,7 +444,7 @@ func baseTemplateLinux(t testing.TB, location string, k8sVersion string, arch st AzureCNIURLLinux: "https://packages.aks.azure.com/azure-cni/v1.6.21/binaries/azure-vnet-cni-linux-amd64-v1.6.21.tgz", AzureCNIURLARM64Linux: "", AzureCNIURLWindows: "", - MaximumLoadBalancerRuleCount: 250, + MaximumLoadBalancerRuleCount: 148, PrivateAzureRegistryServer: "", NetworkPluginMode: "", }, @@ -584,7 +642,7 @@ func baseTemplateLinux(t testing.TB, location string, k8sVersion string, arch st KubeBinariesSASURLBase: "https://packages.aks.azure.com/kubernetes/", WindowsTelemetryGUID: "fb801154-36b9-41bc-89c2-f4d4f05472b0", CNIPluginsDownloadURL: "https://packages.aks.azure.com/cni/cni-plugins-amd64-v0.7.6.tgz", - VnetCNILinuxPluginsDownloadURL: "https://packages.aks.azure.com/azure-cni/v1.1.3/binaries/azure-vnet-cni-linux-amd64-v1.1.3.tgz", + VnetCNILinuxPluginsDownloadURL: "https://packages.aks.azure.com/azure-cni/v1.6.21/binaries/azure-vnet-cni-linux-amd64-v1.6.21.tgz", VnetCNIWindowsPluginsDownloadURL: "https://packages.aks.azure.com/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip", ContainerdDownloadURLBase: "https://storage.googleapis.com/cri-containerd-release/", CSIProxyDownloadURL: "https://packages.aks.azure.com/csi-proxy/v0.1.0/binaries/csi-proxy.tar.gz", diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 0dba705a302..00a46060de7 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -26,6 +26,9 @@ func Test_AzureLinux3OSGuard(t *testing.T) { BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.AgentPoolProfile.LocalDNSProfile = nil }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.LocalDnsProfile = nil + }, Validator: func(ctx context.Context, s *Scenario) { ValidateFIPSProvider(ctx, s) }, @@ -49,6 +52,9 @@ func Test_Flatcar(t *testing.T) { }, } }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.CustomCaCerts = []string{encodedTestCert} + }, Validator: func(ctx context.Context, s *Scenario) { ValidateFileHasContent(ctx, s, "/etc/protocols", "protocols definition file") ValidateFileIsRegularFile(ctx, s, "/etc/ssl/certs/ca-certificates.crt") @@ -88,6 +94,9 @@ func Test_Flatcar_ARM64(t *testing.T) { nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5" nbc.IsARM64 = true }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.VmSize = "Standard_D2pds_V5" + }, Validator: func(ctx context.Context, s *Scenario) { }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { @@ -107,6 +116,9 @@ func Test_AzureLinuxV3_ARM64(t *testing.T) { nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5" nbc.IsARM64 = true }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.VmSize = "Standard_D2pds_V5" + }, Validator: func(ctx context.Context, s *Scenario) { }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { @@ -126,6 +138,9 @@ func Test_Flatcar_AzureCNI(t *testing.T) { nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.NetworkConfig.NetworkPlugin = aksnodeconfigv1.NetworkPlugin_NETWORK_PLUGIN_AZURE + }, Validator: func(ctx context.Context, s *Scenario) { ServiceCanRestartValidator(ctx, s, "chronyd", 10) ValidateFileHasContent(ctx, s, "/etc/systemd/system/chronyd.service.d/10-chrony-restarts.conf", "Restart=always") @@ -145,6 +160,9 @@ func Test_Ubuntu2204_AzureCNI(t *testing.T) { nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.NetworkConfig.NetworkPlugin = aksnodeconfigv1.NetworkPlugin_NETWORK_PLUGIN_AZURE + }, Validator: func(ctx context.Context, s *Scenario) { }, }, @@ -205,6 +223,9 @@ func Test_ACL(t *testing.T) { }, } }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.CustomCaCerts = []string{encodedTestCert} + }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) }, @@ -232,6 +253,9 @@ func Test_ACL_ARM64(t *testing.T) { nbc.AgentPoolProfile.VMSize = "Standard_D2pds_v6" nbc.IsARM64 = true }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.VmSize = "Standard_D2pds_v6" + }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) vmss.SKU.Name = to.Ptr("Standard_D2pds_v6") @@ -324,6 +348,9 @@ func Test_ACL_AzureCNI(t *testing.T) { nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.NetworkConfig.NetworkPlugin = aksnodeconfigv1.NetworkPlugin_NETWORK_PLUGIN_AZURE + }, Validator: func(ctx context.Context, s *Scenario) { ServiceCanRestartValidator(ctx, s, "chronyd", 10) ValidateFileHasContent(ctx, s, "/etc/systemd/system/chronyd.service.d/10-chrony-restarts.conf", "Restart=always") @@ -392,6 +419,9 @@ func Test_ACL_DisableSSH(t *testing.T) { BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.SSHStatus = datamodel.SSHOff }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.EnableSsh = to.Ptr(false) + }, SkipSSHConnectivityValidation: true, // Skip SSH connectivity validation since SSH is down SkipDefaultValidation: true, // Skip default validation since it requires SSH connectivity Validator: func(ctx context.Context, s *Scenario) { @@ -509,6 +539,9 @@ func Test_AzureLinuxV3_AzureCNI(t *testing.T) { nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.NetworkConfig.NetworkPlugin = aksnodeconfigv1.NetworkPlugin_NETWORK_PLUGIN_AZURE + }, }, }) } @@ -527,6 +560,10 @@ func Test_AzureLinuxV3(t *testing.T) { }, } }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.MessageOfTheDay = "Zm9vYmFyDQo=" + config.CustomCaCerts = []string{encodedTestCert} + }, Validator: func(ctx context.Context, s *Scenario) { ValidateFileHasContent(ctx, s, "/etc/motd", "foobar") ValidateFileHasContent(ctx, s, "/etc/dnf/automatic.conf", "emit_via = stdio") @@ -760,6 +797,10 @@ func Test_Ubuntu2204(t *testing.T) { }, } }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.MessageOfTheDay = "Zm9vYmFyDQo=" + config.CustomCaCerts = []string{encodedTestCert} + }, Validator: func(ctx context.Context, s *Scenario) { ValidateInstalledPackageVersion(ctx, s, "moby-containerd", components.GetExpectedPackageVersions("containerd", "ubuntu", "r2204")[0]) ValidateInstalledPackageVersion(ctx, s, "moby-runc", components.GetExpectedPackageVersions("runc", "ubuntu", "r2204")[0]) @@ -781,6 +822,8 @@ func Test_Ubuntu2204FIPS(t *testing.T) { VHD: config.VHDUbuntu2204FIPSContainerd, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.Properties.AdditionalCapabilities = &armcompute.AdditionalCapabilities{ EnableFips1403Encryption: to.Ptr(true), @@ -830,6 +873,8 @@ func Test_Ubuntu2204Gen2FIPS(t *testing.T) { VHD: config.VHDUbuntu2204Gen2FIPSContainerd, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.Properties.AdditionalCapabilities = &armcompute.AdditionalCapabilities{ EnableFips1403Encryption: to.Ptr(true), @@ -859,6 +904,8 @@ func Test_Ubuntu2204Gen2FIPSTL(t *testing.T) { VHD: config.VHDUbuntu2204Gen2FIPSTLContainerd, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) vmss.Properties.AdditionalCapabilities = &armcompute.AdditionalCapabilities{ @@ -888,6 +935,9 @@ func Test_Ubuntu2204_EntraIDSSH(t *testing.T) { // Enable Entra ID SSH authentication nbc.SSHStatus = datamodel.EntraIDSSH }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.DisablePubkeyAuth = to.Ptr(true) + }, SkipSSHConnectivityValidation: true, // Skip SSH connectivity validation since Entra ID SSH disables private key authentication SkipDefaultValidation: true, // Skip default validation since it requires SSH connectivity Validator: func(ctx context.Context, s *Scenario) { @@ -939,6 +989,9 @@ func Test_AzureLinuxV3_DisableSSH(t *testing.T) { BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.SSHStatus = datamodel.SSHOff }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.EnableSsh = to.Ptr(false) + }, SkipSSHConnectivityValidation: true, // Skip SSH connectivity validation since SSH is down SkipDefaultValidation: true, // Skip default validation since it requires SSH connectivity Validator: func(ctx context.Context, s *Scenario) { @@ -958,6 +1011,9 @@ func Test_Ubuntu2204_DisableSSH(t *testing.T) { BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.SSHStatus = datamodel.SSHOff }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.EnableSsh = to.Ptr(false) + }, SkipSSHConnectivityValidation: true, // Skip SSH connectivity validation since SSH is down SkipDefaultValidation: true, // Skip default validation since it requires SSH connectivity Validator: func(ctx context.Context, s *Scenario) { @@ -977,6 +1033,9 @@ func Test_Flatcar_DisableSSH(t *testing.T) { BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.SSHStatus = datamodel.SSHOff }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.EnableSsh = to.Ptr(false) + }, SkipSSHConnectivityValidation: true, // Skip SSH connectivity validation since SSH is down SkipDefaultValidation: true, // Skip default validation since it requires SSH connectivity Validator: func(ctx context.Context, s *Scenario) { @@ -1280,6 +1339,9 @@ func Test_Ubuntu2204ARM64(t *testing.T) { nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5" nbc.IsARM64 = true }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.VmSize = "Standard_D2pds_V5" + }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.SKU.Name = to.Ptr("Standard_D2pds_V5") }, @@ -1296,6 +1358,9 @@ func Test_Ubuntu2204_ArtifactStreaming(t *testing.T) { BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.EnableArtifactStreaming = true }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.EnableArtifactStreaming = true + }, Validator: func(ctx context.Context, s *Scenario) { ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") @@ -1318,6 +1383,10 @@ func Test_Ubuntu2204_ArtifactStreaming_ARM64(t *testing.T) { nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5" nbc.IsARM64 = true }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.EnableArtifactStreaming = true + config.VmSize = "Standard_D2pds_V5" + }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.SKU.Name = to.Ptr("Standard_D2pds_V5") }, @@ -1364,6 +1433,9 @@ func Test_AzureLinuxV3_ArtifactStreaming(t *testing.T) { BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.EnableArtifactStreaming = true }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.EnableArtifactStreaming = true + }, Validator: func(ctx context.Context, s *Scenario) { ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index 374bb80598d..71d0a8e35cb 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -101,10 +101,27 @@ func RunScenario(t *testing.T, s *Scenario) { require.NoError(t, err) }) } + + if supportsScriptlessAKSNodeConfig(s) { + t.Run("scriptless_anc", func(t *testing.T) { + t.Parallel() + sCopy := copyScenario(s) + if sCopy.Runtime == nil { + sCopy.Runtime = &ScenarioRuntime{} + } + sCopy.Runtime.EnableScriptlessANC = true + err := runScenario(t, sCopy) + require.NoError(t, err) + }) + } } func supportsScriptlessNBCCSECmd(s *Scenario) bool { - return s.AKSNodeConfigMutator == nil && !s.IsWindows() && len(s.Config.CustomDataWriteFiles) <= 0 && !s.VHDCaching && !config.Config.TestPreProvision && !s.SkipScriptlessNBC + return !s.Tags.Scriptless && !s.IsWindows() && len(s.Config.CustomDataWriteFiles) <= 0 && !s.VHDCaching && !config.Config.TestPreProvision && !s.SkipScriptlessNBC +} + +func supportsScriptlessAKSNodeConfig(s *Scenario) bool { + return s.AKSNodeConfigMutator != nil && s.BootstrapConfigMutator != nil && !s.IsWindows() && len(s.Config.CustomDataWriteFiles) <= 0 && !s.VHDCaching && !config.Config.TestPreProvision } func runScenarioWithPreProvision(t *testing.T, original *Scenario) { @@ -275,14 +292,15 @@ func prepareAKSNode(ctx context.Context, s *Scenario) (*ScenarioVM, error) { if s.BootstrapConfigMutator != nil { s.BootstrapConfigMutator(s.Runtime.Cluster, nbc) } - if s.AKSNodeConfigMutator != nil { + if s.AKSNodeConfigMutator != nil && (s.Runtime.EnableScriptlessANC || s.Tags.Scriptless) { nodeconfig := nbcToAKSNodeConfigV1(nbc) s.AKSNodeConfigMutator(s.Runtime.Cluster, nodeconfig) s.Runtime.AKSNodeConfig = nodeconfig // AKSNodeConfig scenarios use aks-node-controller, not GetNodeBootstrapping. - // Clear NBC so validators that check NBC fields (e.g., ValidateScriptlessCSECmd) - // don't fire incorrectly — those validations only apply to NBC-based provisioning. - s.Runtime.NBC = nil + // NBC is kept for comparison mode (compareEnvs) where both configs are needed, + // but disable scriptless flags so validators don't fire incorrectly. + nbc.EnableScriptlessCSECmd = false + nbc.EnableScriptlessNBCCSECmd = false } publicKeyData := datamodel.PublicKey{KeyData: string(config.VMSSHPublicKey)} @@ -352,9 +370,6 @@ func maybeSkipScenario(ctx context.Context, t testing.TB, s *Scenario) { s.Tags.Arch = s.VHD.Arch s.Tags.ImageName = s.VHD.Name s.Tags.VHDCaching = s.VHDCaching - if s.AKSNodeConfigMutator != nil { - s.Tags.Scriptless = true - } if config.Config.TagsToRun != "" { matches, err := s.Tags.MatchesFilters(config.Config.TagsToRun) diff --git a/e2e/types.go b/e2e/types.go index 9c6a3b177ce..e3bcd31a03d 100644 --- a/e2e/types.go +++ b/e2e/types.go @@ -140,6 +140,7 @@ type ScenarioRuntime struct { VM *ScenarioVM VMSSName string EnableScriptlessNBCCSECmd bool + EnableScriptlessANC bool CSETimingReport *CSETimingReport // eagerly extracted before GA can sweep events } @@ -278,11 +279,11 @@ func (s *Scenario) KubeletConfigFileEnabled() bool { if s.Runtime == nil { return false } - if nbc := s.Runtime.NBC; nbc != nil && (nbc.EnableKubeletConfigFile || - (nbc.AgentPoolProfile != nil && (nbc.AgentPoolProfile.CustomKubeletConfig != nil || nbc.AgentPoolProfile.CustomLinuxOSConfig != nil))) { + if nodeConfig := s.Runtime.AKSNodeConfig; nodeConfig != nil && nodeConfig.KubeletConfig != nil && nodeConfig.KubeletConfig.EnableKubeletConfigFile { return true } - if nodeConfig := s.Runtime.AKSNodeConfig; nodeConfig != nil && nodeConfig.KubeletConfig != nil && nodeConfig.KubeletConfig.EnableKubeletConfigFile { + if nbc := s.Runtime.NBC; nbc != nil && (nbc.EnableKubeletConfigFile || + (nbc.AgentPoolProfile != nil && (nbc.AgentPoolProfile.CustomKubeletConfig != nil || nbc.AgentPoolProfile.CustomLinuxOSConfig != nil))) { return true } return false diff --git a/e2e/validation.go b/e2e/validation.go index 6f063fb8d86..bf8c737ae2f 100644 --- a/e2e/validation.go +++ b/e2e/validation.go @@ -49,6 +49,7 @@ func ValidateCommonLinux(ctx context.Context, s *Scenario) { ValidateWaagentLog(ctx, s) ValidateScriptlessCSECmd(ctx, s) ValidateScriptlessNBCCSECmd(ctx, s) + ValidateScriptlessPhase3(ctx, s) ValidateNodeExporter(ctx, s) ValidateSysctlConfig(ctx, s, map[string]string{ diff --git a/e2e/validators.go b/e2e/validators.go index 272cd33962f..5213f1895c3 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -2680,6 +2680,20 @@ func ValidateScriptlessNBCCSECmd(ctx context.Context, s *Scenario) { } } +// ValidateScriptlessPhase3 validates that there are not diffs between ANC generated cse cmd NBC cse cmd vars +func ValidateScriptlessPhase3(ctx context.Context, s *Scenario) { + s.T.Helper() + if s.Runtime.EnableScriptlessANC { + logFile := "/var/log/azure/aks-node-controller.log" + if !fileHasContent(ctx, s, logFile, "env compare: no differences found between provision-config and nbc-cmd env vars") { + // Grep for all env-compare diff markers to show what's different. + diffCmd := "sudo grep -E 'differs|only-in-pc|only-in-nbc|env var differences' " + logFile + " || true" + result := execScriptOnVMForScenarioValidateExitCode(ctx, s, diffCmd, 0, "could not grep for differences in aks-node-controller.log") + s.T.Fatalf("expected no env var differences between provision-config and nbc-cmd, but found differences:\n%s", result.stdout) + } + } +} + // ValidateStaleCachedKubeBinariesRemoved validates that stale versioned kube binaries (e.g. kubelet-1.29.0, kubectl-1.29.0) // have been removed from /opt/bin/ after the correct version is installed. func ValidateStaleCachedKubeBinariesRemoved(ctx context.Context, s *Scenario) { diff --git a/e2e/vmss.go b/e2e/vmss.go index 4abf14f8931..a399b999dbe 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -90,26 +90,74 @@ func ConfigureAndCreateVMSS(ctx context.Context, s *Scenario) (*ScenarioVM, erro // avoiding the race condition where runcmd or boothook scripts execute before networking is available. // Flatcar cannot use boothooks (coreos-cloudinit doesn't support MIME multipart), so it uses cloud-config // with a coreos.units block to define and start the service instead. -func CustomDataWithHack(s *Scenario, binaryURL string) (string, error) { - cloudConfigTemplate := `#cloud-boothook +func CustomDataWithHack(s *Scenario, nbcCmdScript, binaryURL string) (string, error) { + configPath := "/opt/azure/containers/aks-node-controller-config-hack.json" + nbcCmdPath := "/opt/azure/containers/aks-node-controller-nbc-cmd-hack.sh" + var err error + + // Build provision flags conditionally based on what's provided. + var flags []string + var encodedNBCCSECmd string + if s.Runtime.AKSNodeConfig != nil { + flags = append(flags, "--provision-config="+configPath) + } + if nbcCmdScript != "" { + flags = append(flags, "--nbc-cmd="+nbcCmdPath) + encodedNBCCSECmd, err = gzipAndBase64Encode([]byte(nbcCmdScript)) + if err != nil { + return "", fmt.Errorf("failed to gzip nbc cmd script: %w", err) + } + } + provisionFlags := strings.Join(flags, " ") + + // Encode AKSNodeConfig if provided. + var encodedAksNodeConfigJSON string + if s.Runtime.AKSNodeConfig != nil { + aksNodeConfigJSON, err := nodeconfigutils.MarshalConfigurationV1(s.Runtime.AKSNodeConfig) + if err != nil { + return "", fmt.Errorf("failed to marshal nbc, error: %w", err) + } + encodedAksNodeConfigJSON, err = gzipAndBase64Encode(aksNodeConfigJSON) + if err != nil { + return "", fmt.Errorf("failed to gzip aks node config: %w", err) + } + } + + var customData string + if s.VHD.Flatcar { + customData = buildFlatcarCloudConfig(encodedAksNodeConfigJSON, configPath, encodedNBCCSECmd, nbcCmdPath, binaryURL, provisionFlags) + } else { + customData = buildBoothookCloudConfig(encodedAksNodeConfigJSON, configPath, encodedNBCCSECmd, nbcCmdPath, binaryURL, provisionFlags) + } + return base64.StdEncoding.EncodeToString([]byte(customData)), nil +} + +func buildBoothookCloudConfig(encodedConfig, configPath, encodedNBCCmd, nbcCmdPath, binaryURL, provisionFlags string) string { + var sb strings.Builder + sb.WriteString(`#cloud-boothook #!/bin/bash set -euo pipefail mkdir -p /opt/azure/containers /opt/azure/bin -cat <<'EOF' | base64 -d > %[1]s -%[2]s -EOF -chmod 0600 %[1]s - +`) + if encodedConfig != "" { + fmt.Fprintf(&sb, "cat <<'EOF' | base64 -d | gzip -d > %s\n%s\nEOF\nchmod 0600 %s\n", + configPath, encodedConfig, configPath) + } + if encodedNBCCmd != "" { + fmt.Fprintf(&sb, "\ncat <<'EOF' | base64 -d | gzip -d > %s\n%s\nEOF\nchmod 0755 %s\n", + nbcCmdPath, encodedNBCCmd, nbcCmdPath) + } + fmt.Fprintf(&sb, ` cat <<'SCRIPT' > /opt/azure/bin/run-aks-node-controller-hack.sh #!/bin/bash set -euo pipefail mkdir -p /opt/azure/bin -curl -fSL --retry 10 --retry-delay 2 "%[3]s" -o /opt/azure/bin/aks-node-controller-hack +curl -fSL --retry 10 --retry-delay 2 "%s" -o /opt/azure/bin/aks-node-controller-hack chmod +x /opt/azure/bin/aks-node-controller-hack -/opt/azure/bin/aks-node-controller-hack provision --provision-config=%[1]s +/opt/azure/bin/aks-node-controller-hack provision %s SCRIPT chmod +x /opt/azure/bin/run-aks-node-controller-hack.sh @@ -130,28 +178,50 @@ UNIT systemctl daemon-reload systemctl start --no-block aks-node-controller-hack.service -` - if s.VHD.Flatcar { - // Flatcar uses coreos-cloudinit which only supports a subset of cloud-config features - // and does not handle MIME multipart or boothooks. Use coreos.units to define the service instead. - // https://github.com/flatcar/coreos-cloudinit/blob/main/Documentation/cloud-config.md#coreos-parameters - cloudConfigTemplate = `#cloud-config -write_files: -- path: %[1]s +`, binaryURL, provisionFlags) + return sb.String() +} + +func buildFlatcarCloudConfig(encodedConfig, configPath, encodedNBCCmd, nbcCmdPath, binaryURL, provisionFlags string) string { + // Flatcar uses coreos-cloudinit which only supports a subset of cloud-config features + // and does not handle MIME multipart or boothooks. Use coreos.units to define the service instead. + // https://github.com/flatcar/coreos-cloudinit/blob/main/Documentation/cloud-config.md#coreos-parameters + var sb strings.Builder + sb.WriteString("#cloud-config\nwrite_files:\n") + if encodedConfig != "" { + fmt.Fprintf(&sb, `- path: %s.gz.b64 permissions: "0600" owner: root - content: !!binary | - %[2]s -- path: /opt/azure/bin/run-aks-node-controller-hack.sh + content: | + %s +`, configPath, encodedConfig) + } + if encodedNBCCmd != "" { + fmt.Fprintf(&sb, `- path: %s.gz.b64 + permissions: "0600" + owner: root + content: | + %s +`, nbcCmdPath, encodedNBCCmd) + } + // Build a decode script that decompresses the gzipped+base64 files before running ANC. + var decodeSteps strings.Builder + if encodedConfig != "" { + fmt.Fprintf(&decodeSteps, " base64 -d %s.gz.b64 | gzip -d > %s && chmod 0600 %s\n", configPath, configPath, configPath) + } + if encodedNBCCmd != "" { + fmt.Fprintf(&decodeSteps, " base64 -d %s.gz.b64 | gzip -d > %s && chmod 0755 %s\n", nbcCmdPath, nbcCmdPath, nbcCmdPath) + } + fmt.Fprintf(&sb, `- path: /opt/azure/bin/run-aks-node-controller-hack.sh permissions: "0755" owner: root content: | #!/bin/bash set -euo pipefail mkdir -p /opt/azure/bin - curl -fSL --retry 10 --retry-delay 2 "%[3]s" -o /opt/azure/bin/aks-node-controller-hack +%s curl -fSL --retry 10 --retry-delay 2 "%s" -o /opt/azure/bin/aks-node-controller-hack chmod +x /opt/azure/bin/aks-node-controller-hack - /opt/azure/bin/aks-node-controller-hack provision --provision-config=%[1]s + /opt/azure/bin/aks-node-controller-hack provision %s # Flatcar specific configuration. It supports only a subset of cloud-init features https://github.com/flatcar/coreos-cloudinit/blob/main/Documentation/cloud-config.md#coreos-parameters coreos: units: @@ -167,18 +237,23 @@ coreos: ExecStart=/opt/azure/bin/run-aks-node-controller-hack.sh [Install] WantedBy=multi-user.target -` - } +`, decodeSteps.String(), binaryURL, provisionFlags) + return sb.String() +} - aksNodeConfigJSON, err := nodeconfigutils.MarshalConfigurationV1(s.Runtime.AKSNodeConfig) - if err != nil { - return "", fmt.Errorf("failed to marshal nbc, error: %w", err) +// gzipAndBase64Encode compresses data with gzip then base64-encodes it. +// This matches the production baker.go approach (getBase64EncodedGzippedCustomScriptFromStr) +// to keep custom data within the Azure 65535-byte limit. +func gzipAndBase64Encode(data []byte) (string, error) { + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + if _, err := w.Write(data); err != nil { + return "", fmt.Errorf("failed to gzip data: %w", err) } - encodedAksNodeConfigJSON := base64.StdEncoding.EncodeToString(aksNodeConfigJSON) - configPath := "/opt/azure/containers/aks-node-controller-config-hack.json" - - customDataYAML := fmt.Sprintf(cloudConfigTemplate, configPath, encodedAksNodeConfigJSON, binaryURL) - return base64.StdEncoding.EncodeToString([]byte(customDataYAML)), nil + if err := w.Close(); err != nil { + return "", fmt.Errorf("failed to finalize gzip data: %w", err) + } + return base64.StdEncoding.EncodeToString(buf.Bytes()), nil } // CustomDataWithNBCCmdHack is similar to baker.boothooktemplate, but it uses a hack to run new aks-node-controller binary. @@ -329,9 +404,19 @@ func createVMSSModel(ctx context.Context, s *Scenario) armcompute.VirtualMachine require.NoError(s.T, err) var cse, customData string + if s.Runtime.NBC != nil { + nodeBootstrapping, err = ab.GetNodeBootstrapping(ctx, s.Runtime.NBC) + require.NoError(s.T, err) + } + if s.Runtime.AKSNodeConfig != nil { cse = nodeconfigutils.CSE - customData = func() string { + + var nbcCSECmd string + if s.Runtime.EnableScriptlessANC { + nbcCSECmd = nodeBootstrapping.CSE + } + customData = func(nbcCSECmd string) string { if config.Config.DisableScriptLessCompilation { var data string var err error @@ -345,14 +430,12 @@ func createVMSSModel(ctx context.Context, s *Scenario) armcompute.VirtualMachine } binaryURL, err := CachedCompileAndUploadAKSNodeController(ctx, s.VHD.Arch) require.NoError(s.T, err, "failed to compile and upload aks-node-controller binary") - data, err := CustomDataWithHack(s, binaryURL) + data, err := CustomDataWithHack(s, nbcCSECmd, binaryURL) require.NoError(s.T, err, "failed to generate custom data from AKSNodeConfig with hack") return data - }() + }(nbcCSECmd) } else { - nodeBootstrapping, err = ab.GetNodeBootstrapping(ctx, s.Runtime.NBC) - require.NoError(s.T, err) cse = nodeBootstrapping.CSE customData = nodeBootstrapping.CustomData if s.Runtime.NBC.EnableScriptlessNBCCSECmd && !config.Config.DisableScriptLessCompilation && !s.Tags.NetworkIsolated && !s.Runtime.NBC.PreProvisionOnly { diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 485d1739a6a..a7d2604cfd6 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -1481,10 +1481,7 @@ func isMariner(osSku string) bool { return osSku == datamodel.OSSKUCBLMariner || osSku == datamodel.OSSKUMariner || osSku == datamodel.OSSKUAzureLinux } -const sysctlTemplateString = `# This is a partial workaround to this upstream Kubernetes issue: -# https://github.com/kubernetes/kubernetes/issues/41916#issuecomment-312428731 -net.ipv4.tcp_retries2=8 -net.core.message_burst=80 +const sysctlTemplateString = `net.core.message_burst=80 net.core.message_cost=40 {{- if .CustomLinuxOSConfig}} {{- if .CustomLinuxOSConfig.Sysctls}} @@ -1493,11 +1490,6 @@ net.core.somaxconn={{.CustomLinuxOSConfig.Sysctls.NetCoreSomaxconn}} {{- else}} net.core.somaxconn=16384 {{- end}} -{{- if .CustomLinuxOSConfig.Sysctls.NetIpv4TcpMaxSynBacklog}} -net.ipv4.tcp_max_syn_backlog={{.CustomLinuxOSConfig.Sysctls.NetIpv4TcpMaxSynBacklog}} -{{- else}} -net.ipv4.tcp_max_syn_backlog=16384 -{{- end}} {{- if .CustomLinuxOSConfig.Sysctls.NetIpv4NeighDefaultGcThresh1}} net.ipv4.neigh.default.gc_thresh1={{.CustomLinuxOSConfig.Sysctls.NetIpv4NeighDefaultGcThresh1}} {{- else}} @@ -1513,19 +1505,27 @@ net.ipv4.neigh.default.gc_thresh3={{.CustomLinuxOSConfig.Sysctls.NetIpv4NeighDef {{- else}} net.ipv4.neigh.default.gc_thresh3=16384 {{- end}} +{{- if .CustomLinuxOSConfig.Sysctls.NetIpv4TcpMaxSynBacklog}} +net.ipv4.tcp_max_syn_backlog={{.CustomLinuxOSConfig.Sysctls.NetIpv4TcpMaxSynBacklog}} {{- else}} -net.core.somaxconn=16384 net.ipv4.tcp_max_syn_backlog=16384 +{{- end}} +net.ipv4.tcp_retries2=8 +{{- else}} +net.core.somaxconn=16384 net.ipv4.neigh.default.gc_thresh1=4096 net.ipv4.neigh.default.gc_thresh2=8192 net.ipv4.neigh.default.gc_thresh3=16384 +net.ipv4.tcp_max_syn_backlog=16384 +net.ipv4.tcp_retries2=8 {{- end}} {{- else}} net.core.somaxconn=16384 -net.ipv4.tcp_max_syn_backlog=16384 net.ipv4.neigh.default.gc_thresh1=4096 net.ipv4.neigh.default.gc_thresh2=8192 net.ipv4.neigh.default.gc_thresh3=16384 +net.ipv4.tcp_max_syn_backlog=16384 +net.ipv4.tcp_retries2=8 {{- end}} {{- if .CustomLinuxOSConfig}} {{- if .CustomLinuxOSConfig.Sysctls}} @@ -1608,30 +1608,29 @@ vm.vfs_cache_pressure={{$s.VMVfsCachePressure}} {{- end}} ` -const kubenetCniTemplate = ` -{ - "cniVersion": "0.3.1", - "name": "kubenet", - "plugins": [{ - "type": "bridge", - "bridge": "cbr0", - "mtu": 1500, - "addIf": "eth0", - "isGateway": true, - "ipMasq": false, - "promiscMode": true, - "hairpinMode": false, - "ipam": { - "type": "host-local", - "ranges": [{{range $i, $range := .PodCIDRRanges}}{{if $i}}, {{end}}[{"subnet": "{{$range}}"}]{{end}}], - "routes": [{{range $i, $route := .Routes}}{{if $i}}, {{end}}{"dst": "{{$route}}"}{{end}}] - } - }, - { - "type": "portmap", - "capabilities": {"portMappings": true}, - "externalSetMarkChain": "KUBE-MARK-MASQ" - }] +const kubenetCniTemplate = `{ + "cniVersion": "0.3.1", + "name": "kubenet", + "plugins": [{ + "type": "bridge", + "bridge": "cbr0", + "mtu": 1500, + "addIf": "eth0", + "isGateway": true, + "ipMasq": false, + "promiscMode": true, + "hairpinMode": false, + "ipam": { + "type": "host-local", + "ranges": [{{range $i, $range := .PodCIDRRanges}}{{if $i}}, {{end}}[{"subnet": "{{$range}}"}]{{end}}], + "routes": [{{range $i, $route := .Routes}}{{if $i}}, {{end}}{"dst": "{{$route}}"}{{end}}] + } + }, + { + "type": "portmap", + "capabilities": {"portMappings": true}, + "externalSetMarkChain": "KUBE-MARK-MASQ" + }] } ` @@ -1717,10 +1716,10 @@ root = "{{GetDataDir}}"{{- end}} type = "snapshot" address = "/run/containerd/tardev-snapshotter.sock" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc] - pod_annotations = ["io.katacontainers.*"] snapshotter = "tardev" runtime_type = "io.containerd.kata-cc.v2" privileged_without_host_devices = true + pod_annotations = ["io.katacontainers.*"] [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc.options] ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml" {{- end}} @@ -1733,7 +1732,6 @@ root = "{{GetDataDir}}"{{- end}} snapshotter = "overlaybd" disable_snapshot_annotations = false {{- end}} - [plugins."io.containerd.cri.v1.images".pinned_images] sandbox = "{{GetPodInfraContainerSpec}}" {{- if IsKubernetesVersionGe "1.22.0"}} @@ -1742,15 +1740,14 @@ root = "{{GetDataDir}}"{{- end}} {{- end}} [plugins."io.containerd.cri.v1.images".registry.headers] X-Meta-Source-Client = ["azure/aks"] - [plugins."io.containerd.cri.v1.runtime".containerd] {{- if IsNSeriesSKU }} default_runtime_name = "nvidia-container-runtime" [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime] runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime.options] - BinaryName = "/usr/bin/nvidia-container-runtime" - SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + SystemdCgroup = true [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] runtime_type = "io.containerd.runc.v2" [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] @@ -1759,13 +1756,13 @@ root = "{{GetDataDir}}"{{- end}} default_runtime_name = "runc" [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] - BinaryName = "/usr/bin/runc" - SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] - BinaryName = "/usr/bin/runc" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/runc" {{- end}} {{- if and (IsKubenet) (not HasCalicoNetworkPolicy) }} [plugins."io.containerd.cri.v1.runtime".cni] @@ -1773,10 +1770,8 @@ root = "{{GetDataDir}}"{{- end}} conf_dir = "/etc/cni/net.d" conf_template = "/etc/containerd/kubenet_template.conf" {{- end}} - [metrics] address = "0.0.0.0:10257" - {{- if IsArtifactStreamingEnabled }} [proxy_plugins] [proxy_plugins.overlaybd] @@ -1794,10 +1789,10 @@ root = "{{GetDataDir}}"{{- end}} type = "snapshot" address = "/run/containerd/tardev-snapshotter.sock" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc] - pod_annotations = ["io.katacontainers.*"] snapshotter = "tardev" runtime_type = "io.containerd.kata-cc.v2" privileged_without_host_devices = true + pod_annotations = ["io.katacontainers.*"] [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc.options] ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml" {{- end}} @@ -1805,13 +1800,11 @@ root = "{{GetDataDir}}"{{- end}} containerdV2NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} - [plugins."io.containerd.cri.v1.images"] {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" disable_snapshot_annotations = false {{- end}} - [plugins."io.containerd.cri.v1.images".pinned_images] sandbox = "{{GetPodInfraContainerSpec}}" {{- if IsKubernetesVersionGe "1.22.0"}} @@ -1820,7 +1813,6 @@ root = "{{GetDataDir}}"{{- end}} {{- end}} [plugins."io.containerd.cri.v1.images".registry.headers] X-Meta-Source-Client = ["azure/aks"] - [plugins."io.containerd.cri.v1.runtime".containerd] default_runtime_name = "runc" [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] @@ -1838,10 +1830,8 @@ root = "{{GetDataDir}}"{{- end}} conf_dir = "/etc/cni/net.d" conf_template = "/etc/containerd/kubenet_template.conf" {{- end}} - [metrics] address = "0.0.0.0:10257" - {{- if IsArtifactStreamingEnabled }} [proxy_plugins] [proxy_plugins.overlaybd] @@ -1916,10 +1906,10 @@ root = "{{GetDataDir}}"{{- end}} type = "snapshot" address = "/run/containerd/tardev-snapshotter.sock" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc] - pod_annotations = ["io.katacontainers.*"] snapshotter = "tardev" runtime_type = "io.containerd.kata-cc.v2" privileged_without_host_devices = true + pod_annotations = ["io.katacontainers.*"] [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc.options] ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml" {{- end}} @@ -1985,8 +1975,7 @@ func GenerateLocalDNSCoreFile( // (mcr.microsoft.com, packages.aks.azure.com, etc.) are included in root domain server blocks. // When false, hosts blocks are omitted — used as a fallback when enableAKSLocalDNSHostsSetup fails at // provisioning time, following the same dual-config pattern used for containerd GPU/no-GPU configs. -const localDNSCoreFileTemplateString = ` -# *********************************************************************************** +const localDNSCoreFileTemplateString = `# *********************************************************************************** # WARNING: Changes to this file will be overwritten and not persisted. # *********************************************************************************** # whoami (used for health check of DNS) diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 20748e7acae..dc1057997ff 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -27,8 +27,7 @@ import ( - KEY="VALUE WITH WHITSPACE". */ const cseRegexString = `([^=\s]+)=(\"[^\"]*\"|[^\s]*)` -const expectedlocalDNSCorefileWithoutOverrides = ` -# *********************************************************************************** +const expectedlocalDNSCorefileWithoutOverrides = `# *********************************************************************************** # WARNING: Changes to this file will be overwritten and not persisted. # *********************************************************************************** # whoami (used for health check of DNS) @@ -403,8 +402,7 @@ var _ = Describe("Assert generated customData and cseCmd", func() { Expect(err).To(BeNil()) Expect(localDNSCoreFile).ToNot(BeEmpty()) - expectedlocalDNSCorefile := ` -# *********************************************************************************** + expectedlocalDNSCorefile := `# *********************************************************************************** # WARNING: Changes to this file will be overwritten and not persisted. # *********************************************************************************** # whoami (used for health check of DNS) @@ -593,8 +591,7 @@ testdomain456.com:53 { Expect(err).To(BeNil()) Expect(localDNSCoreFile).ToNot(BeEmpty()) - expectedlocalDNSCorefile := ` -# *********************************************************************************** + expectedlocalDNSCorefile := `# *********************************************************************************** # WARNING: Changes to this file will be overwritten and not persisted. # *********************************************************************************** # whoami (used for health check of DNS) @@ -1060,7 +1057,7 @@ var _ = Describe("getLinuxNodeCSECommand", func() { vars, err := getDecodedVarsFromCseCmd([]byte(cseCmd)) Expect(err).NotTo(HaveOccurred()) Expect(vars).To(HaveKey("KUBELET_FLAGS")) - Expect(vars["KUBELET_FLAGS"]).To(Equal("--image-gc-high-threshold=85 --max-pods=110 --pod-max-pids=-1 ")) + Expect(vars["KUBELET_FLAGS"]).To(Equal("--image-gc-high-threshold=85 --max-pods=110 --pod-max-pids=-1")) }) It("should handle different distros", func() { diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go index a1bae0c8e5e..d4e7227fac3 100644 --- a/pkg/agent/utils.go +++ b/pkg/agent/utils.go @@ -393,11 +393,11 @@ func GetOrderedKubeletConfigFlagString(config *datamodel.NodeBootstrappingConfig } } sort.Strings(keys) - var buf bytes.Buffer + pairs := make([]string, 0, len(keys)) for _, key := range keys { - buf.WriteString(fmt.Sprintf("%s=%s ", key, k[key])) + pairs = append(pairs, fmt.Sprintf("%s=%s", key, k[key])) } - return buf.String() + return strings.Join(pairs, " ") } func getOrderedKubeletConfigFlagWithCustomConfigurationString(customConfig, defaultConfig map[string]string) string { @@ -418,11 +418,11 @@ func getOrderedKubeletConfigFlagWithCustomConfigurationString(customConfig, defa } } sort.Strings(keys) - var buf bytes.Buffer + pairs := make([]string, 0, len(keys)) for _, key := range keys { - buf.WriteString(fmt.Sprintf("%s=%s ", key, config[key])) + pairs = append(pairs, fmt.Sprintf("%s=%s", key, config[key])) } - return buf.String() + return strings.Join(pairs, " ") } func getKubeletCustomConfiguration(properties *datamodel.Properties) map[string]string { diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index b8b42429ff2..94e84cb07cb 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -581,7 +581,7 @@ var _ = Describe("Test GetOrderedKubeletConfigFlagString", func() { AgentPoolProfile: &datamodel.AgentPoolProfile{}, } actucalStr := GetOrderedKubeletConfigFlagString(config) - expectStr := "--event-qps=0 --image-gc-high-threshold=85 --node-status-update-frequency=10s " + expectStr := "--event-qps=0 --image-gc-high-threshold=85 --node-status-update-frequency=10s" Expect(expectStr).To(Equal(actucalStr)) }) @@ -614,7 +614,7 @@ var _ = Describe("Test GetOrderedKubeletConfigFlagString", func() { AgentPoolProfile: &datamodel.AgentPoolProfile{}, } - expectStr := "--event-qps=0 --image-gc-high-threshold=85 --node-status-update-frequency=20s --seccomp-default=true --streaming-connection-idle-timeout=4h0m0s " + expectStr := "--event-qps=0 --image-gc-high-threshold=85 --node-status-update-frequency=20s --seccomp-default=true --streaming-connection-idle-timeout=4h0m0s" actucalStr := GetOrderedKubeletConfigFlagString(config) Expect(expectStr).To(Equal(actucalStr)) }) @@ -642,7 +642,7 @@ var _ = Describe("Test GetOrderedKubeletConfigFlagString", func() { AgentPoolProfile: &datamodel.AgentPoolProfile{}, } - expectedStr := "--node-labels=topology.kubernetes.io/region=southcentralus " + expectedStr := "--node-labels=topology.kubernetes.io/region=southcentralus" actualStr := GetOrderedKubeletConfigFlagString(config) Expect(expectedStr).To(Equal(actualStr)) }) @@ -677,7 +677,7 @@ var _ = Describe("Test GetOrderedKubeletConfigFlagString", func() { }, } - expectedStr := "--node-labels=topology.kubernetes.io/region=southcentralus --seccomp-default=true " + expectedStr := "--node-labels=topology.kubernetes.io/region=southcentralus --seccomp-default=true" actualStr := GetOrderedKubeletConfigFlagString(config) Expect(expectedStr).To(Equal(actualStr)) })