test: (scriptless) Enable scriptless phase 3 in AB e2es

aks-node-controller/app.go

@@ -23,6 +23,17 @@
 	"github.com/urfave/cli/v3"
 )
 
+func isDeprecatedCSEVar(key string) bool {
+	switch key {
+	case "CLOUD_INIT_STATUS_SCRIPT",
+		"HYPERKUBE_URL",
+		"MCR_REPOSITORY_BASE",
+		"BLOCK_OUTBOUND_NETWORK":
+		return true
+	}
+	return false
+}
+
 type App struct {
 	// cmdRun is a function that runs the given command.
 	// the goal of this field is to make it easier to test the app by mocking the command runner.
@@ -257,14 +268,7 @@
 	}
 
 	// Extract CSE-specific env vars from provision config by filtering out unmodified OS env vars.
-	osEnv := envSliceToMap(os.Environ())
-	pcAllEnv := envSliceToMap(provisionConfigCmd.Env)
-	pcEnv := make(map[string]string, len(pcAllEnv))
-	for k, v := range pcAllEnv {
-		if osVal, inOS := osEnv[k]; !inOS || osVal != v {
-			pcEnv[k] = v
-		}
-	}
+	pcEnv := extractCSEEnvVars(provisionConfigCmd.Env)
 
 	// Parse env vars directly from the NBC command file content.
 	nbcCmdContent, err := os.ReadFile(flags.NBCCmd)
@@ -274,8 +278,36 @@
 	}
 	nbcEnv := parseEnvVarsFromNBCCmdContent(string(nbcCmdContent))
 
-	// Collect all keys from both environments.
-	allKeys := make(map[string]struct{})
+	diffs := diffEnvMaps(pcEnv, nbcEnv)
+
+	now := time.Now()
+	if len(diffs) == 0 {
+		slog.Info("env compare: no differences found between provision-config and nbc-cmd env vars")
+		eventLogger.LogEvent("CompareEnvs", "env vars match between provision-config and nbc-cmd", helpers.EventLevelInformational, now, now)
+	} else {
+		message := fmt.Sprintf("env var differences (%d): %s", len(diffs), strings.Join(diffs, "; "))
+		slog.Info(message)
+		eventLogger.LogEvent("CompareEnvs", message, helpers.EventLevelInformational, now, now)
+	}
+}
+
+// extractCSEEnvVars filters a command's env slice to only CSE-specific variables
+// by removing entries that match the current OS environment.
+func extractCSEEnvVars(cmdEnv []string) map[string]string {
+	osEnv := envSliceToMap(os.Environ())
+	allEnv := envSliceToMap(cmdEnv)
+	cseEnv := make(map[string]string, len(allEnv))
+	for k, v := range allEnv {
+		if osVal, inOS := osEnv[k]; !inOS || osVal != v {
+			cseEnv[k] = v
+		}
+	}
+	return cseEnv
+}
+
+// diffEnvMaps compares two environment variable maps and returns a sorted list of human-readable differences.
+func diffEnvMaps(pcEnv, nbcEnv map[string]string) []string {
+	allKeys := make(map[string]struct{}, len(pcEnv)+len(nbcEnv))
 	for k := range pcEnv {
 		allKeys[k] = struct{}{}
 	}
@@ -297,21 +329,29 @@
 		case inPC && !inNBC:
 			diffs = append(diffs, fmt.Sprintf("only-in-pc: %s", key))
 		case !inPC && inNBC:
-			diffs = append(diffs, fmt.Sprintf("only-in-nbc: %s", key))
-		case pcVal != nbcVal:
+			if !isDeprecatedCSEVar(key) {
+				diffs = append(diffs, fmt.Sprintf("only-in-nbc: %s", key))
+			}
+		case !envValsEqual(pcVal, nbcVal):
 			diffs = append(diffs, fmt.Sprintf("differs: %s", key))
 		}
 	}
+	return diffs
+}
 
-	now := time.Now()
-	if len(diffs) == 0 {
-		slog.Info("env compare: no differences found between provision-config and nbc-cmd env vars")
-		eventLogger.LogEvent("CompareEnvs", "env vars match between provision-config and nbc-cmd", helpers.EventLevelInformational, now, now)
-	} else {
-		message := fmt.Sprintf("env var differences (%d): %s", len(diffs), strings.Join(diffs, "; "))
-		slog.Info(message)
-		eventLogger.LogEvent("CompareEnvs", message, helpers.EventLevelInformational, now, now)
+// envValsEqual compares two environment variable values, treating them as equal
+// if they differ only in the presence of double quotes around substrings.
+// This handles cases like PROXY_VARS where the legacy path strips inner quotes
+// due to shell quoting collision while the scriptless path preserves them.
+func envValsEqual(a, b string) bool {
+	if a == b {
+		return true
 	}
+	return stripDoubleQuotes(a) == stripDoubleQuotes(b)
+}
+
+func stripDoubleQuotes(s string) string {
+	return strings.ReplaceAll(s, "\"", "")
 }
 
 // parseEnvVarsFromNBCCmdContent extracts environment variable assignments from an NBC command string.
@@ -359,14 +399,14 @@
 }
 
 // parseEnvValue parses the value portion of a KEY=VALUE assignment starting at position i.
-// It handles concatenated quoted and unquoted segments. Returns the parsed value and the new position.
+// It handles concatenated quoted (single or double) and unquoted segments. Returns the parsed value and the new position.
 func parseEnvValue(content string, i int) (string, int) {
 	n := len(content)
 	var value strings.Builder
 	for i < n {
 		switch {
 		case content[i] == '"':
-			// Quoted section: read until closing quote.
+			// Double-quoted section: read until closing double quote.
 			i++ // skip opening quote
 			for i < n && content[i] != '"' {
 				value.WriteByte(content[i])
@@ -375,6 +415,16 @@
 			if i < n {
 				i++ // skip closing quote
 			}
+		case content[i] == '\'':
+			// Single-quoted section: read until closing single quote.
+			i++ // skip opening quote
+			for i < n && content[i] != '\'' {
+				value.WriteByte(content[i])
+				i++
+			}
+			if i < n {
+				i++ // skip closing quote
+			}
 		case isDelimiter(content[i]):
 			return value.String(), i
 		default:
@@ -401,19 +451,28 @@
 	return (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'
 }
 
-// skipToken advances past the current non-whitespace token, respecting double-quoted sections.
+// skipToken advances past the current non-whitespace token, respecting quoted sections.
 func skipToken(content string, i int) int {
 	n := len(content)
 	for i < n && content[i] != ' ' && content[i] != '\t' && content[i] != '\n' && content[i] != ';' {
-		if content[i] == '"' {
+		switch {
+		case content[i] == '"':
 			i++
 			for i < n && content[i] != '"' {
 				i++
 			}
 			if i < n {
 				i++
 			}
-		} else {
+		case content[i] == '\'':
+			i++
+			for i < n && content[i] != '\'' {
+				i++
+			}
+			if i < n {
+				i++
+			}
+		default:
 			i++
 		}
 	}
@@ -450,6 +509,7 @@
 	// If both flags are provided, compare environments before proceeding.
 	// This is best-effort and should not block provisioning.
 	if flags.ProvisionConfig != "" && flags.NBCCmd != "" {
+		slog.Info("ProvisionConfig and NBCCmd both provided, comparing envs")
 		compareEnvs(ctx, flags, a.eventLogger)
 	}
 

aks-node-controller/app_test.go

@@ -526,6 +526,12 @@ func TestParseEnvVarsFromNBCCmdContent(t *testing.T) {
 		assert.Equal(t, "false", got["GPU_NEEDS_FABRIC_MANAGER"])
 		assert.Equal(t, "900", got["CSE_TIMEOUT"])
 	})
+
+	t.Run("single-quoted values", func(t *testing.T) {
+		content := `PROXY_VARS='export HTTPS_PROXY="https://proxy:8443"; export http_proxy="http://proxy:8080";'`
+		got := parseEnvVarsFromNBCCmdContent(content)
+		assert.Equal(t, `export HTTPS_PROXY="https://proxy:8443"; export http_proxy="http://proxy:8080";`, got["PROXY_VARS"])
+	})
 }
 
 // compareEnvsConfigEnv builds a CSE env map from the test provision config,

aks-node-controller/helpers/const.go

@@ -7,8 +7,8 @@ const (
 	NetworkPluginKubenet = "kubenet"
 	NetworkPolicyAzure   = "azure"
 	NetworkPolicyCalico  = "calico"
-	LoadBalancerBasic    = "basic"
-	LoadBalancerStandard = "standard"
+	LoadBalancerBasic    = "Basic"
+	LoadBalancerStandard = "Standard"
 	VMSizeStandardDc2s   = "Standard_DC2s"
 	VMSizeStandardDc4s   = "Standard_DC4s"
 	DefaultLinuxUser     = "azureuser"

aks-node-controller/parser/helper.go

@@ -181,6 +181,7 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat
 		return "", fmt.Errorf("AKSNodeConfig is nil")
 	}
 
+	// TODO: add containerdv2 support
 	// the containerd config template is different based on whether the node is with GPU or not.
 	_template := containerdConfigTemplate
 	if noGPU {
@@ -402,7 +403,7 @@ func getSysctlContent(s *aksnodeconfigv1.SysctlConfig) string {
 		m["vm.vfs_cache_pressure"] = s.GetVmVfsCachePressure()
 	}
 
-	return base64.StdEncoding.EncodeToString([]byte(createSortedKeyValuePairs(m, "\n")))
+	return base64.StdEncoding.EncodeToString([]byte(createSortedKeyValuePairs(m, "\n") + "\n"))
 }
 
 func getShouldConfigContainerdUlimits(u *aksnodeconfigv1.UlimitConfig) bool {
@@ -471,24 +472,18 @@ func getPortRangeEndValue(portRange string) int {
 
 // createSortedKeyValuePairs creates a string with key=value pairs, sorted by key, with custom delimiter.
 func createSortedKeyValuePairs[T any](m map[string]T, delimiter string) string {
-	keys := []string{}
+	keys := make([]string, 0, len(m))
 	for key := range m {
 		keys = append(keys, key)
 	}
 
 	// we are sorting the keys for deterministic output for readability and testing.
 	sort.Strings(keys)
-	var buf bytes.Buffer
-	i := 0
+	pairs := make([]string, 0, len(keys))
 	for _, key := range keys {
-		i++
-		// set the last delimiter to empty string
-		if i == len(keys) {
-			delimiter = ""
-		}
-		buf.WriteString(fmt.Sprintf("%s=%v%s", key, m[key], delimiter))
+		pairs = append(pairs, fmt.Sprintf("%s=%v", key, m[key]))
 	}
-	return buf.String()
+	return strings.Join(pairs, delimiter)
 }
 
 func getExcludeMasterFromStandardLB(lb *aksnodeconfigv1.LoadBalancerConfig) bool {
@@ -652,7 +647,7 @@ func marshalToJSON(v any) ([]byte, error) {
 		}
 
 		var rawMessage json.RawMessage = data
-		jsonByte, err := json.MarshalIndent(rawMessage, "", "  ")
+		jsonByte, err := json.MarshalIndent(rawMessage, "", "    ")
 		if err != nil {
 			log.Printf("error marshalling kubelet config file content: %v", err)
 			return nil, err

aks-node-controller/parser/helper_test.go

@@ -166,7 +166,8 @@ net.ipv4.neigh.default.gc_thresh1=4096
 net.ipv4.neigh.default.gc_thresh2=8192
 net.ipv4.neigh.default.gc_thresh3=16384
 net.ipv4.tcp_max_syn_backlog=16384
-net.ipv4.tcp_retries2=8`)),
+net.ipv4.tcp_retries2=8
+`)),
 		},
 		{
 			name: "SysctlConfig with custom values",
@@ -187,7 +188,8 @@ net.ipv4.neigh.default.gc_thresh1=4096
 net.ipv4.neigh.default.gc_thresh2=8192
 net.ipv4.neigh.default.gc_thresh3=16384
 net.ipv4.tcp_max_syn_backlog=9999
-net.ipv4.tcp_retries2=8`)),
+net.ipv4.tcp_retries2=8
+`)),
 		},
 	}
 	for _, tt := range tests {

aks-node-controller/parser/parser.go

@@ -33,7 +33,7 @@ func executeBootstrapTemplate(inputContract *aksnodeconfigv1.Configuration) (str
 func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string {
 	cloudProviderSettings := getCloudProviderSettings(config)
 	env := map[string]string{
-		"PROVISION_OUTPUT":                                     "/var/log/azure/cluster-provision.log",
+		"PROVISION_OUTPUT":                                     "/var/log/azure/cluster-provision-cse-output.log",
 		"MOBY_VERSION":                                         "",
 		"CLOUDPROVIDER_BACKOFF":                                fmt.Sprintf("%v", cloudProviderSettings.backoff),
 		"CLOUDPROVIDER_BACKOFF_MODE":                           cloudProviderSettings.backoffMode,
@@ -47,7 +47,7 @@ func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string {
 		"CLOUDPROVIDER_RATELIMIT_BUCKET":                       fmt.Sprintf("%v", cloudProviderSettings.rateLimitBucket),
 		"CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE":                 fmt.Sprintf("%v", cloudProviderSettings.rateLimitBucketWrite),
 		"CLI_TOOL":                                             "ctr",
-		"NETWORK_MODE":                                         "transparent",
+		"NETWORK_MODE":                                         "",
 		"ADMINUSER":                                            getLinuxAdminUsername(config.GetLinuxAdminUsername()),
 		"TENANT_ID":                                            config.GetAuthConfig().GetTenantId(),
 		"KUBERNETES_VERSION":                                   config.GetKubernetesVersion(),
@@ -194,7 +194,8 @@ func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string {
 		"SERVICE_ACCOUNT_IMAGE_PULL_DEFAULT_TENANT_ID": config.GetServiceAccountImagePullProfile().GetDefaultTenantId(),
 		"IDENTITY_BINDINGS_LOCAL_AUTHORITY_SNI":        config.GetServiceAccountImagePullProfile().GetLocalAuthoritySni(),
 		"CSE_TIMEOUT":                                  getCSETimeout(config),
-		"SKIP_WAAGENT_HOLD":                            "true",
+		"SKIP_WAAGENT_HOLD":                            "false",
+		"NETWORK_ISOLATED_CLUSTER_TEST_MODE":           "false", // temp: needs to be added to config
 	}
 
 	for i, cert := range config.CustomCaCerts {

aks-node-controller/parser/templates/containerd.toml.gtpl

@@ -56,8 +56,8 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
 {{- if .GetEnableArtifactStreaming }}
 [proxy_plugins]
   [proxy_plugins.overlaybd]
-	type = "snapshot"
-	address = "/run/overlaybd-snapshotter/overlaybd.sock"
+    type = "snapshot"
+    address = "/run/overlaybd-snapshotter/overlaybd.sock"
 {{- end}}
 {{- if .GetIsKata }}
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]

aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl

@@ -40,8 +40,8 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
 {{- if .GetEnableArtifactStreaming }}
 [proxy_plugins]
   [proxy_plugins.overlaybd]
-	type = "snapshot"
-	address = "/run/overlaybd-snapshotter/overlaybd.sock"
+    type = "snapshot"
+    address = "/run/overlaybd-snapshotter/overlaybd.sock"
 {{- end}}
 {{- if .GetIsKata }}
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]

aks-node-controller/parser/templates/localdns.toml.gtpl

@@ -131,13 +131,11 @@ health-check.localdns.local:53 {
     template ANY ANY internal.cloudapp.net {
         match "^(?:[^.]+\.){4,}internal\.cloudapp\.net\.$"
         rcode NXDOMAIN
-
         fallthrough
-
     }
     template ANY ANY reddog.microsoft.com {
         rcode NXDOMAIN
     }
     {{- end}}
 }
-{{- end}}
+{{- end}}