test: add FIPS provider validation to FIPS scenario tests

e2e/config/vhd.go

@@ -127,6 +127,16 @@ var (
 		// Secure TLS Bootstrapping isn't currently supported on FIPS-enabled VHDs
 		UnsupportedSecureTLSBootstrapping: true,
 	}
+	VHDAzureLinuxV3Gen2FIPS = &Image{
+		Name:                "AzureLinuxV3gen2fips",
+		OS:                  OSAzureLinux,
+		Arch:                "amd64",
+		Distro:              datamodel.AKSAzureLinuxV3Gen2FIPS,
+		Gallery:             imageGalleryLinux,
+		UnsupportedLocalDns: true,
+		// Secure TLS Bootstrapping isn't currently supported on FIPS-enabled VHDs
+		UnsupportedSecureTLSBootstrapping: true,
+	}
 	// this is a particular 2204gen2containerd image originally built with private packages,
 	// if we ever want to update this then we'd need to run a new VHD build using private package overrides
 	VHDUbuntu2204Gen2ContainerdPrivateKubePkg = &Image{

e2e/scenario_test.go

@@ -26,7 +26,9 @@ func Test_AzureLinux3OSGuard(t *testing.T) {
 			BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
 				nbc.AgentPoolProfile.LocalDNSProfile = nil
 			},
-			Validator: func(ctx context.Context, s *Scenario) {},
+			Validator: func(ctx context.Context, s *Scenario) {
+				ValidateFIPSProvider(ctx, s)
+			},
 			VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
 				vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties)
 			},
@@ -260,6 +262,29 @@ func Test_ACLGen2FIPSTL(t *testing.T) {
 				ValidateFileHasContent(ctx, s, "/etc/os-release", "ID=azurelinux")
 				ValidateFileHasContent(ctx, s, "/etc/os-release", "VARIANT_ID=azurecontainerlinux")
 				ValidateACLFIPSEnabled(ctx, s)
+				ValidateFIPSProvider(ctx, s)
+			},
+		},
+	})
+}
+
+func Test_AzureLinuxV3Gen2FIPS(t *testing.T) {
+	RunScenario(t, &Scenario{
+		Description: "Tests that a node using the Azure Linux V3 Gen2 FIPS VHD can be properly bootstrapped and FIPS is active at runtime",
+		Config: Config{
+			Cluster: ClusterKubenet,
+			VHD:     config.VHDAzureLinuxV3Gen2FIPS,
+			BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
+				// LocalDNS isn't currently supported on FIPS-enabled VHDs.
+				nbc.AgentPoolProfile.LocalDNSProfile = nil
+			},
+			VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
+				vmss.Properties.AdditionalCapabilities = &armcompute.AdditionalCapabilities{
+					EnableFips1403Encryption: to.Ptr(true),
+				}
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+				ValidateFIPSProvider(ctx, s)
 			},
 		},
 	})
@@ -768,6 +793,7 @@ func Test_Ubuntu2204FIPS(t *testing.T) {
 				ValidateInstalledPackageVersion(ctx, s, "moby-containerd", components.GetExpectedPackageVersions("containerd", "ubuntu", "r2204")[0])
 				ValidateInstalledPackageVersion(ctx, s, "moby-runc", components.GetExpectedPackageVersions("runc", "ubuntu", "r2204")[0])
 				ValidateSSHServiceEnabled(ctx, s)
+				ValidateFIPSProvider(ctx, s)
 			},
 		},
 	})
@@ -787,6 +813,7 @@ func Test_Ubuntu2004FIPS(t *testing.T) {
 				ValidateInstalledPackageVersion(ctx, s, "moby-containerd", components.GetExpectedPackageVersions("containerd", "ubuntu", "r2004")[0])
 				ValidateInstalledPackageVersion(ctx, s, "moby-runc", components.GetExpectedPackageVersions("runc", "ubuntu", "r2004")[0])
 				ValidateSSHServiceEnabled(ctx, s)
+				ValidateFIPSProvider(ctx, s)
 			},
 		},
 	})
@@ -815,6 +842,7 @@ func Test_Ubuntu2204Gen2FIPS(t *testing.T) {
 				ValidateInstalledPackageVersion(ctx, s, "moby-containerd", components.GetExpectedPackageVersions("containerd", "ubuntu", "r2204")[0])
 				ValidateInstalledPackageVersion(ctx, s, "moby-runc", components.GetExpectedPackageVersions("runc", "ubuntu", "r2204")[0])
 				ValidateSSHServiceEnabled(ctx, s)
+				ValidateFIPSProvider(ctx, s)
 			},
 		},
 	})
@@ -844,6 +872,7 @@ func Test_Ubuntu2204Gen2FIPSTL(t *testing.T) {
 				ValidateInstalledPackageVersion(ctx, s, "moby-containerd", components.GetExpectedPackageVersions("containerd", "ubuntu", "r2204")[0])
 				ValidateInstalledPackageVersion(ctx, s, "moby-runc", components.GetExpectedPackageVersions("runc", "ubuntu", "r2204")[0])
 				ValidateSSHServiceEnabled(ctx, s)
+				ValidateFIPSProvider(ctx, s)
 			},
 		},
 	})
@@ -2698,7 +2727,9 @@ func Test_AzureLinux3OSGuard_PMC_Install(t *testing.T) {
 			BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
 				nbc.AgentPoolProfile.LocalDNSProfile = nil
 			},
-			Validator: func(ctx context.Context, s *Scenario) {},
+			Validator: func(ctx context.Context, s *Scenario) {
+				ValidateFIPSProvider(ctx, s)
+			},
 			VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
 				vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties)
 				if vmss.Tags == nil {

e2e/validators.go

@@ -486,16 +486,13 @@ func ValidateFileExists(ctx context.Context, s *Scenario, fileName string) {
 	}
 }
 
+// ValidateACLFIPSEnabled asserts ACL-specific FIPS markers are present on the node:
+// the /etc/system-fips marker file written by vhdbuilder/scripts/linux/acl/tool_installs_acl.sh.
+// Kernel FIPS mode (/proc/sys/crypto/fips_enabled == 1) is universal and is asserted by
+// ValidateFIPSProvider; callers should compose the two validators when both are needed.
 func ValidateACLFIPSEnabled(ctx context.Context, s *Scenario) {
 	s.T.Helper()
 	ValidateFileExists(ctx, s, "/etc/system-fips")
-	execScriptOnVMForScenarioValidateExitCode(
-		ctx,
-		s,
-		`test "$(cat /proc/sys/crypto/fips_enabled)" = "1"`,
-		0,
-		"expected /proc/sys/crypto/fips_enabled to be 1",
-	)
 }
 
 func ValidateFileDoesNotExist(ctx context.Context, s *Scenario, fileName string) {
@@ -653,6 +650,112 @@ func ValidateFileExcludesExactContent(ctx context.Context, s *Scenario, fileName
 	}
 }
 
+// ValidateFIPSProvider verifies that FIPS is properly configured on the node:
+//  1. Kernel FIPS mode is enabled (/proc/sys/crypto/fips_enabled == 1).
+//  2. OpenSSL (3.x) has an active FIPS or SymCrypt provider loaded. The check is
+//     skipped on hosts shipping OpenSSL 1.1.x (e.g. Ubuntu 20.04 FIPS), which use
+//     the legacy FIPS module rather than the providers interface.
+//  3. /opt/cni/bin/portmap runs without panicking (regression guard for ICM 51000001009688
+//     where the OpenSSL FIPS provider was not loaded on AzureLinux V3 FIPS nodes).
+func ValidateFIPSProvider(ctx context.Context, s *Scenario) {
+	s.T.Helper()
+
+	// 1. Kernel FIPS mode.
+	fipsEnabled := execScriptOnVMForScenarioValidateExitCode(ctx, s, "cat /proc/sys/crypto/fips_enabled", 0, "could not read /proc/sys/crypto/fips_enabled")
+	require.Equal(s.T, "1", strings.TrimSpace(fipsEnabled.stdout), "expected /proc/sys/crypto/fips_enabled to be 1, got %q", fipsEnabled.stdout)
+
+	// 2. OpenSSL provider must include an active fips or symcrypt provider on OpenSSL 3.x.
+	// 1.1.x (Ubuntu 20.04 FIPS) uses the legacy FIPS module and is skipped. Merge stderr
+	// (`2>&1`) so a version banner written to stderr still parses.
+	opensslVersion := execScriptOnVMForScenarioValidateExitCode(ctx, s, "openssl version 2>&1", 0, "could not run openssl version")
+	versionFields := strings.Fields(opensslVersion.stdout)
+	require.GreaterOrEqual(s.T, len(versionFields), 2,
+		"could not parse openssl version output: %q", opensslVersion.stdout)
+	version := versionFields[1]
+	switch {
+	case strings.HasPrefix(version, "3."):
+		providers := execScriptOnVMForScenarioValidateExitCode(ctx, s, "openssl list -providers", 0, "could not list openssl providers")
+		// Prefix match so "symcrypt" covers AzureLinux V3 / ACL's "symcryptprovider". See ICM 51000001009688.
+		require.True(s.T, opensslProviderActive(providers.stdout, "fips", "symcrypt"),
+			"expected openssl to have an active fips or symcrypt provider, got:\n%s", providers.stdout)
+	case strings.HasPrefix(version, "1.1."):
+		s.T.Logf("openssl providers check skipped: detected version %q (legacy FIPS module)", strings.TrimSpace(opensslVersion.stdout))
+	default:
+		s.T.Fatalf("unexpected openssl version %q: FIPS VHDs are expected to ship OpenSSL 3.x or 1.1.x", strings.TrimSpace(opensslVersion.stdout))
+	}
+
+	// 3. portmap panic check (best-effort). The original FIPS regression manifested as
+	// /opt/cni/bin/portmap panicking when the OpenSSL FIPS provider was missing. Skip if
+	// the binary isn't at the standard CNI path (e.g. ACL stages it under /opt/cni/downloads/);
+	// checks 1 and 2 are already authoritative. Match specific Go runtime panic markers
+	// rather than the bare substring `runtime error:` which appears in CNI usage text.
+	portmapBin := "/opt/cni/bin/portmap"
+	portmapPresent := execScriptOnVMForScenario(ctx, s, fmt.Sprintf("test -x %s", portmapBin))
+	if portmapPresent.exitCode != "0" {
+		s.T.Logf("portmap panic check skipped: %s not present or not executable on this VHD", portmapBin)
+		s.T.Logf("FIPS provider validation passed")
+		return
+	}
+	portmap := execScriptOnVMForScenario(ctx, s, fmt.Sprintf("%s < /dev/null", portmapBin))
+	panicMarkers := []*regexp.Regexp{
+		regexp.MustCompile(`(?m)^panic:`),
+		regexp.MustCompile(`(?m)^fatal error:`),
+		regexp.MustCompile(`runtime\.gopanic`),
+		regexp.MustCompile(`goroutine \d+ \[running\]`),
+	}
+	for _, re := range panicMarkers {
+		require.False(s.T, re.MatchString(portmap.stderr),
+			"portmap runtime failure matched %q, indicating FIPS provider misconfiguration:\nstdout:\n%s\nstderr:\n%s", re, portmap.stdout, portmap.stderr)
+		require.False(s.T, re.MatchString(portmap.stdout),
+			"portmap runtime failure matched %q, indicating FIPS provider misconfiguration:\nstdout:\n%s\nstderr:\n%s", re, portmap.stdout, portmap.stderr)
+	}
+
+	s.T.Logf("FIPS provider validation passed")
+}
+
+// Package-level regex compiled once at init.
+var (
+	// Provider header: indented (spaces or tabs) name with no key/value separator.
+	// Group 1 = indent (used to scope status lines to their block), group 2 = name.
+	opensslProviderHeaderRE = regexp.MustCompile(`^([ \t]+)(\S+)\s*$`)
+	opensslStatusLineRE     = regexp.MustCompile(`^[ \t]+status:\s*(\S+)`)
+)
+
+// opensslProviderActive parses `openssl list -providers` output and returns true if any
+// provider whose name has one of the given prefixes is reported as `status: active`.
+// The status line is scoped to its enclosing provider block (strictly more indented than
+// the header) so an active default provider cannot mask an inactive fips/symcrypt one.
+func opensslProviderActive(output string, providerPrefixes ...string) bool {
+	matches := func(name string) bool {
+		for _, p := range providerPrefixes {
+			if strings.HasPrefix(name, p) {
+				return true
+			}
+		}
+		return false
+	}
+	var current string
+	var headerIndent int
+	for _, line := range strings.Split(output, "\n") {
+		line = strings.TrimRight(line, "\r")
+		if m := opensslProviderHeaderRE.FindStringSubmatch(line); m != nil {
+			headerIndent = len(m[1])
+			current = m[2]
+			continue
+		}
+		if current == "" || !matches(current) {
+			continue
+		}
+		if m := opensslStatusLineRE.FindStringSubmatch(line); m != nil && m[1] == "active" {
+			leading := len(line) - len(strings.TrimLeft(line, " \t"))
+			if leading > headerIndent {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func ServiceCanRestartValidator(ctx context.Context, s *Scenario, serviceName string, restartTimeoutInSeconds int) {
 	s.T.Helper()
 	steps := []string{

vhdbuilder/packer/test/linux-vhd-content-test.sh

@@ -591,44 +591,120 @@ testChrony() {
 testFips() {
   local test="testFips"
   echo "$test:Start"
+  # Sourcing cse_helpers.sh overwrites the global OS_VERSION from /etc/os-release
+  # VERSION_ID (e.g. "2.0", "3.0", "3.0.20260506" on ACL, "20.04"), so the value
+  # passed in as $1 is the /etc/os-release one, not the pipeline-style OS_VERSION.
   os_version=$1
   enable_fips=$2
 
   # shellcheck disable=SC3010
-  if [[ (${os_version} == "20.04" || ${os_version} == "22.04" || ${os_version} == "V2" || ${os_version} == "acl") && ${enable_fips,,} == "true" ]]; then
+  if [[ ${enable_fips,,} != "true" ]]; then
+    echo "$test:Finish"
+    return
+  fi
+
+  # Known FIPS-capable VERSION_ID values. New FIPS-enabled distros MUST be added
+  # here; otherwise the test fails loudly instead of silently no-op'ing (ICM
+  # 51000001009688). ACL VERSION_ID carries a date suffix (e.g. "3.0.20260506"),
+  # hence the "3.0.*" entry. `err` writes to stderr only, so we `return` after.
+  case "${os_version}" in
+    20.04|22.04|24.04|2.0|2.0.*|3.0|3.0.*) ;;
+    *)
+      err $test "testFips invoked with enable_fips=true on unrecognized os_version '${os_version}'; add it to the allowlist."
+      echo "$test:Finish"
+      return
+      ;;
+  esac
+
+  if [ -f /proc/sys/crypto/fips_enabled ]; then
+    fips_enabled=$(cat /proc/sys/crypto/fips_enabled)
+    if [ "${fips_enabled}" = "1" ]; then
+      echo "FIPS is enabled."
+    else
+      err $test "content of /proc/sys/crypto/fips_enabled is not 1."
+    fi
+  else
+    err $test "FIPS is not enabled."
+  fi
+
+  if [ "${os_version}" = "20.04" ]; then
     kernel=$(uname -r)
-    if [ -f /proc/sys/crypto/fips_enabled ]; then
-      fips_enabled=$(cat /proc/sys/crypto/fips_enabled)
-      if [ "${fips_enabled}" = "1" ]; then
-        echo "FIPS is enabled."
-      else
-        err $test "content of /proc/sys/crypto/fips_enabled is not 1."
-      fi
+    if [ -f /usr/src/linux-headers-${kernel}/Makefile ]; then
+      echo "fips header files exist."
     else
-      err $test "FIPS is not enabled."
+      err $test "fips header files don't exist."
     fi
+  fi
 
-    if [ ${os_version} = "20.04" ]; then
-      if [ -f /usr/src/linux-headers-${kernel}/Makefile ]; then
-        echo "fips header files exist."
-      else
-        err $test "fips header files don't exist."
-      fi
+  if [ "${OS_SKU}" = "AzureContainerLinux" ]; then
+    if [ -f /etc/system-fips ]; then
+      echo "/etc/system-fips marker file exists."
+    else
+      err $test "/etc/system-fips marker file does not exist."
+    fi
+    if [ -f /boot/EFI/Linux/acl.efi.extra.d/fips.addon.efi ]; then
+      echo "ACL FIPS UKI addon file exists in active ESP location."
+    else
+      err $test "ACL FIPS UKI addon file does not exist in active ESP location."
     fi
+  fi
 
-    if [ ${os_version} = "acl" ]; then
-      if [ -f /etc/system-fips ]; then
-        echo "/etc/system-fips marker file exists."
-      else
-        err $test "/etc/system-fips marker file does not exist."
-      fi
-      if [ -f /boot/EFI/Linux/acl.efi.extra.d/fips.addon.efi ]; then
-        echo "ACL FIPS UKI addon file exists in active ESP location."
+  # OpenSSL must have an active FIPS or SymCrypt provider on 3.x (ICM 51000001009688
+  # was caused by kernel FIPS on with no provider, causing portmap to panic). Ubuntu
+  # 20.04 ships 1.1.x and uses the legacy FIPS module — skip there. Keep in sync with
+  # the Go validator in e2e/validators.go.
+  if ! command -v openssl >/dev/null 2>&1; then
+    err $test "openssl binary not found on a FIPS-enabled VHD."
+    echo "$test:Finish"
+    return
+  fi
+  openssl_version_raw=$(openssl version 2>&1)
+  openssl_version=$(echo "${openssl_version_raw}" | awk '{print $2}')
+  case "${openssl_version}" in
+    "")
+      err $test "could not parse openssl version (raw output: '${openssl_version_raw}')."
+      ;;
+    3.*)
+      providers_output=$(openssl list -providers 2>&1)
+      echo "openssl list -providers output:"
+      echo "${providers_output}"
+      # Walk each provider block, scoping `status: active` to its enclosing header.
+      # Prefix match so "symcrypt" covers AzureLinux V3 / ACL's "symcryptprovider".
+      # Use `[ \t]` (not `[[:space:]]`) so a trailing CR can't break header detection;
+      # also strip \r explicitly.
+      fips_active=""
+      current_provider=""
+      header_indent=0
+      while IFS= read -r line; do
+        line="${line%$'\r'}"
+        # shellcheck disable=SC3010
+        if [[ "${line}" =~ ^([\ $'\t']+)([^[:space:]:]+)[\ $'\t']*$ ]]; then
+          header_indent=${#BASH_REMATCH[1]}
+          current_provider="${BASH_REMATCH[2]}"
+          continue
+        fi
+        # shellcheck disable=SC3010
+        if [[ "${current_provider}" == fips* || "${current_provider}" == symcrypt* ]] \
+          && [[ "${line}" =~ ^([\ $'\t']+)status:[\ $'\t']+active ]]; then
+          if [ ${#BASH_REMATCH[1]} -gt ${header_indent} ]; then
+            fips_active="${current_provider}"
+            break
+          fi
+        fi
+      done <<< "${providers_output}"
+      if [ -n "${fips_active}" ]; then
+        echo "openssl provider '${fips_active}' is registered and active."
       else
-        err $test "ACL FIPS UKI addon file does not exist in active ESP location."
+        err $test "openssl does not have an active fips or symcrypt provider."
       fi
-    fi
-  fi
+      ;;
+    1.1.*)
+      echo "openssl providers check skipped: detected version '${openssl_version_raw}' (legacy FIPS module)."
+      ;;
+    *)
+      err $test "unexpected openssl version '${openssl_version_raw}': FIPS VHDs are expected to ship OpenSSL 3.x or 1.1.x."
+      ;;
+  esac
 
   echo "$test:Finish"
 }