diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 621f2f82287..451244b7f7f 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -127,6 +127,16 @@ var ( // Secure TLS Bootstrapping isn't currently supported on FIPS-enabled VHDs UnsupportedSecureTLSBootstrapping: true, } + VHDAzureLinuxV3Gen2FIPS = &Image{ + Name: "AzureLinuxV3gen2fips", + OS: OSAzureLinux, + Arch: "amd64", + Distro: datamodel.AKSAzureLinuxV3Gen2FIPS, + Gallery: imageGalleryLinux, + UnsupportedLocalDns: true, + // Secure TLS Bootstrapping isn't currently supported on FIPS-enabled VHDs + UnsupportedSecureTLSBootstrapping: true, + } // this is a particular 2204gen2containerd image originally built with private packages, // if we ever want to update this then we'd need to run a new VHD build using private package overrides VHDUbuntu2204Gen2ContainerdPrivateKubePkg = &Image{ diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 1ff774cf356..0dba705a302 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -26,7 +26,9 @@ func Test_AzureLinux3OSGuard(t *testing.T) { BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.AgentPoolProfile.LocalDNSProfile = nil }, - Validator: func(ctx context.Context, s *Scenario) {}, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFIPSProvider(ctx, s) + }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) }, @@ -260,6 +262,29 @@ func Test_ACLGen2FIPSTL(t *testing.T) { ValidateFileHasContent(ctx, s, "/etc/os-release", "ID=azurelinux") ValidateFileHasContent(ctx, s, "/etc/os-release", "VARIANT_ID=azurecontainerlinux") ValidateACLFIPSEnabled(ctx, s) + ValidateFIPSProvider(ctx, s) + }, + }, + }) +} + +func Test_AzureLinuxV3Gen2FIPS(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that a node using the Azure Linux V3 Gen2 FIPS VHD can be properly bootstrapped and FIPS is active at runtime", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3Gen2FIPS, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + // LocalDNS isn't currently supported on FIPS-enabled VHDs. + nbc.AgentPoolProfile.LocalDNSProfile = nil + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties.AdditionalCapabilities = &armcompute.AdditionalCapabilities{ + EnableFips1403Encryption: to.Ptr(true), + } + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFIPSProvider(ctx, s) }, }, }) @@ -768,6 +793,7 @@ func Test_Ubuntu2204FIPS(t *testing.T) { ValidateInstalledPackageVersion(ctx, s, "moby-containerd", components.GetExpectedPackageVersions("containerd", "ubuntu", "r2204")[0]) ValidateInstalledPackageVersion(ctx, s, "moby-runc", components.GetExpectedPackageVersions("runc", "ubuntu", "r2204")[0]) ValidateSSHServiceEnabled(ctx, s) + ValidateFIPSProvider(ctx, s) }, }, }) @@ -787,6 +813,7 @@ func Test_Ubuntu2004FIPS(t *testing.T) { ValidateInstalledPackageVersion(ctx, s, "moby-containerd", components.GetExpectedPackageVersions("containerd", "ubuntu", "r2004")[0]) ValidateInstalledPackageVersion(ctx, s, "moby-runc", components.GetExpectedPackageVersions("runc", "ubuntu", "r2004")[0]) ValidateSSHServiceEnabled(ctx, s) + ValidateFIPSProvider(ctx, s) }, }, }) @@ -815,6 +842,7 @@ func Test_Ubuntu2204Gen2FIPS(t *testing.T) { ValidateInstalledPackageVersion(ctx, s, "moby-containerd", components.GetExpectedPackageVersions("containerd", "ubuntu", "r2204")[0]) ValidateInstalledPackageVersion(ctx, s, "moby-runc", components.GetExpectedPackageVersions("runc", "ubuntu", "r2204")[0]) ValidateSSHServiceEnabled(ctx, s) + ValidateFIPSProvider(ctx, s) }, }, }) @@ -844,6 +872,7 @@ func Test_Ubuntu2204Gen2FIPSTL(t *testing.T) { ValidateInstalledPackageVersion(ctx, s, "moby-containerd", components.GetExpectedPackageVersions("containerd", "ubuntu", "r2204")[0]) ValidateInstalledPackageVersion(ctx, s, "moby-runc", components.GetExpectedPackageVersions("runc", "ubuntu", "r2204")[0]) ValidateSSHServiceEnabled(ctx, s) + ValidateFIPSProvider(ctx, s) }, }, }) @@ -2698,7 +2727,9 @@ func Test_AzureLinux3OSGuard_PMC_Install(t *testing.T) { BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.AgentPoolProfile.LocalDNSProfile = nil }, - Validator: func(ctx context.Context, s *Scenario) {}, + Validator: func(ctx context.Context, s *Scenario) { + ValidateFIPSProvider(ctx, s) + }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) if vmss.Tags == nil { diff --git a/e2e/validators.go b/e2e/validators.go index 96597ada092..bd5de6ba3e9 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -486,16 +486,13 @@ func ValidateFileExists(ctx context.Context, s *Scenario, fileName string) { } } +// ValidateACLFIPSEnabled asserts ACL-specific FIPS markers are present on the node: +// the /etc/system-fips marker file written by vhdbuilder/scripts/linux/acl/tool_installs_acl.sh. +// Kernel FIPS mode (/proc/sys/crypto/fips_enabled == 1) is universal and is asserted by +// ValidateFIPSProvider; callers should compose the two validators when both are needed. func ValidateACLFIPSEnabled(ctx context.Context, s *Scenario) { s.T.Helper() ValidateFileExists(ctx, s, "/etc/system-fips") - execScriptOnVMForScenarioValidateExitCode( - ctx, - s, - `test "$(cat /proc/sys/crypto/fips_enabled)" = "1"`, - 0, - "expected /proc/sys/crypto/fips_enabled to be 1", - ) } func ValidateFileDoesNotExist(ctx context.Context, s *Scenario, fileName string) { @@ -653,6 +650,112 @@ func ValidateFileExcludesExactContent(ctx context.Context, s *Scenario, fileName } } +// ValidateFIPSProvider verifies that FIPS is properly configured on the node: +// 1. Kernel FIPS mode is enabled (/proc/sys/crypto/fips_enabled == 1). +// 2. OpenSSL (3.x) has an active FIPS or SymCrypt provider loaded. The check is +// skipped on hosts shipping OpenSSL 1.1.x (e.g. Ubuntu 20.04 FIPS), which use +// the legacy FIPS module rather than the providers interface. +// 3. /opt/cni/bin/portmap runs without panicking (regression guard for ICM 51000001009688 +// where the OpenSSL FIPS provider was not loaded on AzureLinux V3 FIPS nodes). +func ValidateFIPSProvider(ctx context.Context, s *Scenario) { + s.T.Helper() + + // 1. Kernel FIPS mode. + fipsEnabled := execScriptOnVMForScenarioValidateExitCode(ctx, s, "cat /proc/sys/crypto/fips_enabled", 0, "could not read /proc/sys/crypto/fips_enabled") + require.Equal(s.T, "1", strings.TrimSpace(fipsEnabled.stdout), "expected /proc/sys/crypto/fips_enabled to be 1, got %q", fipsEnabled.stdout) + + // 2. OpenSSL provider must include an active fips or symcrypt provider on OpenSSL 3.x. + // 1.1.x (Ubuntu 20.04 FIPS) uses the legacy FIPS module and is skipped. Merge stderr + // (`2>&1`) so a version banner written to stderr still parses. + opensslVersion := execScriptOnVMForScenarioValidateExitCode(ctx, s, "openssl version 2>&1", 0, "could not run openssl version") + versionFields := strings.Fields(opensslVersion.stdout) + require.GreaterOrEqual(s.T, len(versionFields), 2, + "could not parse openssl version output: %q", opensslVersion.stdout) + version := versionFields[1] + switch { + case strings.HasPrefix(version, "3."): + providers := execScriptOnVMForScenarioValidateExitCode(ctx, s, "openssl list -providers", 0, "could not list openssl providers") + // Prefix match so "symcrypt" covers AzureLinux V3 / ACL's "symcryptprovider". See ICM 51000001009688. + require.True(s.T, opensslProviderActive(providers.stdout, "fips", "symcrypt"), + "expected openssl to have an active fips or symcrypt provider, got:\n%s", providers.stdout) + case strings.HasPrefix(version, "1.1."): + s.T.Logf("openssl providers check skipped: detected version %q (legacy FIPS module)", strings.TrimSpace(opensslVersion.stdout)) + default: + s.T.Fatalf("unexpected openssl version %q: FIPS VHDs are expected to ship OpenSSL 3.x or 1.1.x", strings.TrimSpace(opensslVersion.stdout)) + } + + // 3. portmap panic check (best-effort). The original FIPS regression manifested as + // /opt/cni/bin/portmap panicking when the OpenSSL FIPS provider was missing. Skip if + // the binary isn't at the standard CNI path (e.g. ACL stages it under /opt/cni/downloads/); + // checks 1 and 2 are already authoritative. Match specific Go runtime panic markers + // rather than the bare substring `runtime error:` which appears in CNI usage text. + portmapBin := "/opt/cni/bin/portmap" + portmapPresent := execScriptOnVMForScenario(ctx, s, fmt.Sprintf("test -x %s", portmapBin)) + if portmapPresent.exitCode != "0" { + s.T.Logf("portmap panic check skipped: %s not present or not executable on this VHD", portmapBin) + s.T.Logf("FIPS provider validation passed") + return + } + portmap := execScriptOnVMForScenario(ctx, s, fmt.Sprintf("%s < /dev/null", portmapBin)) + panicMarkers := []*regexp.Regexp{ + regexp.MustCompile(`(?m)^panic:`), + regexp.MustCompile(`(?m)^fatal error:`), + regexp.MustCompile(`runtime\.gopanic`), + regexp.MustCompile(`goroutine \d+ \[running\]`), + } + for _, re := range panicMarkers { + require.False(s.T, re.MatchString(portmap.stderr), + "portmap runtime failure matched %q, indicating FIPS provider misconfiguration:\nstdout:\n%s\nstderr:\n%s", re, portmap.stdout, portmap.stderr) + require.False(s.T, re.MatchString(portmap.stdout), + "portmap runtime failure matched %q, indicating FIPS provider misconfiguration:\nstdout:\n%s\nstderr:\n%s", re, portmap.stdout, portmap.stderr) + } + + s.T.Logf("FIPS provider validation passed") +} + +// Package-level regex compiled once at init. +var ( + // Provider header: indented (spaces or tabs) name with no key/value separator. + // Group 1 = indent (used to scope status lines to their block), group 2 = name. + opensslProviderHeaderRE = regexp.MustCompile(`^([ \t]+)(\S+)\s*$`) + opensslStatusLineRE = regexp.MustCompile(`^[ \t]+status:\s*(\S+)`) +) + +// opensslProviderActive parses `openssl list -providers` output and returns true if any +// provider whose name has one of the given prefixes is reported as `status: active`. +// The status line is scoped to its enclosing provider block (strictly more indented than +// the header) so an active default provider cannot mask an inactive fips/symcrypt one. +func opensslProviderActive(output string, providerPrefixes ...string) bool { + matches := func(name string) bool { + for _, p := range providerPrefixes { + if strings.HasPrefix(name, p) { + return true + } + } + return false + } + var current string + var headerIndent int + for _, line := range strings.Split(output, "\n") { + line = strings.TrimRight(line, "\r") + if m := opensslProviderHeaderRE.FindStringSubmatch(line); m != nil { + headerIndent = len(m[1]) + current = m[2] + continue + } + if current == "" || !matches(current) { + continue + } + if m := opensslStatusLineRE.FindStringSubmatch(line); m != nil && m[1] == "active" { + leading := len(line) - len(strings.TrimLeft(line, " \t")) + if leading > headerIndent { + return true + } + } + } + return false +} + func ServiceCanRestartValidator(ctx context.Context, s *Scenario, serviceName string, restartTimeoutInSeconds int) { s.T.Helper() steps := []string{ diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index 9d538ec7c06..2c879f28a69 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -591,44 +591,120 @@ testChrony() { testFips() { local test="testFips" echo "$test:Start" + # Sourcing cse_helpers.sh overwrites the global OS_VERSION from /etc/os-release + # VERSION_ID (e.g. "2.0", "3.0", "3.0.20260506" on ACL, "20.04"), so the value + # passed in as $1 is the /etc/os-release one, not the pipeline-style OS_VERSION. os_version=$1 enable_fips=$2 # shellcheck disable=SC3010 - if [[ (${os_version} == "20.04" || ${os_version} == "22.04" || ${os_version} == "V2" || ${os_version} == "acl") && ${enable_fips,,} == "true" ]]; then + if [[ ${enable_fips,,} != "true" ]]; then + echo "$test:Finish" + return + fi + + # Known FIPS-capable VERSION_ID values. New FIPS-enabled distros MUST be added + # here; otherwise the test fails loudly instead of silently no-op'ing (ICM + # 51000001009688). ACL VERSION_ID carries a date suffix (e.g. "3.0.20260506"), + # hence the "3.0.*" entry. `err` writes to stderr only, so we `return` after. + case "${os_version}" in + 20.04|22.04|24.04|2.0|2.0.*|3.0|3.0.*) ;; + *) + err $test "testFips invoked with enable_fips=true on unrecognized os_version '${os_version}'; add it to the allowlist." + echo "$test:Finish" + return + ;; + esac + + if [ -f /proc/sys/crypto/fips_enabled ]; then + fips_enabled=$(cat /proc/sys/crypto/fips_enabled) + if [ "${fips_enabled}" = "1" ]; then + echo "FIPS is enabled." + else + err $test "content of /proc/sys/crypto/fips_enabled is not 1." + fi + else + err $test "FIPS is not enabled." + fi + + if [ "${os_version}" = "20.04" ]; then kernel=$(uname -r) - if [ -f /proc/sys/crypto/fips_enabled ]; then - fips_enabled=$(cat /proc/sys/crypto/fips_enabled) - if [ "${fips_enabled}" = "1" ]; then - echo "FIPS is enabled." - else - err $test "content of /proc/sys/crypto/fips_enabled is not 1." - fi + if [ -f /usr/src/linux-headers-${kernel}/Makefile ]; then + echo "fips header files exist." else - err $test "FIPS is not enabled." + err $test "fips header files don't exist." fi + fi - if [ ${os_version} = "20.04" ]; then - if [ -f /usr/src/linux-headers-${kernel}/Makefile ]; then - echo "fips header files exist." - else - err $test "fips header files don't exist." - fi + if [ "${OS_SKU}" = "AzureContainerLinux" ]; then + if [ -f /etc/system-fips ]; then + echo "/etc/system-fips marker file exists." + else + err $test "/etc/system-fips marker file does not exist." + fi + if [ -f /boot/EFI/Linux/acl.efi.extra.d/fips.addon.efi ]; then + echo "ACL FIPS UKI addon file exists in active ESP location." + else + err $test "ACL FIPS UKI addon file does not exist in active ESP location." fi + fi - if [ ${os_version} = "acl" ]; then - if [ -f /etc/system-fips ]; then - echo "/etc/system-fips marker file exists." - else - err $test "/etc/system-fips marker file does not exist." - fi - if [ -f /boot/EFI/Linux/acl.efi.extra.d/fips.addon.efi ]; then - echo "ACL FIPS UKI addon file exists in active ESP location." + # OpenSSL must have an active FIPS or SymCrypt provider on 3.x (ICM 51000001009688 + # was caused by kernel FIPS on with no provider, causing portmap to panic). Ubuntu + # 20.04 ships 1.1.x and uses the legacy FIPS module — skip there. Keep in sync with + # the Go validator in e2e/validators.go. + if ! command -v openssl >/dev/null 2>&1; then + err $test "openssl binary not found on a FIPS-enabled VHD." + echo "$test:Finish" + return + fi + openssl_version_raw=$(openssl version 2>&1) + openssl_version=$(echo "${openssl_version_raw}" | awk '{print $2}') + case "${openssl_version}" in + "") + err $test "could not parse openssl version (raw output: '${openssl_version_raw}')." + ;; + 3.*) + providers_output=$(openssl list -providers 2>&1) + echo "openssl list -providers output:" + echo "${providers_output}" + # Walk each provider block, scoping `status: active` to its enclosing header. + # Prefix match so "symcrypt" covers AzureLinux V3 / ACL's "symcryptprovider". + # Use `[ \t]` (not `[[:space:]]`) so a trailing CR can't break header detection; + # also strip \r explicitly. + fips_active="" + current_provider="" + header_indent=0 + while IFS= read -r line; do + line="${line%$'\r'}" + # shellcheck disable=SC3010 + if [[ "${line}" =~ ^([\ $'\t']+)([^[:space:]:]+)[\ $'\t']*$ ]]; then + header_indent=${#BASH_REMATCH[1]} + current_provider="${BASH_REMATCH[2]}" + continue + fi + # shellcheck disable=SC3010 + if [[ "${current_provider}" == fips* || "${current_provider}" == symcrypt* ]] \ + && [[ "${line}" =~ ^([\ $'\t']+)status:[\ $'\t']+active ]]; then + if [ ${#BASH_REMATCH[1]} -gt ${header_indent} ]; then + fips_active="${current_provider}" + break + fi + fi + done <<< "${providers_output}" + if [ -n "${fips_active}" ]; then + echo "openssl provider '${fips_active}' is registered and active." else - err $test "ACL FIPS UKI addon file does not exist in active ESP location." + err $test "openssl does not have an active fips or symcrypt provider." fi - fi - fi + ;; + 1.1.*) + echo "openssl providers check skipped: detected version '${openssl_version_raw}' (legacy FIPS module)." + ;; + *) + err $test "unexpected openssl version '${openssl_version_raw}': FIPS VHDs are expected to ship OpenSSL 3.x or 1.1.x." + ;; + esac echo "$test:Finish" }