diff --git a/cli/cmd/check.go b/cli/cmd/check.go index edfd1d3c7d829..d9c728dd9bd56 100644 --- a/cli/cmd/check.go +++ b/cli/cmd/check.go @@ -139,12 +139,13 @@ func configureAndRunChecks(cmd *cobra.Command, wout io.Writer, werr io.Writer, o checks := []healthcheck.CategoryID{ healthcheck.KubernetesAPIChecks, healthcheck.KubernetesVersionChecks, + healthcheck.GatewayAPICRDChecks, healthcheck.LinkerdVersionChecks, } crdManifest := bytes.Buffer{} err = renderCRDs(cmd.Context(), nil, &crdManifest, valuespkg.Options{ - // GatewayAPI CRDs are optional so don't check for them. + // Gateway API CRDs are checked separately by GatewayAPICRDChecks. Values: []string{ "installGatewayAPI=false", }, diff --git a/cli/cmd/install.go b/cli/cmd/install.go index cc18a9bb4c302..8b62058514c67 100644 --- a/cli/cmd/install.go +++ b/cli/cmd/install.go @@ -28,7 +28,6 @@ import ( valuespkg "helm.sh/helm/v3/pkg/cli/values" "helm.sh/helm/v3/pkg/engine" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -206,51 +205,6 @@ func checkNoConfig(ctx context.Context, k8sAPI *k8s.KubernetesAPI) error { return nil } -type GatewayAPICRDs int - -const ( - Absent GatewayAPICRDs = iota - Linkerd - External -) - -// checkGatewayAPICRDs returns true if the Gateway API CRDs are installed in the -// cluster, and false otherwise. -func checkGatewayAPICRDs(ctx context.Context, k8sAPI *k8s.KubernetesAPI) (GatewayAPICRDs, error) { - crds := k8sAPI.Apiextensions.ApiextensionsV1().CustomResourceDefinitions() - result := Absent - names := []string{ - "httproutes.gateway.networking.k8s.io", - "grpcroutes.gateway.networking.k8s.io", - } - for _, name := range names { - crd, err := crds.Get(ctx, name, metav1.GetOptions{}) - if err == nil && crd != nil { - if crd.Annotations[k8s.CreatedByAnnotation] != "" { - return Linkerd, nil - } - result = External - if !crdIncludesV1(crd) { - return result, fmt.Errorf("the %s CRD is missing the v1 version, please upgrade to Gateway API v1.1.1 or later", name) - } - } else if kerrors.IsNotFound(err) { - // No action if CRD is not found. - } else { - return Absent, err - } - } - return result, nil -} - -func crdIncludesV1(crd *v1.CustomResourceDefinition) bool { - for _, version := range crd.Spec.Versions { - if version.Name == "v1" { - return true - } - } - return false -} - func installCRDs(ctx context.Context, k8sAPI *k8s.KubernetesAPI, w io.Writer, options valuespkg.Options, format string) error { if err := checkNoConfig(ctx, k8sAPI); err != nil { return err @@ -390,14 +344,14 @@ func renderChartToBuffer(files []*loader.BufferedFile, values map[string]interfa return &buf, vals, nil } -func updateDefaultValues(installed GatewayAPICRDs, defaultValues map[string]interface{}) map[string]interface{} { - if installed == Absent { +func updateDefaultValues(installed healthcheck.GatewayAPICRDs, defaultValues map[string]interface{}) map[string]interface{} { + if installed == healthcheck.GatewayAPIAbsent { // if GW API is not installed, default to false defaultValues["installGatewayAPI"] = false - } else if installed == Linkerd { + } else if installed == healthcheck.GatewayAPILinkerd { // if it is installed by Linkerd, default to true defaultValues["installGatewayAPI"] = true - } else if installed == External { + } else if installed == healthcheck.GatewayAPIExternal { // if it is external, default to false as we are not managing it defaultValues["installGatewayAPI"] = false } @@ -405,7 +359,7 @@ func updateDefaultValues(installed GatewayAPICRDs, defaultValues map[string]inte return defaultValues } -func validateFinalValues(installed GatewayAPICRDs, finalValues map[string]interface{}) error { +func validateFinalValues(installed healthcheck.GatewayAPICRDs, finalValues map[string]interface{}) error { installing := false if installGatewayAPI, ok := finalValues["installGatewayAPI"]; ok { @@ -416,21 +370,17 @@ func validateFinalValues(installed GatewayAPICRDs, finalValues map[string]interf installing = enableHttpRoutes == true } - if installed == Absent { + if installed == healthcheck.GatewayAPIAbsent { if !installing { // if we are not installing GW API Resources and they are not present, error - return errors.New(`The Gateway API CRDs must be installed prior to installing Linkerd. Run: - -kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml - -or see https://gateway-api.sigs.k8s.io/guides/#installing-gateway-api for more options.`) + return healthcheck.GatewayAPICRDsMissingError() } - } else if installed == Linkerd { + } else if installed == healthcheck.GatewayAPILinkerd { if !installing { // if they are installed and managed by Linkerd, we cannot uninstall them return errors.New("Linkerd is providing GW API, but your current install configuration will remove it") } - } else if installed == External { + } else if installed == healthcheck.GatewayAPIExternal { if installing { // if they are installed but are external, we cannot be installing as well return errors.New("Linkerd cannot install the Gateway API CRDs because they are already installed by an external source. Please set `installGatewayAPI` to `false`.") @@ -476,7 +426,7 @@ func renderCRDs(ctx context.Context, k *k8s.KubernetesAPI, w io.Writer, options // If any of the Gateway API CRDs are installed, we default to rendering the // Gateway API CRDs. if k != nil { - installed, err := checkGatewayAPICRDs(ctx, k) + installed, err := healthcheck.CheckGatewayAPICRDs(ctx, k) if err != nil { return err } diff --git a/pkg/healthcheck/healthcheck.go b/pkg/healthcheck/healthcheck.go index 08de4dc750ac3..01c96edb82f22 100644 --- a/pkg/healthcheck/healthcheck.go +++ b/pkg/healthcheck/healthcheck.go @@ -54,6 +54,11 @@ const ( // requirements. KubernetesVersionChecks CategoryID = "kubernetes-version" + // GatewayAPICRDChecks validates that the Gateway API CRDs exist. + // These checks are dependent on the output of KubernetesAPIChecks, so those + // checks must be added first. + GatewayAPICRDChecks CategoryID = "gateway-api-crd" + // LinkerdPreInstall* checks enabled by `linkerd check --pre` // LinkerdPreInstallChecks adds checks to validate that the control plane @@ -191,6 +196,14 @@ var ExpectedServiceAccountNames = []string{ "linkerd-proxy-injector", } +type GatewayAPICRDs int + +const ( + GatewayAPIAbsent GatewayAPICRDs = iota + GatewayAPILinkerd + GatewayAPIExternal +) + var ( retryWindow = 5 * time.Second // RequestTimeout is the time it takes for a request to timeout @@ -551,6 +564,20 @@ func (hc *HealthChecker) allCategories() []*Category { }, false, ), + NewCategory( + GatewayAPICRDChecks, + []Checker{ + { + description: "Gateway API CRDs are installed", + hintAnchor: "gateway-api-crd", + fatal: true, + check: func(ctx context.Context) error { + return CheckGatewayAPICRDsInstalled(ctx, hc.kubeAPI) + }, + }, + }, + false, + ), NewCategory( LinkerdPreInstallChecks, []Checker{ @@ -2201,6 +2228,54 @@ func CheckCustomResourceDefinitions(ctx context.Context, k8sAPI *k8s.KubernetesA return nil } +// CheckGatewayAPICRDs returns whether the Gateway API CRDs are installed in the +// cluster, and whether they were created by Linkerd or an external source. +func CheckGatewayAPICRDs(ctx context.Context, k8sAPI *k8s.KubernetesAPI) (GatewayAPICRDs, error) { + crds := k8sAPI.Apiextensions.ApiextensionsV1().CustomResourceDefinitions() + result := GatewayAPIAbsent + names := []string{ + "httproutes.gateway.networking.k8s.io", + "grpcroutes.gateway.networking.k8s.io", + } + for _, name := range names { + crd, err := crds.Get(ctx, name, metav1.GetOptions{}) + if err == nil && crd != nil { + if crd.Annotations[k8s.CreatedByAnnotation] != "" { + return GatewayAPILinkerd, nil + } + result = GatewayAPIExternal + if !crdIncludesV1(crd) { + return result, fmt.Errorf("the %s CRD is missing the v1 version, please upgrade to Gateway API v1.1.1 or later", name) + } + } else if kerrors.IsNotFound(err) { + // No action if CRD is not found. + } else { + return GatewayAPIAbsent, err + } + } + return result, nil +} + +// CheckGatewayAPICRDsInstalled verifies that the Gateway API CRDs are installed. +func CheckGatewayAPICRDsInstalled(ctx context.Context, k8sAPI *k8s.KubernetesAPI) error { + installed, err := CheckGatewayAPICRDs(ctx, k8sAPI) + if err != nil { + return err + } + if installed == GatewayAPIAbsent { + return GatewayAPICRDsMissingError() + } + return nil +} + +func GatewayAPICRDsMissingError() error { + return errors.New(`The Gateway API CRDs must be installed prior to installing Linkerd. Run: + +kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml + +or see https://gateway-api.sigs.k8s.io/guides/#installing-gateway-api for more options.`) +} + func crdHasVersion(crd *apiextv1.CustomResourceDefinition, version string) bool { for _, crdVersion := range crd.Spec.Versions { if crdVersion.Name == version { @@ -2210,6 +2285,15 @@ func crdHasVersion(crd *apiextv1.CustomResourceDefinition, version string) bool return false } +func crdIncludesV1(crd *apiextv1.CustomResourceDefinition) bool { + for _, version := range crd.Spec.Versions { + if version.Name == "v1" { + return true + } + } + return false +} + // CheckNodesHaveNonDockerRuntime checks that each node has a non-Docker // runtime. This check is only called if proxyInit is not running as root // which is a problem for clusters with a Docker container runtime. diff --git a/pkg/healthcheck/healthcheck_test.go b/pkg/healthcheck/healthcheck_test.go index ea871be3c44db..c80e4fed24ef7 100644 --- a/pkg/healthcheck/healthcheck_test.go +++ b/pkg/healthcheck/healthcheck_test.go @@ -75,6 +75,89 @@ func (hc *HealthChecker) addCheckAsCategory( hc.AppendCategories(testCategory) } +func TestGatewayAPICRDChecks(t *testing.T) { + externalGatewayAPIManifest := `--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: httproutes.gateway.networking.k8s.io +spec: + versions: + - name: v1 +` + unsupportedGatewayAPIManifest := `--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: httproutes.gateway.networking.k8s.io +spec: + versions: + - name: v1alpha1 +` + + testCases := []struct { + name string + resources []string + wantSuccess bool + wantErr string + }{ + { + name: "passes when Gateway API CRDs are installed", + resources: []string{externalGatewayAPIManifest}, + wantSuccess: true, + }, + { + name: "fails when Gateway API CRDs are missing", + wantSuccess: false, + wantErr: "The Gateway API CRDs must be installed prior to installing Linkerd", + }, + { + name: "fails when Gateway API CRDs do not include v1", + resources: []string{unsupportedGatewayAPIManifest}, + wantSuccess: false, + wantErr: "missing the v1 version", + }, + } + + for _, tc := range testCases { + tc := tc // pin + t.Run(tc.name, func(t *testing.T) { + hc := NewHealthChecker( + []CategoryID{GatewayAPICRDChecks}, + &Options{}, + ) + + var err error + hc.kubeAPI, err = k8s.NewFakeAPI(tc.resources...) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + var gotErr error + success, _ := hc.RunChecks(func(result *CheckResult) { + gotErr = result.Err + }) + if success != tc.wantSuccess { + t.Fatalf("expected success=%v, got %v", tc.wantSuccess, success) + } + + if tc.wantErr == "" { + if gotErr != nil { + t.Fatalf("expected no error, got %s", gotErr) + } + return + } + + if gotErr == nil { + t.Fatalf("expected error containing %q", tc.wantErr) + } + if !strings.Contains(gotErr.Error(), tc.wantErr) { + t.Fatalf("expected error containing %q, got %q", tc.wantErr, gotErr.Error()) + } + }) + } +} + func TestHealthChecker(t *testing.T) { nullObserver := func(*CheckResult) {} diff --git a/test/integration/deep/install_test.go b/test/integration/deep/install_test.go index a241140a199b9..d4f23758e17a5 100644 --- a/test/integration/deep/install_test.go +++ b/test/integration/deep/install_test.go @@ -95,11 +95,25 @@ func TestInstallCNIPlugin(t *testing.T) { testutil.AnnotatedFatalf(t, "'kubectl apply' command failed", "'kubectl apply' command failed\n%s", out) } +} + +// TestInstall will install the linkerd control plane to be used in the rest of +// the deep suite tests. +func TestInstall(t *testing.T) { + err := TestHelper.InstallGatewayAPI() + if err != nil { + testutil.AnnotatedFatal(t, "failed to install gateway-api", err) + } + + // perform a linkerd check + checkArgs := []string{"check", "--pre", "--wait=5m"} + if TestHelper.CNI() { + checkArgs = []string{"check", "--pre", "--linkerd-cni-enabled", "--wait=5m"} + } - // perform a linkerd check with --linkerd-cni-enabled timeout := time.Minute err = testutil.RetryFor(timeout, func() error { - out, err = TestHelper.LinkerdRun("check", "--pre", "--linkerd-cni-enabled", "--wait=5m") + _, err = TestHelper.LinkerdRun(checkArgs...) if err != nil { return err } @@ -108,15 +122,6 @@ func TestInstallCNIPlugin(t *testing.T) { if err != nil { testutil.AnnotatedFatal(t, fmt.Sprintf("'linkerd check' command timed-out (%s)", timeout), err) } -} - -// TestInstall will install the linkerd control plane to be used in the rest of -// the deep suite tests. -func TestInstall(t *testing.T) { - err := TestHelper.InstallGatewayAPI() - if err != nil { - testutil.AnnotatedFatal(t, "failed to install gateway-api", err) - } // Install CRDs cmd := []string{ diff --git a/test/integration/install/install_test.go b/test/integration/install/install_test.go index 2efed2b978828..9ad2b701c25af 100644 --- a/test/integration/install/install_test.go +++ b/test/integration/install/install_test.go @@ -81,6 +81,10 @@ func TestCheckPreInstall(t *testing.T) { t.Skip("Skipping pre-install check for upgrade test") } + if err := TestHelper.InstallGatewayAPI(); err != nil { + testutil.AnnotatedFatal(t, "failed to install gateway-api", err) + } + if err := TestHelper.TestCheckPre(); err != nil { t.Fatalf("'linkerd check --pre' command failed: %s", err) } @@ -188,11 +192,6 @@ func TestInstallOrUpgradeCli(t *testing.T) { args = append(args, []string{"--skip-outbound-ports", skippedOutboundPorts}...) } } else { - err := TestHelper.InstallGatewayAPI() - if err != nil { - testutil.AnnotatedFatal(t, "failed to install gateway-api", err) - } - // install CRDs first exec := append([]string{cmd}, append(args, "--crds")...) out, err := TestHelper.LinkerdRun(exec...) diff --git a/test/integration/install/smoke/testdata/check.proxy.golden b/test/integration/install/smoke/testdata/check.proxy.golden index 2155085f0c3cb..8d55674267dee 100644 --- a/test/integration/install/smoke/testdata/check.proxy.golden +++ b/test/integration/install/smoke/testdata/check.proxy.golden @@ -7,6 +7,10 @@ kubernetes-version ------------------ √ is running the minimum Kubernetes API version +gateway-api-crd +--------------- +√ Gateway API CRDs are installed + linkerd-existence ----------------- √ 'linkerd-config' config map exists diff --git a/test/integration/install/testdata/check.proxy.golden b/test/integration/install/testdata/check.proxy.golden index 2155085f0c3cb..8d55674267dee 100644 --- a/test/integration/install/testdata/check.proxy.golden +++ b/test/integration/install/testdata/check.proxy.golden @@ -7,6 +7,10 @@ kubernetes-version ------------------ √ is running the minimum Kubernetes API version +gateway-api-crd +--------------- +√ Gateway API CRDs are installed + linkerd-existence ----------------- √ 'linkerd-config' config map exists