diff --git a/utils/kubernetes/apply-helm-chart.go b/utils/kubernetes/apply-helm-chart.go index fc2e357e..7c25f0cf 100644 --- a/utils/kubernetes/apply-helm-chart.go +++ b/utils/kubernetes/apply-helm-chart.go @@ -241,7 +241,7 @@ type ApplyHelmChartConfig struct { // }, // OverrideValues: vals, // }) -func (client *Client) ApplyHelmChart(cfg ApplyHelmChartConfig) error { +func (c *Client) ApplyHelmChart(cfg ApplyHelmChartConfig) error { setupDefaults(&cfg) if err := setupChartVersion(&cfg); err != nil { @@ -264,11 +264,10 @@ func (client *Client) ApplyHelmChart(cfg ApplyHelmChartConfig) error { return ErrApplyHelmChart(err) } - actionConfig, cleanup, err := createHelmActionConfig(client, cfg) + actionConfig, err := c.createHelmActionConfig(cfg, c.getRESTClientGetter()) if err != nil { return ErrApplyHelmChart(err) } - defer cleanup() // Before installing a helm chart, check if it already exists in the cluster // this is a workaround make the helm chart installation idempotent @@ -408,96 +407,16 @@ func checkIfInstallable(ch *chart.Chart) error { } // createHelmActionConfig generates the actionConfig with the appropriate defaults -func createHelmActionConfig(c *Client, cfg ApplyHelmChartConfig) (*action.Configuration, func(), error) { +func (c *Client) createHelmActionConfig(cfg ApplyHelmChartConfig, restClientGetter genericclioptions.RESTClientGetter) (*action.Configuration, error) { // Set the environment variable needed by the Init methods _ = os.Setenv("HELM_DRIVER_SQL_CONNECTION_STRING", cfg.SQLConnectionString) - var tempFiles []string - cleanup := func() { - for _, f := range tempFiles { - _ = os.Remove(f) - } - } - - // KubeConfig setup - kubeConfig := genericclioptions.NewConfigFlags(false) - // Set KubeConfig to DevNull to prevent read from local kubeconfig - // to prevent conflicts between "data" and "files" properties (CAFile, CAData and KeyFile, KeyData) - // ConfigFlags only allows setting CAFile, KeyFile but not CAData, KeyData. - // When the library reads the original kubeconfig containing cert data / key data AND we specify cert file / key file, these configurations conflict - devNull := os.DevNull - kubeConfig.KubeConfig = &devNull - kubeConfig.APIServer = &c.RestConfig.Host - kubeConfig.BearerToken = &c.RestConfig.BearerToken - kubeConfig.Insecure = &c.RestConfig.Insecure - - // Set username and password for basic auth if available - if c.RestConfig.Username != "" { - kubeConfig.Username = &c.RestConfig.Username - } - if c.RestConfig.Password != "" { - kubeConfig.Password = &c.RestConfig.Password - } - - // Only set CA file if not running in insecure mode - if !c.RestConfig.Insecure { - if len(c.RestConfig.CAData) > 0 { - caFileName, err := setDataAndReturnFilename(c.RestConfig.CAData) - if err != nil { - cleanup() // Clean up any files created so far - return nil, nil, err - } - tempFiles = append(tempFiles, caFileName) - kubeConfig.CAFile = &caFileName - } - } - - // Set client certificate data if available - if len(c.RestConfig.CertData) > 0 { - certFileName, err := setDataAndReturnFilename(c.RestConfig.CertData) - if err != nil { - cleanup() - return nil, nil, err - } - tempFiles = append(tempFiles, certFileName) - kubeConfig.CertFile = &certFileName - } - - // Set client key data if available - if len(c.RestConfig.KeyData) > 0 { - keyFileName, err := setDataAndReturnFilename(c.RestConfig.KeyData) - if err != nil { - cleanup() // Clean up any files created so far - return nil, nil, err - } - tempFiles = append(tempFiles, keyFileName) - kubeConfig.KeyFile = &keyFileName - } - actionConfig := new(action.Configuration) - if err := actionConfig.Init(kubeConfig, cfg.Namespace, string(cfg.HelmDriver), cfg.Logger); err != nil { - cleanup() // Clean up any files created so far - return nil, nil, ErrApplyHelmChart(err) + if err := actionConfig.Init(restClientGetter, cfg.Namespace, string(cfg.HelmDriver), cfg.Logger); err != nil { + return nil, ErrApplyHelmChart(err) } - return actionConfig, cleanup, nil -} - -// Populates a file in temp directory with the passed data and returns the filename -func setDataAndReturnFilename(data []byte) (string, error) { - f, err := os.CreateTemp("", "") - if err != nil { - return "", err - } - defer func() { _ = f.Close() }() // Close file immediately after writing - - _, err = f.Write(data) - if err != nil { - _ = os.Remove(f.Name()) // Clean up on write error - return "", err - } - - return f.Name(), nil + return actionConfig, nil } // generateAction generates an action function using action.Configuration @@ -554,7 +473,8 @@ func createHelmPathFromHelmChartLocation(loc HelmChartLocation) (string, error) getter.Provider{ Schemes: []string{"http", "https"}, New: getter.NewHTTPGetter, - }}, + }, + }, ) if err != nil { return "", ErrApplyHelmChart(err) @@ -643,7 +563,7 @@ func (helmEntries HelmEntries) GetEntryWithAppVersion(entry, appVersion string) return HelmEntryMetadata{}, false } -// GetEntryWithAppVersion takes in the entry name and the appversion and returns the corresponding +// GetEntryWithChartVersion takes in the entry name and the appversion and returns the corresponding // metadata for the parameters if it exists func (helmEntries HelmEntries) GetEntryWithChartVersion(entry, chartVersion string) (HelmEntryMetadata, bool) { hem, ok := helmEntries[entry] diff --git a/utils/kubernetes/client-config-getter.go b/utils/kubernetes/client-config-getter.go new file mode 100644 index 00000000..ab76fa17 --- /dev/null +++ b/utils/kubernetes/client-config-getter.go @@ -0,0 +1,101 @@ +package kubernetes + +import ( + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +type clientConfigRESTClientGetter struct { + clientConfig clientcmd.ClientConfig +} + +type restConfigClientConfig struct { + restConfig *rest.Config +} + +var _ genericclioptions.RESTClientGetter = (*clientConfigRESTClientGetter)(nil) +var _ clientcmd.ClientConfig = (*restConfigClientConfig)(nil) + +func newClientConfigRESTClientGetter(clientConfig clientcmd.ClientConfig) genericclioptions.RESTClientGetter { + return &clientConfigRESTClientGetter{clientConfig: clientConfig} +} + +func newRESTConfigRESTClientGetter(config *rest.Config) genericclioptions.RESTClientGetter { + return newClientConfigRESTClientGetter(&restConfigClientConfig{ + restConfig: rest.CopyConfig(config), + }) +} + +func (g *clientConfigRESTClientGetter) ToRESTConfig() (*rest.Config, error) { + config, err := g.clientConfig.ClientConfig() + if err != nil { + return nil, err + } + configureRESTConfig(config) + return config, nil +} + +func (g *clientConfigRESTClientGetter) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) { + config, err := g.ToRESTConfig() + if err != nil { + return nil, err + } + + discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) + if err != nil { + return nil, err + } + + return memory.NewMemCacheClient(discoveryClient), nil +} + +func (g *clientConfigRESTClientGetter) ToRESTMapper() (meta.RESTMapper, error) { + discoveryClient, err := g.ToDiscoveryClient() + if err != nil { + return nil, err + } + + mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) + return restmapper.NewShortcutExpander(mapper, discoveryClient, func(string) {}), nil +} + +func (g *clientConfigRESTClientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig { + return g.clientConfig +} + +func (c *restConfigClientConfig) RawConfig() (clientcmdapi.Config, error) { + const connectionName = "meshkit-connection" + + config := clientcmdapi.NewConfig() + config.Clusters[connectionName] = &clientcmdapi.Cluster{ + Server: c.restConfig.Host, + TLSServerName: c.restConfig.ServerName, + InsecureSkipTLSVerify: c.restConfig.Insecure, + CertificateAuthority: c.restConfig.CAFile, + CertificateAuthorityData: c.restConfig.CAData, + DisableCompression: c.restConfig.DisableCompression, + } + config.Contexts[connectionName] = &clientcmdapi.Context{ + Cluster: connectionName, + } + config.CurrentContext = connectionName + return *config, nil +} + +func (c *restConfigClientConfig) ClientConfig() (*rest.Config, error) { + return rest.CopyConfig(c.restConfig), nil +} + +func (c *restConfigClientConfig) Namespace() (string, bool, error) { + return "default", false, nil +} + +func (c *restConfigClientConfig) ConfigAccess() clientcmd.ConfigAccess { + return nil +} diff --git a/utils/kubernetes/client-config-getter_test.go b/utils/kubernetes/client-config-getter_test.go new file mode 100644 index 00000000..f80f953a --- /dev/null +++ b/utils/kubernetes/client-config-getter_test.go @@ -0,0 +1,269 @@ +package kubernetes + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +const execCredentialHelperEnv = "MESHKIT_EXEC_CREDENTIAL_HELPER" + +func testExecKubeConfig() []byte { + return []byte(`apiVersion: v1 +kind: Config +clusters: +- name: test-cluster + cluster: + server: https://cluster.example.com + insecure-skip-tls-verify: true +contexts: +- name: test-context + context: + cluster: test-cluster + user: test-user +current-context: test-context +users: +- name: test-user + user: + exec: + apiVersion: client.authentication.k8s.io/v1 + command: credential-plugin + interactiveMode: Never +`) +} + +func renewableExecKubeConfig(t *testing.T, serverURL, stateFile string) []byte { + t.Helper() + + const ( + clusterName = "test-cluster" + contextName = "test-context" + userName = "test-user" + ) + config := clientcmdapi.NewConfig() + config.Clusters[clusterName] = &clientcmdapi.Cluster{ + Server: serverURL, + InsecureSkipTLSVerify: true, + } + config.AuthInfos[userName] = &clientcmdapi.AuthInfo{ + Exec: &clientcmdapi.ExecConfig{ + APIVersion: "client.authentication.k8s.io/v1", + Command: os.Args[0], + Args: []string{"-test.run=TestExecCredentialHelperProcess"}, + Env: []clientcmdapi.ExecEnvVar{ + {Name: execCredentialHelperEnv, Value: "1"}, + {Name: "MESHKIT_EXEC_CREDENTIAL_STATE_FILE", Value: stateFile}, + }, + InteractiveMode: clientcmdapi.NeverExecInteractiveMode, + }, + } + config.Contexts[contextName] = &clientcmdapi.Context{ + Cluster: clusterName, + AuthInfo: userName, + } + config.CurrentContext = contextName + + data, err := clientcmd.Write(*config) + if err != nil { + t.Fatalf("clientcmd.Write() error = %v", err) + } + return data +} + +func TestExecCredentialHelperProcess(t *testing.T) { + if os.Getenv(execCredentialHelperEnv) != "1" { + return + } + + stateFile := os.Getenv("MESHKIT_EXEC_CREDENTIAL_STATE_FILE") + token := "expired-token" + state := "1" + if _, err := os.Stat(stateFile); err == nil { + token = "fresh-token" + state = "2" + } + if err := os.WriteFile(stateFile, []byte(state), 0o600); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "write exec credential state: %v", err) + os.Exit(1) + } + + _, _ = fmt.Fprintf(os.Stdout, `{"apiVersion":"client.authentication.k8s.io/v1","kind":"ExecCredential","status":{"token":%q}}`, token) + os.Exit(0) +} + +func TestNewRetainsKubeConfigLoader(t *testing.T) { + client, err := New(testExecKubeConfig()) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + loader := client.getRESTClientGetter().ToRawKubeConfigLoader() + if loader == nil { + t.Fatal("New() did not retain the kubeconfig loader") + } + + rawConfig, err := loader.RawConfig() + if err != nil { + t.Fatalf("RawConfig() error = %v", err) + } + authInfo := rawConfig.AuthInfos["test-user"] + if authInfo == nil || authInfo.Exec == nil { + t.Fatal("retained kubeconfig loader lost exec authentication") + } + if authInfo.Exec.Command != "credential-plugin" { + t.Fatalf("exec command = %q, want %q", authInfo.Exec.Command, "credential-plugin") + } +} + +func TestClientConfigRESTClientGetterPreservesLoader(t *testing.T) { + _, loader, err := detectKubeConfig(testExecKubeConfig()) + if err != nil { + t.Fatalf("detectKubeConfig() error = %v", err) + } + + getter := newClientConfigRESTClientGetter(loader) + if getter.ToRawKubeConfigLoader() != loader { + t.Fatal("RESTClientGetter did not return the retained kubeconfig loader") + } + + config, err := getter.ToRESTConfig() + if err != nil { + t.Fatalf("ToRESTConfig() error = %v", err) + } + if config.ExecProvider == nil || config.ExecProvider.Command != "credential-plugin" { + t.Fatal("RESTClientGetter did not preserve exec authentication") + } + if config.QPS != 50 || config.Burst != 100 { + t.Fatalf("REST config rate limits = (%v, %d), want (50, 100)", config.QPS, config.Burst) + } +} + +func TestNewInitializesRESTClientGetter(t *testing.T) { + client, err := New(testExecKubeConfig()) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + getter := client.getRESTClientGetter() + if getter == nil { + t.Fatal("New() did not initialize a RESTClientGetter") + } + + config, err := getter.ToRESTConfig() + if err != nil { + t.Fatalf("ToRESTConfig() error = %v", err) + } + if config.ExecProvider == nil || config.ExecProvider.Command != "credential-plugin" { + t.Fatal("initialized RESTClientGetter lost exec authentication") + } + if client.RestConfig.QPS != 50 || client.RestConfig.Burst != 100 { + t.Fatalf("client rate limits = (%v, %d), want (50, 100)", client.RestConfig.QPS, client.RestConfig.Burst) + } +} + +func TestRESTClientGetterFallsBackForDirectClient(t *testing.T) { + client := &Client{RestConfig: rest.Config{ + Host: "https://cluster.example.com", + BearerToken: "test-token", + TLSClientConfig: rest.TLSClientConfig{ + Insecure: true, + }, + QPS: 1, + Burst: 2, + }} + + getter := client.getRESTClientGetter() + config, err := getter.ToRESTConfig() + if err != nil { + t.Fatalf("ToRESTConfig() error = %v", err) + } + if config.Host != client.RestConfig.Host || config.BearerToken != client.RestConfig.BearerToken { + t.Fatalf("REST config = (%q, %q), want (%q, %q)", config.Host, config.BearerToken, client.RestConfig.Host, client.RestConfig.BearerToken) + } + if config.QPS != 50 || config.Burst != 100 { + t.Fatalf("REST config rate limits = (%v, %d), want (50, 100)", config.QPS, config.Burst) + } + + namespace, explicit, err := getter.ToRawKubeConfigLoader().Namespace() + if err != nil { + t.Fatalf("Namespace() error = %v", err) + } + if namespace != "default" || explicit { + t.Fatalf("Namespace() = (%q, %v), want (%q, false)", namespace, explicit, "default") + } + + rawConfig, err := getter.ToRawKubeConfigLoader().RawConfig() + if err != nil { + t.Fatalf("RawConfig() error = %v", err) + } + cluster := rawConfig.Clusters["meshkit-connection"] + if cluster == nil || cluster.Server != client.RestConfig.Host { + t.Fatalf("raw cluster = %#v, want server %q", cluster, client.RestConfig.Host) + } +} + +func TestRESTConfigGetterReturnsIndependentCopies(t *testing.T) { + getter := newRESTConfigRESTClientGetter(&rest.Config{ + Host: "https://cluster.example.com", + BearerToken: "test-token", + }) + + first, err := getter.ToRESTConfig() + if err != nil { + t.Fatalf("first ToRESTConfig() error = %v", err) + } + first.Host = "https://different.example.com" + first.BearerToken = "different-token" + + second, err := getter.ToRESTConfig() + if err != nil { + t.Fatalf("second ToRESTConfig() error = %v", err) + } + if second.Host != "https://cluster.example.com" || second.BearerToken != "test-token" { + t.Fatalf("second REST config = (%q, %q), want original values", second.Host, second.BearerToken) + } +} + +func TestRESTClientGetterExecutesAndRenewsCredentials(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer fresh-token" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"gitVersion":"v1.34.0"}`)) + })) + defer server.Close() + + stateFile := filepath.Join(t.TempDir(), "exec-credential-state") + client, err := New(renewableExecKubeConfig(t, server.URL, stateFile)) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + discoveryClient, err := client.getRESTClientGetter().ToDiscoveryClient() + if err != nil { + t.Fatalf("ToDiscoveryClient() error = %v", err) + } + if _, err := discoveryClient.ServerVersion(); err == nil { + t.Fatal("first ServerVersion() unexpectedly succeeded with the rejected credential") + } + if _, err := discoveryClient.ServerVersion(); err != nil { + t.Fatalf("second ServerVersion() error after credential renewal = %v", err) + } + + state, err := os.ReadFile(stateFile) + if err != nil { + t.Fatalf("ReadFile(%q) error = %v", stateFile, err) + } + if string(state) != "2" { + t.Fatalf("exec credential helper state = %q, want %q", state, "2") + } +} diff --git a/utils/kubernetes/client.go b/utils/kubernetes/client.go index dd0e1942..3d526405 100644 --- a/utils/kubernetes/client.go +++ b/utils/kubernetes/client.go @@ -12,22 +12,27 @@ import ( // DetectKubeConfig detects the kubeconfig for the kubernetes cluster and returns it func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { + config, _, err = detectKubeConfig(configfile) + return config, err +} + +func detectKubeConfig(configfile []byte) (config *rest.Config, loader clientcmd.ClientConfig, err error) { if len(configfile) > 0 { var cfgFile []byte _, cfgFile, err = ProcessConfig(configfile, "") if err != nil { - return nil, err + return nil, nil, err } - if config, err = clientcmd.RESTConfigFromKubeConfig(cfgFile); err == nil { - return config, nil + if config, loader, err = loadClientConfigFromKubeconfig(cfgFile); err == nil { + return config, loader, nil } } // If deployed within the cluster if config, err = rest.InClusterConfig(); err == nil { - return config, nil + return config, nil, nil } // Look for kubeconfig from the path mentioned in $KUBECONFIG @@ -35,10 +40,10 @@ func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { if kubeconfig != "" { _, cfgFile, err := ProcessConfig(kubeconfig, "") if err != nil { - return nil, err + return nil, nil, err } - if config, err = clientcmd.RESTConfigFromKubeConfig(cfgFile); err == nil { - return config, nil + if config, loader, err = loadClientConfigFromKubeconfig(cfgFile); err == nil { + return config, loader, nil } } @@ -46,13 +51,27 @@ func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { path := filepath.Join(utils.GetHome(), ".kube", "config") _, cfgFile, err := ProcessConfig(path, "") if err != nil { - return nil, err + return nil, nil, err + } + if config, loader, err = loadClientConfigFromKubeconfig(cfgFile); err == nil { + return config, loader, nil + } + + return nil, nil, ErrRestConfigFromKubeConfig(err) +} + +func loadClientConfigFromKubeconfig(kubeconfig []byte) (*rest.Config, clientcmd.ClientConfig, error) { + loader, err := clientcmd.NewClientConfigFromBytes(kubeconfig) + if err != nil { + return nil, nil, err } - if config, err = clientcmd.RESTConfigFromKubeConfig(cfgFile); err == nil { - return config, nil + + config, err := loader.ClientConfig() + if err != nil { + return nil, nil, err } - return nil, ErrRestConfigFromKubeConfig(err) + return config, loader, nil } // ProcessConfig handles loading, validating, and optionally saving or returning a kubeconfig diff --git a/utils/kubernetes/kubernetes.go b/utils/kubernetes/kubernetes.go index 5454730f..abd5f84e 100644 --- a/utils/kubernetes/kubernetes.go +++ b/utils/kubernetes/kubernetes.go @@ -1,6 +1,7 @@ package kubernetes import ( + "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -10,15 +11,25 @@ type Client struct { RestConfig rest.Config `json:"restconfig,omitempty"` KubeClient *kubernetes.Clientset `json:"kubeclient,omitempty"` DynamicKubeClient dynamic.Interface `json:"dynamicKubeClient,omitempty"` + restClientGetter genericclioptions.RESTClientGetter } func New(kubeconfig []byte) (*Client, error) { - restConfig, err := DetectKubeConfig(kubeconfig) + restConfig, kubeConfigLoader, err := detectKubeConfig(kubeconfig) + if err != nil { + return nil, err + } + + var restClientGetter genericclioptions.RESTClientGetter + if kubeConfigLoader != nil { + restClientGetter = newClientConfigRESTClientGetter(kubeConfigLoader) + } else { + restClientGetter = newRESTConfigRESTClientGetter(restConfig) + } + restConfig, err = restClientGetter.ToRESTConfig() if err != nil { return nil, err } - restConfig.QPS = float32(50) - restConfig.Burst = int(100) // if insecure variable is kept true, allow that if restConfig.TLSClientConfig.Insecure { //nolint:staticcheck @@ -41,5 +52,20 @@ func New(kubeconfig []byte) (*Client, error) { RestConfig: *restConfig, DynamicKubeClient: dyclient, KubeClient: kclient, + restClientGetter: restClientGetter, }, nil } + +func configureRESTConfig(config *rest.Config) { + config.QPS = float32(50) + config.Burst = int(100) +} + +func (c *Client) getRESTClientGetter() genericclioptions.RESTClientGetter { + if c.restClientGetter != nil { + return c.restClientGetter + } + + // Preserve compatibility for clients constructed directly instead of through New. + return newRESTConfigRESTClientGetter(&c.RestConfig) +}