fix: authenticate Helm downloads with image pull secrets

README.md

@@ -188,7 +188,7 @@ or via a profile section in the user-config file.
 ### Deploy to Cluster
 Apply the generated deployment files to your Kubernetes cluster by using --deploy. This phase requires --kubeconfig and can be skipped if --deploy is not specified.
 
-The deploy step installs (or upgrades) the `nvidia/network-operator` Helm chart in-process before applying the post-install CRs. The chart version and Helm repository URL are taken from the embedded release catalog and can be selected via `--network-operator-release <MAJOR.MINOR>`. Each profile renders a per-profile `values.yaml` next to the CR manifests; `l8k deploy` reads that file and runs the install. When a release already exists with different values, deploy fails fast — pass `--overwrite-existing` to promote to `helm upgrade --install`.
+The deploy step installs (or upgrades) the `nvidia/network-operator` Helm chart in-process before applying the post-install CRs. The chart version and Helm repository URL are taken from the embedded release catalog and can be selected via `--network-operator-release <MAJOR.MINOR>`. Each profile renders a per-profile `values.yaml` next to the CR manifests; `l8k deploy` reads that file and runs the install. When `networkOperator.imagePullSecrets` is configured, l8k reads matching Docker credentials from Secrets already present in the operator namespace and uses them for the chart download (including the `nvcr.io` to `helm.ngc.nvidia.com` NGC credential mapping). Secret data remains in memory and is never logged. When a release already exists with different values, deploy fails fast — pass `--overwrite-existing` to promote to `helm upgrade --install`.
 
 Deploy preflight does not treat `SriovNetworkPoolConfig`,
 `SriovNetworkNodePolicy`, or `OVSNetwork` objects labeled with
@@ -243,7 +243,7 @@ Available Commands:
 Common Flags:
       --config-dir string                   Directory containing optional l8k-config.yaml and presets/ overrides
       --enabled-plugins string              Comma-separated list of plugins to enable (default "network-operator")
-      --image-pull-secrets strings          Image pull secret names for Network Operator components (comma-separated)
+      --image-pull-secrets strings          Image pull secret names for Network Operator components and authenticated Helm downloads (comma-separated)
       --kubeconfig string                   Path to kubeconfig file for cluster deployment (required when using --deploy; falls back to $KUBECONFIG, then ~/.kube/config)
       --network-operator-namespace string   Override the network operator namespace from the config file
       --network-operator-release string     Network Operator release line to deploy (MAJOR.MINOR). Selects component image tags + repository from a built-in catalog and drives version-gated template sections. Supported: 26.1, 26.4, 26.7

docs/reference/cli.md

@@ -43,7 +43,7 @@ Run `l8k <command> --help` for the authoritative flag list. Run `l8k schema` for
 | `--node-selector` | Selector persisted for generated resources. It does not filter discovery scheduling. |
 | `--keep-namespace` | Keep the temporary `nvidia-k8s-launch-kit` namespace and daemon workload for inspection. |
 | `--collapse-nic-rails` | Collapse eligible multi-port NICs into one rail. Enabled by default; known dual-port models retain a rail per port. |
-| `--image-pull-secrets` | Secret names used to pull the discovery daemon and propagated into generated policies and Network Operator Helm values. |
+| `--image-pull-secrets` | Secret names used to pull the discovery daemon, propagated into generated policies and Helm values, and reused for authenticated Helm chart downloads when the registry host matches. |
 | `--enabled-plugins` | Comma-separated plugins. The supported deployment plugin is `network-operator`. |
 
 Discovery also accepts the profile and Spectrum-X flags below. Explicit flags override values from `--user-config` and discovered defaults.
@@ -84,7 +84,7 @@ Discovery also accepts the profile and Spectrum-X flags below. Explicit flags ov
 | `--network-namespaces` | Namespaces that receive secondary-network resources and example workloads. |
 | `--workload-manifest` | Replace the profile's example workload with a Pod or workload-controller manifest. |
 | `--enable-doca-driver` | Override `docaDriver.enable` and include the DOCA driver deployment. |
-| `--image-pull-secrets` | Secret names propagated into generated Network Operator policies and Helm values. |
+| `--image-pull-secrets` | Secret names propagated into generated Network Operator policies and Helm values. Matching credentials already present in the operator namespace authenticate the Helm chart download. |
 | `--deploy` | Deploy immediately after generation. |
 | `--kubeconfig` | Kubeconfig used with `--deploy`. |
 | `--dry-run` | Preview the deploy stage used with `--deploy`. |

docs/reference/configuration.md

@@ -60,10 +60,20 @@ The release line fills Network Operator versions, component image tags, DOCA dri
 | `operatorRepository` | Registry path for the Network Operator controller image. |
 | `helmRepoURL` | Chart repository used by `l8k deploy`. Empty means Helm phase 0 is skipped. |
 | `namespace` | Namespace for the Helm release and namespaced Network Operator resources. |
-| `imagePullSecrets` | Secret names propagated into the discovery daemon, generated policies, and Helm values for the Network Operator and enabled subcharts. |
+| `imagePullSecrets` | Secret names propagated into the discovery daemon, generated policies, and Helm values for the Network Operator and enabled subcharts. During deploy, matching credentials also authenticate the Helm chart download. |
 
 When `selectedRelease` is set, catalog values replace explicit version and repository fields so the cohort remains consistent.
 
+For an authenticated Helm repository, each referenced Secret must already
+exist in `networkOperator.namespace` before `l8k deploy` starts, and the
+kubeconfig must allow `get` on Secrets there. l8k reads
+`kubernetes.io/dockerconfigjson` and legacy `kubernetes.io/dockercfg` data in
+memory and never logs or persists the credential. Credentials are sent only
+when the Docker registry host exactly matches the Helm repository host. The
+one explicit cross-host mapping is NGC: `nvcr.io` credentials use the same
+`$oauthtoken` and API key required by `helm.ngc.nvidia.com`. Unrelated registry
+credentials are never forwarded to the chart server.
+
 ## Network And Workload Namespaces
 
 ```yaml

docs/user/quick-start.md

@@ -56,6 +56,12 @@ Deploy runs in phases:
 3. Apply per-group `NicNodePolicy` resources and wait for readiness.
 4. Apply the remaining CRs and verify each one reaches a terminal state.
 
+If `networkOperator.imagePullSecrets` is configured, create the named Secret
+in `networkOperator.namespace` before deploying. Phase 0 reads compatible
+Docker credentials through the Kubernetes API to authenticate the chart
+download; the kubeconfig therefore needs `get secrets` permission in that
+namespace.
+
 Preview the server-side apply without persisting resources:
 
 ```bash

docs/user/troubleshooting.md

@@ -12,6 +12,7 @@ Validation is the normal deployment acceptance stage. When it does not produce a
 | Failure area | First checks |
 | --- | --- |
 | Discovery | Bootstrap pod status, events, image pulls, and `NicDevice` publication in `nvidia-k8s-launch-kit`. |
+| Helm chart download | For an HTTP 401 or image-pull-Secret error, verify each configured Secret exists in `networkOperator.namespace`, the kubeconfig can read it, and its Docker config has credentials for the Helm host or `nvcr.io` for an NGC Helm repository. |
 | Deploy preflight | Existing Helm values, generated values, stray custom resources, and whether overwrite was explicitly intended. |
 | Reconciliation | `NicClusterPolicy`, `NicNodePolicy`, and component pod conditions in the Network Operator namespace. |
 | SR-IOV | `SriovNetworkNodeState` sync status, VF totals, node selectors, and advertised resources. |

pkg/cmd/discover.go

@@ -141,7 +141,7 @@ func init() {
 		fmt.Sprintf("Network Operator release line to deploy (MAJOR.MINOR). Supported: %s",
 			strings.Join(releases.SupportedReleases(), ", ")))
 	discoverCmd.Flags().StringVar(&nodeSelector, "node-selector", "feature.node.kubernetes.io/pci-15b3.present=true", "Node selector written into the saved cluster-config (used at deploy time). Does NOT gate discovery scheduling — the daemon runs on all nodes and NIC nodes are detected via a sysfs PCI-vendor probe")
-	discoverCmd.Flags().StringSliceVar(&imagePullSecrets, "image-pull-secrets", nil, "Image pull secret names for Network Operator components (comma-separated)")
+	discoverCmd.Flags().StringSliceVar(&imagePullSecrets, "image-pull-secrets", nil, "Image pull secret names for Network Operator components and authenticated Helm downloads (comma-separated)")
 	discoverCmd.Flags().StringVar(&enabledPlugins, "enabled-plugins", "network-operator", "Comma-separated list of plugins to enable")
 	discoverCmd.Flags().BoolVar(&keepNamespace, "keep-namespace", false, "Skip teardown of the nvidia-k8s-launch-kit namespace (for debugging)")
 	discoverCmd.Flags().BoolVar(&collapseNicRails, "collapse-nic-rails", true, collapseNicRailsFlagHelp)

pkg/cmd/generate.go

@@ -220,7 +220,7 @@ func init() {
 	generateCmd.Flags().StringVar(&networkOperatorRelease, "network-operator-release", "",
 		fmt.Sprintf("Network Operator release line to deploy (MAJOR.MINOR). Supported: %s",
 			strings.Join(releases.SupportedReleases(), ", ")))
-	generateCmd.Flags().StringSliceVar(&imagePullSecrets, "image-pull-secrets", nil, "Image pull secret names for Network Operator components (comma-separated)")
+	generateCmd.Flags().StringSliceVar(&imagePullSecrets, "image-pull-secrets", nil, "Image pull secret names for Network Operator components and authenticated Helm downloads (comma-separated)")
 	generateCmd.Flags().StringVar(&enabledPlugins, "enabled-plugins", "network-operator", "Comma-separated list of plugins to enable")
 
 	// Deploy (optional)

pkg/cmd/root.go

@@ -278,7 +278,7 @@ func init() {
 	rootCmd.Flags().StringVar(&nodeSelector, "node-selector", "feature.node.kubernetes.io/pci-15b3.present=true", "Node selector written into the saved cluster-config (used at deploy time). Does NOT gate discovery scheduling — the daemon runs on all nodes and NIC nodes are detected via a sysfs PCI-vendor probe")
 	rootCmd.Flags().BoolVar(&collapseNicRails, "collapse-nic-rails", true, collapseNicRailsFlagHelp)
 	rootCmd.Flags().StringVar(&forPreset, "for", "", forFlagHelp())
-	rootCmd.Flags().StringSliceVar(&imagePullSecrets, "image-pull-secrets", nil, "Image pull secret names for Network Operator components (comma-separated)")
+	rootCmd.Flags().StringSliceVar(&imagePullSecrets, "image-pull-secrets", nil, "Image pull secret names for Network Operator components and authenticated Helm downloads (comma-separated)")
 	rootCmd.Flags().StringVar(&saveDeploymentFiles, "save-deployment-files", "./deployment", "Save generated deployment files to the specified directory")
 	rootCmd.Flags().StringSliceVar(&networkNamespaces, "network-namespaces", nil, "Comma-separated namespaces for the secondary-network CRs and example test DaemonSets. One independent copy is rendered per namespace (shared resources like IPPools and NodePolicies are NOT duplicated). Overrides config networkNamespaces; default: 'default'.")
 	rootCmd.Flags().BoolVar(&enableDocaDriver, "enable-doca-driver", false, "Enable DOCA driver deployment (overrides config file docaDriver.enable)")

pkg/cmd/schema.go

@@ -219,7 +219,7 @@ var schemaCmd = &cobra.Command{
 				},
 				"--image-pull-secrets": {
 					Type:        "[]string",
-					Description: "Image pull secret names for Network Operator components (comma-separated)",
+					Description: "Image pull secret names for Network Operator components and authenticated Helm downloads (comma-separated)",
 				},
 				"--network-operator-release": {
 					Type:        "string",

pkg/config/default-config.yaml

@@ -25,6 +25,11 @@ networkOperator:
   # `helm.ngc.nvidia.com/nvidia`; staging under `helm.ngc.nvidia.com/nvstaging/mellanox`.
   helmRepoURL: https://helm.ngc.nvidia.com/nvidia
   namespace: nvidia-network-operator
+  # imagePullSecrets are rendered into every enabled Network Operator
+  # component. During `l8k deploy`, a matching docker-registry Secret that
+  # already exists in this namespace is also used to authenticate the Helm
+  # chart download. For NGC, nvcr.io credentials are valid for
+  # helm.ngc.nvidia.com. Secret data is read in memory and is never logged.
   # imagePullSecrets:
   #   - my-registry-secret
 

pkg/networkoperatorplugin/helm.go

@@ -134,12 +134,21 @@ func InstallOrUpgrade(
 	}
 
 	loadChart := func() (*chart.Chart, error) {
-		chartPath, cleanup, perr := pullChart(ctx, cfg.HelmRepoURL, networkOperatorChartName, chartVersion)
+		credentials, cerr := loadHelmRepositoryCredentials(ctx, restConfig, cfg, namespace)
+		if cerr != nil {
+			return nil, pkgerrors.NewDeploymentError(
+				"failed to read configured image pull secrets for Helm repository authentication",
+				cerr,
+				fmt.Sprintf("ensure each networkOperator.imagePullSecrets Secret exists in namespace %s and the kubeconfig can get secrets", namespace),
+			)
+		}
+		chartPath, cleanup, perr := pullChart(
+			ctx, cfg.HelmRepoURL, networkOperatorChartName, chartVersion, credentials)
 		if perr != nil {
 			return nil, pkgerrors.NewDeploymentError(
 				fmt.Sprintf("failed to fetch network-operator chart %s from %s", chartVersion, cfg.HelmRepoURL),
 				perr,
-				"verify the helm repository URL and that the chart version exists",
+				"verify the helm repository URL, chart version, and image pull secret credentials",
 			)
 		}
 		defer cleanup()
@@ -302,7 +311,33 @@ func runUpgrade(
 // pullChart fetches a chart tarball from repoURL into a temp directory and
 // returns the local path plus a cleanup func. Uses helm's downloader
 // directly — no repo cache, no `helm repo add` side effects.
-func pullChart(_ context.Context, repoURL, chartName, chartVersion string) (string, func(), error) {
+func pullChart(
+	ctx context.Context,
+	repoURL, chartName, chartVersion string,
+	credentials []helmRepositoryCredential,
+) (string, func(), error) {
+	if len(credentials) == 0 {
+		return pullChartWithCredential(repoURL, chartName, chartVersion, helmRepositoryCredential{})
+	}
+
+	var errs []error
+	for _, credential := range credentials {
+		if err := ctx.Err(); err != nil {
+			return "", func() {}, err
+		}
+		saved, cleanup, err := pullChartWithCredential(repoURL, chartName, chartVersion, credential)
+		if err == nil {
+			return saved, cleanup, nil
+		}
+		errs = append(errs, fmt.Errorf("credentials from Secret %q: %w", credential.SourceSecret, err))
+	}
+	return "", func() {}, errors.Join(errs...)
+}
+
+func pullChartWithCredential(
+	repoURL, chartName, chartVersion string,
+	credential helmRepositoryCredential,
+) (string, func(), error) {
 	tmpDir, err := os.MkdirTemp("", "l8k-helm-chart-*")
 	if err != nil {
 		return "", func() {}, fmt.Errorf("create temp dir for chart pull: %w", err)
@@ -316,8 +351,10 @@ func pullChart(_ context.Context, repoURL, chartName, chartVersion string) (stri
 	settings.RepositoryCache = tmpDir
 
 	getters := getter.All(settings)
-	chartURL, err := repo.FindChartInRepoURL(
+	chartURL, err := repo.FindChartInAuthRepoURL(
 		repoURL,
+		credential.Username,
+		credential.Password,
 		chartName,
 		chartVersion,
 		"", // certFile
@@ -337,6 +374,16 @@ func pullChart(_ context.Context, repoURL, chartName, chartVersion string) (stri
 		RepositoryConfig: settings.RepositoryConfig,
 		RepositoryCache:  settings.RepositoryCache,
 	}
+	// An index may point at a chart archive on another host. Do not forward
+	// the repository credential across that boundary; getter's URL guard is
+	// relative to chartURL at this stage, not the original repository URL.
+	if credential.Username != "" && credential.Password != "" &&
+		sameRegistryHost(repoURL, chartURL) {
+		dl.Options = append(dl.Options,
+			getter.WithBasicAuth(credential.Username, credential.Password),
+			getter.WithPassCredentialsAll(false),
+		)
+	}
 	saved, _, err := dl.DownloadTo(chartURL, chartVersion, tmpDir)
 	if err != nil {
 		cleanup()
@@ -345,6 +392,12 @@ func pullChart(_ context.Context, repoURL, chartName, chartVersion string) (stri
 	return saved, cleanup, nil
 }
 
+func sameRegistryHost(left, right string) bool {
+	leftHost, leftErr := registryHost(left)
+	rightHost, rightErr := registryHost(right)
+	return leftErr == nil && rightErr == nil && leftHost == rightHost
+}
+
 // isPendingStatus reports whether the helm release is mid-operation
 // (install/upgrade/rollback). These are the statuses that cause helm's
 // "another operation in progress" lock — usually a sign that a previous