Add PathPattern (directory discovery) to ArtifactGenerator

api/v1beta1/artifactgenerator_types.go

@@ -54,6 +54,7 @@ type CommonMetadata struct {
 }
 
 // ArtifactGeneratorSpec defines the desired state of ArtifactGenerator.
+// +kubebuilder:validation:XValidation:rule="has(self.pathPattern) && size(self.pathPattern) > 0 || self.artifacts.all(a, a.name.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$'))",message="artifact names must be valid Kubernetes object names when pathPattern is not set"
 type ArtifactGeneratorSpec struct {
 	// CommonMetadata specifies the common labels and annotations that are
 	// applied to all resources. Any existing label or annotation will be
@@ -68,6 +69,14 @@ type ArtifactGeneratorSpec struct {
 	// +required
 	Sources []SourceReference `json:"sources"`
 
+	// PathPattern specifies a directory traversal pattern to match within the sources.
+	// The format is "@<alias>/<pattern>". Named captures in the pattern (e.g. "{app}")
+	// can be used as placeholders in OutputArtifacts fields.
+	// +kubebuilder:validation:Pattern="^@([a-z0-9]([a-z0-9_-]*[a-z0-9])?)/(.*)$"
+	// +kubebuilder:validation:MaxLength=1024
+	// +optional
+	PathPattern string `json:"pathPattern,omitempty"`
+
 	// OutputArtifacts is a list of output artifacts to be generated.
 	// +kubebuilder:validation:MinItems=1
 	// +kubebuilder:validation:MaxItems=1000
@@ -110,7 +119,8 @@ type SourceReference struct {
 // generated by the ArtifactGenerator.
 type OutputArtifact struct {
 	// Name is the name of the generated artifact.
-	// +kubebuilder:validation:Pattern="^[a-z0-9]([a-z0-9-]*[a-z0-9])?$"
+	// When pathPattern is set, this field may use capture placeholders such as "{app}".
+	// +kubebuilder:validation:MinLength=1
 	// +kubebuilder:validation:MaxLength=253
 	// +required
 	Name string `json:"name"`
@@ -144,16 +154,20 @@ type OutputArtifact struct {
 
 type CopyOperation struct {
 	// From specifies the source (by alias) and the glob pattern to match files.
-	// The format is "@<alias>/<glob-pattern>".
+	// The format is "@<alias>/<glob-pattern>". When pathPattern is set,
+	// the path may use capture placeholders such as "{app}".
 	// +kubebuilder:validation:Pattern="^@([a-z0-9]([a-z0-9_-]*[a-z0-9])?)/(.*)$"
+	// +kubebuilder:validation:MinLength=1
 	// +kubebuilder:validation:MaxLength=1024
 	// +required
 	From string `json:"from"`
 
 	// To specifies the destination path within the artifact.
 	// The format is "@artifact/path", the alias "artifact"
-	// refers to the root path of the generated artifact.
+	// refers to the root path of the generated artifact. When pathPattern
+	// is set, the path may use capture placeholders such as "{app}".
 	// +kubebuilder:validation:Pattern="^@(artifact)/(.*)$"
+	// +kubebuilder:validation:MinLength=1
 	// +kubebuilder:validation:MaxLength=1024
 	// +required
 	To string `json:"to"`

config/crd/bases/source.extensions.fluxcd.io_artifactgenerators.yaml

@@ -79,8 +79,10 @@ spec:
                           from:
                             description: |-
                               From specifies the source (by alias) and the glob pattern to match files.
-                              The format is "@<alias>/<glob-pattern>".
+                              The format is "@<alias>/<glob-pattern>". When pathPattern is set,
+                              the path may use capture placeholders such as "{app}".
                             maxLength: 1024
+                            minLength: 1
                             pattern: ^@([a-z0-9]([a-z0-9_-]*[a-z0-9])?)/(.*)$
                             type: string
                           strategy:
@@ -101,8 +103,10 @@ spec:
                             description: |-
                               To specifies the destination path within the artifact.
                               The format is "@artifact/path", the alias "artifact"
-                              refers to the root path of the generated artifact.
+                              refers to the root path of the generated artifact. When pathPattern
+                              is set, the path may use capture placeholders such as "{app}".
                             maxLength: 1024
+                            minLength: 1
                             pattern: ^@(artifact)/(.*)$
                             type: string
                         required:
@@ -112,9 +116,11 @@ spec:
                       minItems: 1
                       type: array
                     name:
-                      description: Name is the name of the generated artifact.
+                      description: |-
+                        Name is the name of the generated artifact.
+                        When pathPattern is set, this field may use capture placeholders such as "{app}".
                       maxLength: 253
-                      pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$
+                      minLength: 1
                       type: string
                     originRevision:
                       description: |-
@@ -159,6 +165,14 @@ spec:
                     description: Labels to be added to the object's metadata.
                     type: object
                 type: object
+              pathPattern:
+                description: |-
+                  PathPattern specifies a directory traversal pattern to match within the sources.
+                  The format is "@<alias>/<pattern>". Named captures in the pattern (e.g. "{app}")
+                  can be used as placeholders in OutputArtifacts fields.
+                maxLength: 1024
+                pattern: ^@([a-z0-9]([a-z0-9_-]*[a-z0-9])?)/(.*)$
+                type: string
               sources:
                 description: |-
                   Sources is a list of references to the Flux source-controller
@@ -210,6 +224,11 @@ spec:
             - artifacts
             - sources
             type: object
+            x-kubernetes-validations:
+            - message: artifact names must be valid Kubernetes object names when pathPattern
+                is not set
+              rule: has(self.pathPattern) && size(self.pathPattern) > 0 || self.artifacts.all(a,
+                a.name.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$'))
           status:
             description: ArtifactGeneratorStatus defines the observed state of ArtifactGenerator.
             properties:

docs/spec/v1beta1/artifactgenerators.md

@@ -227,13 +227,60 @@ spec:
 Sources are watched for changes, and when any source is updated, the controller will
 regenerate the affected artifacts automatically.
 
+### Path Pattern (Directory Discovery)
+
+The `.spec.pathPattern` field allows for dynamic, path-based discovery of artifacts. When specified, the controller will scan the referenced source for directories matching the pattern and dynamically generate one ExternalArtifact for each matched directory.
+
+- `pathPattern` (optional): Specifies a directory traversal pattern within a source in the format `@<alias>/<pattern>`.
+  Named captures in the pattern (e.g., `{app}`, `{env}`) can be used as placeholders in the `artifacts` fields.
+
+When `pathPattern` is used, the generated ExternalArtifacts will automatically have their labels populated with the extracted capture variables.
+
+```yaml
+spec:
+  sources:
+    - alias: monorepo
+      kind: GitRepository
+      name: my-monorepo
+  pathPattern: "@monorepo/apps/{app}/envs/{env}"
+  artifacts:
+    - name: "{app}-{env}"
+      copy:
+        - from: "@monorepo/apps/{app}/envs/{env}/**"
+          to: "@artifact/"
+```
+
+#### Directory Name Constraints
+
+Directory names matched by path pattern captures must comply with
+[Kubernetes label value restrictions](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-semantics):
+
+- Must be 63 characters or fewer
+- Must begin and end with an alphanumeric character (`[a-zA-Z0-9]`)
+- May contain dashes (`-`), underscores (`_`), dots (`.`), and alphanumeric characters
+
+The controller automatically lowercases captured values before using them
+in artifact names and labels. This means a directory named `Auth` will produce
+an artifact with `app=auth`. Copy expressions receive the original captured
+path values so source paths preserve their case.
+
+**Note:** If two directories differ only in case (e.g., `apps/Auth/` and `apps/auth/`),
+they will resolve to the same artifact name after lowercasing. In this case, the
+controller will stall the reconciliation with an error indicating the collision.
+
+If any captured directory name does not conform to the label value restrictions
+(e.g., names starting with a dot like `.hidden`, containing spaces, or exceeding
+63 characters), the reconciliation will fail with a terminal error that includes
+the `pathPattern` and the invalid value.
+
 ### Artifacts
 
 The `.spec.artifacts` field defines the list of ExternalArtifacts to be generated from the sources.
+When `pathPattern` is defined, the artifacts act as templates for each matched directory.
 Each artifact must specify:
 
 - `name` (required): The name of the generated ExternalArtifact resource. It must be unique in the context
-  of the ArtifactGenerator and must conform to Kubernetes resource naming conventions.
+  of the ArtifactGenerator and must conform to Kubernetes resource naming conventions. Supports capture placeholders if `pathPattern` is used.
 - `copy` (required): A list of copy operations to perform from sources to the artifact.
 - `revision` (optional): A specific source revision to use in the format `@alias`.
   If not specified, the revision is automatically computed as `latest@<digest>` based on the artifact content.
@@ -262,10 +309,12 @@ spec:
 
 Each copy operation specifies how to copy files from sources into the generated artifact:
 
-- `from`: Source path in the format `@alias/pattern` where `alias` references 
+- `from`: Source path in the format `@alias/pattern` where `alias` references
   a source and `pattern` is a glob pattern or a specific file/directory path within that source.
+  When `pathPattern` is set, this field may use capture placeholders and must render to this format.
 - `to`: Destination path in the format `@artifact/path` where `artifact` is
   the root of the generated artifact and `path` is the relative path to a file or directory.
+  When `pathPattern` is set, this field may use capture placeholders and must render to this format.
 - `exclude` (optional): A list of glob patterns to filter out from the source selection.
   Any file matched by `from` that also matches an exclude pattern will be ignored.
   Patterns are matched against paths relative to the source alias root or to the

internal/controller/artifactgenerator_controller.go

@@ -193,14 +193,33 @@ func (r *ArtifactGeneratorReconciler) reconcile(ctx context.Context,
 		return ctrl.Result{RequeueAfter: r.DependencyRequeueInterval}, nil
 	}
 
-	// Prepare a slice to hold the references to the created ExternalArtifact objects.
-	eaRefs := make([]swapi.ExternalArtifactReference, 0, len(obj.Spec.OutputArtifacts))
-
 	// Build the artifacts and reconcile the ExternalArtifact objects.
 	// The artifacts will be stored in the storage under the following path:
 	// <storage-root>/<kind>/<namespace>/<name>/<contents-hash>.tar.gz
 	artifactBuilder := builder.New(r.Storage)
-	for _, oa := range obj.Spec.OutputArtifacts {
+
+	reqs, err := buildArtifactRequests(obj, localSources)
+	if err != nil {
+		msg := fmt.Sprintf("failed to expand path pattern: %s", err.Error())
+		if isTerminalPathPatternError(err) {
+			return ctrl.Result{}, r.newTerminalErrorFor(obj,
+				gotkmeta.BuildFailedReason,
+				"%s", msg)
+		}
+		gotkconditions.Delete(obj, gotkmeta.StalledCondition)
+		gotkconditions.MarkFalse(obj,
+			gotkmeta.ReadyCondition,
+			gotkmeta.BuildFailedReason,
+			"%s", msg)
+		r.Event(obj, corev1.EventTypeWarning, gotkmeta.BuildFailedReason, msg)
+		return ctrl.Result{}, err
+	}
+
+	// Prepare a slice to hold the references to the created ExternalArtifact objects.
+	eaRefs := make([]swapi.ExternalArtifactReference, 0, len(reqs))
+
+	for _, req := range reqs {
+		oa := req.OutputArtifact
 		// Build the artifact using the local sources.
 		artifact, err := artifactBuilder.Build(ctx, &oa, localSources, obj.Namespace, tmpDir)
 		if err != nil {
@@ -219,7 +238,7 @@ func (r *ArtifactGeneratorReconciler) reconcile(ctx context.Context,
 		// Reconcile the ExternalArtifact corresponding to the built artifact.
 		// The ExternalArtifact will reference the artifact stored in the storage backend.
 		// If the ExternalArtifact already exists, its status will be updated with the new artifact details.
-		eaRef, err := r.reconcileExternalArtifact(ctx, obj, &oa, artifact)
+		eaRef, err := r.reconcileExternalArtifact(ctx, obj, &oa, artifact, req.Labels)
 		if err != nil {
 			msg := fmt.Sprintf("%s reconcile failed: %s", oa.Name, err.Error())
 			gotkconditions.MarkFalse(obj,
@@ -423,7 +442,8 @@ func (r *ArtifactGeneratorReconciler) fetchSources(ctx context.Context,
 func (r *ArtifactGeneratorReconciler) reconcileExternalArtifact(ctx context.Context,
 	obj *swapi.ArtifactGenerator,
 	outputArtifact *swapi.OutputArtifact,
-	artifact *gotkmeta.Artifact) (*swapi.ExternalArtifactReference, error) {
+	artifact *gotkmeta.Artifact,
+	dynamicLabels map[string]string) (*swapi.ExternalArtifactReference, error) {
 	log := ctrl.LoggerFrom(ctx)
 
 	// Prepare labels for the ExternalArtifact with the managed-by and generator labels.
@@ -438,6 +458,8 @@ func (r *ArtifactGeneratorReconciler) reconcileExternalArtifact(ctx context.Cont
 		}
 	}
 
+	maps.Copy(labels, dynamicLabels)
+
 	labels["app.kubernetes.io/managed-by"] = r.ControllerName
 	labels[swapi.ArtifactGeneratorLabel] = string(obj.GetUID())