From b5ea3279a1aeeea7c5918af7f0aa072699c034da Mon Sep 17 00:00:00 2001 From: moosebay Date: Mon, 9 Feb 2026 10:03:35 +0300 Subject: [PATCH] feat: add cloud secret manager support for expression interpolation Implements infrastructure for resolving cloud secrets in {{}} template expressions (Issue #23). Adds GCP Secret Manager support with #gcp: prefix and optional #fragment JSON field extraction. New packages: - secretresolver: SecretResolver interface, ParseSecretRef, ExtractFragment - secretresolver/gcpsecret: GCP Secret Manager implementation with caching Expression package changes: - Add #gcp:, #aws:, #azure: prefix recognition to resolveVar() dispatch - Thread context.Context through interpolation for network calls - Add SecretReferenceError type with provider/ref/fragment fields - Add WithSecretResolver() builder on UnifiedEnv - Mask secret values as "***" in variable tracking Wiring: - Add SecretResolver field to FlowNodeRequest - Pass resolver through FlowLocalRunner -> FlowNodeRequest -> nodes - Wire into request node (sync + async) and AI node - Use variadic parameter in PrepareHTTPRequestWithTracking for backward-compatible secret resolver injection Tests: - Unit tests for fragment extraction, reference parsing, multi-resolver - Expression tests with mock SecretResolver (GCP, AWS, Azure prefixes) - Integration test skeleton behind gcp_integration build tag Note: go.mod adds cloud.google.com/go/secretmanager dependency. --- PLAN-issue-23.md | 293 ++++++++++++++++++ packages/server/go.mod | 1 + packages/server/pkg/expression/errors.go | 23 ++ packages/server/pkg/expression/file_utils.go | 38 +++ packages/server/pkg/expression/interpolate.go | 57 +++- packages/server/pkg/expression/unified_env.go | 26 +- .../server/pkg/expression/unified_env_test.go | 235 ++++++++++++++ packages/server/pkg/flow/node/nai/nai.go | 3 + packages/server/pkg/flow/node/node.go | 19 +- .../server/pkg/flow/node/nrequest/nrequest.go | 4 +- .../runner/flowlocalrunner/flowlocalrunner.go | 19 +- packages/server/pkg/http/request/request.go | 6 + .../server/pkg/secretresolver/fragment.go | 36 +++ .../pkg/secretresolver/fragment_test.go | 84 +++++ .../pkg/secretresolver/gcpsecret/gcp.go | 95 ++++++ .../gcpsecret/integration_gcp_test.go | 58 ++++ packages/server/pkg/secretresolver/multi.go | 35 +++ .../server/pkg/secretresolver/multi_test.go | 56 ++++ packages/server/pkg/secretresolver/parse.go | 14 + .../server/pkg/secretresolver/parse_test.go | 55 ++++ .../server/pkg/secretresolver/resolver.go | 13 + 21 files changed, 1143 insertions(+), 27 deletions(-) create mode 100644 PLAN-issue-23.md create mode 100644 packages/server/pkg/secretresolver/fragment.go create mode 100644 packages/server/pkg/secretresolver/fragment_test.go create mode 100644 packages/server/pkg/secretresolver/gcpsecret/gcp.go create mode 100644 packages/server/pkg/secretresolver/gcpsecret/integration_gcp_test.go create mode 100644 packages/server/pkg/secretresolver/multi.go create mode 100644 packages/server/pkg/secretresolver/multi_test.go create mode 100644 packages/server/pkg/secretresolver/parse.go create mode 100644 packages/server/pkg/secretresolver/parse_test.go create mode 100644 packages/server/pkg/secretresolver/resolver.go diff --git a/PLAN-issue-23.md b/PLAN-issue-23.md new file mode 100644 index 00000000..71342430 --- /dev/null +++ b/PLAN-issue-23.md @@ -0,0 +1,293 @@ +# Implementation Plan: Cloud Secret Manager Support (Issue #23) + +## Problem + +Users need to load secrets directly from cloud secret managers using a syntax +similar to the existing `{{#env:VAR_NAME}}` support. Currently there is no way +to reference secrets stored in GCP, AWS, or Azure from within request templates +or flow variables. + +## Proposed Syntax + +``` +GCP: {{#gcp:projects/p/secrets/oauth/versions/latest#client_secret}} +AWS: {{#aws:secret-name#client_secret}} (future) +Azure: {{#azure:vault/secret-name#client_secret}} (future) +``` + +The optional `#fragment` suffix is a JSON field selector — if the secret value +is a JSON blob, the fragment extracts a specific key from it. Without a +fragment, the entire raw secret value is returned. + +--- + +## Current Architecture + +The interpolation engine lives in `packages/server/pkg/expression/interpolate.go`. +The `resolveVar()` method dispatches on prefix: + +``` +"#env:" → resolveEnvVar() — os.LookupEnv +"#file:" → resolveFileVar() — os.ReadFile +default → resolveExprVar() — expr-lang evaluation +``` + +Key types: `UnifiedEnv` (environment), `InterpolationResult`, error types +(`EnvReferenceError`, `FileReferenceError`, `InterpolationError`). + +The `UnifiedEnv` uses a builder pattern for optional capabilities: +`WithTracking(tracker)`, `WithFunc(name, fn)`. + +--- + +## Architecture Decisions + +### 1. Provider Interface + Injection (not direct imports) + +The `expression` package should **not** import cloud SDKs directly. This would +force every consumer (including the CLI) to pull in heavy GCP/AWS dependencies +even when cloud secrets are not used. + +Instead, define a `SecretResolver` interface injected into `UnifiedEnv` via a +`WithSecretResolver()` builder method — mirroring the `WithTracking()` pattern. + +```go +// SecretResolver resolves cloud secret manager references. +type SecretResolver interface { + ResolveSecret(ctx context.Context, provider, ref, fragment string) (string, error) +} +``` + +### 2. Package Structure + +``` +packages/server/pkg/secretresolver/ +├── resolver.go # SecretResolver interface +├── parse.go # ParseSecretRef(ref) → (path, fragment) +├── fragment.go # ExtractFragment(value, fragment) → string +├── multi.go # MultiResolver — dispatches by provider name +└── gcpsecret/ + ├── gcp.go # GCP Secret Manager implementation + └── integration_gcp_test.go # Integration test (build tag: gcp_integration) +``` + +The `expression` package imports only the interface from `secretresolver/`. +The concrete GCP implementation lives in `gcpsecret/` and is wired at startup. + +### 3. Context Propagation + +Currently `InterpolateCtx` accepts `context.Context` but the comment says +_"reserved for future use"_. Cloud secret resolution requires context for +network calls. This is the right time to thread context through the full chain: + +- `InterpolateWithResultCtx(ctx, raw)` → `resolveVar(ctx, ...)` → `resolveSecretVar(ctx, ...)` +- Keep context-free `Interpolate()` / `InterpolateWithResult()` as convenience + wrappers using `context.Background()` for backward compatibility. + +### 4. Caching + +Secrets are cached per-`GCPResolver` instance with a configurable TTL +(default: 5 minutes). Cache is keyed by `ref#fragment`. This avoids redundant +API calls when the same secret is referenced multiple times in a flow execution. + +--- + +## File-by-File Changes + +### New Files + +| File | Purpose | +| ---------------------------------------------------------------------- | ------------------------------------------------------------------------- | +| `packages/server/pkg/secretresolver/resolver.go` | `SecretResolver` interface | +| `packages/server/pkg/secretresolver/parse.go` | `ParseSecretRef(ref) → (path, fragment)` using `strings.LastIndex("#")` | +| `packages/server/pkg/secretresolver/fragment.go` | `ExtractFragment(value, fragment)` — JSON field extraction | +| `packages/server/pkg/secretresolver/fragment_test.go` | Unit tests for fragment extraction | +| `packages/server/pkg/secretresolver/parse_test.go` | Unit tests for reference parsing | +| `packages/server/pkg/secretresolver/multi.go` | `MultiResolver` — provider dispatcher with `Register(provider, resolver)` | +| `packages/server/pkg/secretresolver/gcpsecret/gcp.go` | GCP implementation using `cloud.google.com/go/secretmanager/apiv1` | +| `packages/server/pkg/secretresolver/gcpsecret/integration_gcp_test.go` | Integration test behind `gcp_integration` build tag | + +### Modified Files + +| File | Changes | +| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| `packages/server/pkg/expression/file_utils.go` | Add `GCPRefPrefix`, `AWSRefPrefix`, `AzureRefPrefix` constants; `IsSecretReference()`, `ParseSecretReference()` helpers | +| `packages/server/pkg/expression/errors.go` | Add `SecretReferenceError` type (mirrors `EnvReferenceError`) | +| `packages/server/pkg/expression/unified_env.go` | Add `secretResolver` field to `UnifiedEnv`; `WithSecretResolver()` builder; update `Clone()` | +| `packages/server/pkg/expression/interpolate.go` | Thread `context.Context` through `resolveVar()`; add `isSecretReference` case; add `resolveSecretVar()` method | +| `packages/server/pkg/expression/unified_env_test.go` | Add tests with mock `SecretResolver` | +| `packages/server/go.mod` | Add `cloud.google.com/go/secretmanager` dependency | + +### Wiring Points (where resolver gets injected) + +| Location | Change | +| --------------------- | ----------------------------------------------------------------------- | +| Flow builder (server) | Call `.WithSecretResolver(resolver)` when constructing `UnifiedEnv` | +| CLI flow command | Optionally create `GCPResolver` at startup, register in `MultiResolver` | + +--- + +## Key Implementation Details + +### Fragment Extraction + +``` +Input: {{#gcp:projects/my-proj/secrets/oauth-creds/versions/latest#client_secret}} + ^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + prefix resource path fragment + +1. Strip prefix "#gcp:" → "projects/my-proj/secrets/oauth-creds/versions/latest#client_secret" +2. ParseSecretRef (split on last '#') → path, fragment +3. GCP client fetches secret → '{"client_id":"abc","client_secret":"xyz"}' +4. ExtractFragment(raw, "client_secret") → "xyz" +``` + +If no fragment, the entire raw value is returned as-is. + +### Secret Value Masking in Tracking + +When the tracker is enabled, secret reads are recorded with `"***"` instead of +the actual value to prevent secret leakage in flow execution logs, the UI +variable inspector, or debug output. + +```go +if e.tracker != nil { + e.tracker.TrackRead(varRef, "***") // Never log actual secret +} +``` + +### Error Handling + +New error type follows existing patterns: + +```go +type SecretReferenceError struct { + Provider string // "gcp", "aws", "azure" + Ref string // resource path + Fragment string // optional JSON fragment key + Cause error +} +``` + +Error scenarios: + +1. **No resolver configured** — user writes `{{#gcp:...}}` without wiring a resolver +2. **Empty path** — `{{#gcp:}}` +3. **GCP API error** — permission denied, secret not found, network timeout +4. **Fragment extraction failure** — value is not JSON, or key not found +5. **Unsupported provider** — `{{#aws:...}}` when only GCP is registered + +### GCP Resolver Options + +```go +resolver, err := gcpsecret.NewGCPResolver(ctx, + gcpsecret.WithCacheTTL(5 * time.Minute), +) +``` + +Uses Application Default Credentials (ADC). No API keys accepted as parameters. + +--- + +## Testing Strategy + +### Unit Tests (no cloud access, no build tags) + +**Mock resolver for expression tests:** + +```go +type mockSecretResolver struct { + secrets map[string]string + err error +} + +func (m *mockSecretResolver) ResolveSecret(ctx context.Context, provider, ref, fragment string) (string, error) { + if m.err != nil { return "", m.err } + key := provider + ":" + ref + "#" + fragment + val, ok := m.secrets[key] + if !ok { return "", fmt.Errorf("secret not found") } + return val, nil +} +``` + +**Test cases:** + +- `TestInterpolate_GCPSecret_SimpleValue` — raw value resolution +- `TestInterpolate_GCPSecret_WithFragment` — JSON field extraction +- `TestInterpolate_GCPSecret_NoResolver` — clear error +- `TestInterpolate_GCPSecret_EmptyPath` — `ErrEmptyPath` +- `TestInterpolate_GCPSecret_MixedReferences` — `#env:` + `#gcp:` in same string +- `TestInterpolate_GCPSecret_TrackedAsMasked` — tracker records `"***"` +- `TestParseSecretRef_*` — path/fragment splitting +- `TestExtractFragment_*` — JSON extraction edge cases + +### Integration Tests (behind build tag) + +```go +//go:build gcp_integration + +// Guard: RUN_GCP_INTEGRATION_TESTS=true +// Env: GCP_TEST_SECRET_NAME=projects/my-proj/secrets/test/versions/latest +``` + +Run: `RUN_GCP_INTEGRATION_TESTS=true go test -tags gcp_integration -v ./packages/server/pkg/secretresolver/gcpsecret/` + +--- + +## Dependency Impact + +**`packages/server/go.mod`**: Add `cloud.google.com/go/secretmanager`. The +server already has `cloud.google.com/go` and auth packages as transitive +dependencies from Vertex AI, so incremental cost is minimal. + +**`apps/cli/go.mod`**: The CLI imports `packages/server` via `replace` +directive. The GCP Secret Manager dependency becomes transitive. However, Go's +dead code elimination ensures the GCP client code is only included in the +binary if the CLI actually imports `gcpsecret`. If binary size is a concern, a +`//go:build !nogcp` tag can be added later. + +--- + +## Security Considerations + +1. **Credential handling**: ADC only — no API keys or service account JSON as + parameters. Users configure via `GOOGLE_APPLICATION_CREDENTIALS` or GCE metadata. +2. **Secret masking**: Tracker records `"***"`, not actual values. +3. **Error messages**: Never include secret values in error output. +4. **Cache scope**: Per-resolver instance, not global. Prevents cross-tenant leakage. +5. **Timeouts**: `context.Context` provides natural timeout control. Recommend + 10-second deadline per secret fetch. + +--- + +## Phasing + +### Phase 1 (This Issue): GCP + Core Infrastructure + +- `secretresolver/` package with interface, parsing, fragment extraction +- `gcpsecret/` implementation +- `expression/` modifications (context threading, secret dispatch, error type) +- Unit tests with mock resolver +- Integration test behind build tag +- Wiring into flow builder and CLI +- Dependency update in `packages/server/go.mod` + +### Phase 2 (Future): AWS Secrets Manager + +- Create `secretresolver/awssecret/` package +- Register `"aws"` in `MultiResolver` +- Zero changes needed in `expression/` — `#aws:` prefix already handled + +### Phase 3 (Future): Azure Key Vault + +- Create `secretresolver/azuresecret/` package +- Register `"azure"` in `MultiResolver` +- Zero changes needed in `expression/` — `#azure:` prefix already handled + +**Design-for-extensibility elements built into Phase 1:** + +- `SecretResolver` interface is provider-agnostic +- `MultiResolver` dispatches by provider string +- `IsSecretReference()` already checks all three prefixes +- `ParseSecretReference()` already handles all three prefixes +- `SecretReferenceError` includes `Provider` field diff --git a/packages/server/go.mod b/packages/server/go.mod index 528b3f0e..a312ff9c 100644 --- a/packages/server/go.mod +++ b/packages/server/go.mod @@ -3,6 +3,7 @@ module github.com/the-dev-tools/dev-tools/packages/server go 1.25 require ( + cloud.google.com/go/secretmanager v1.14.3 connectrpc.com/connect v1.19.1 github.com/Microsoft/go-winio v0.6.2 github.com/andybalholm/brotli v1.2.0 diff --git a/packages/server/pkg/expression/errors.go b/packages/server/pkg/expression/errors.go index a8ef564a..661f38f8 100644 --- a/packages/server/pkg/expression/errors.go +++ b/packages/server/pkg/expression/errors.go @@ -101,3 +101,26 @@ func (e *EnvReferenceError) Error() string { func (e *EnvReferenceError) Unwrap() error { return e.Cause } + +// SecretReferenceError represents an error when resolving a cloud secret reference. +type SecretReferenceError struct { + Provider string // "gcp", "aws", "azure" + Ref string // The resource path + Fragment string // Optional JSON fragment key + Cause error +} + +func (e *SecretReferenceError) Error() string { + loc := e.Ref + if e.Fragment != "" { + loc += "#" + e.Fragment + } + if e.Cause != nil { + return fmt.Sprintf("secret reference '%s:%s' failed: %v", e.Provider, loc, e.Cause) + } + return fmt.Sprintf("secret reference '%s:%s' failed", e.Provider, loc) +} + +func (e *SecretReferenceError) Unwrap() error { + return e.Cause +} diff --git a/packages/server/pkg/expression/file_utils.go b/packages/server/pkg/expression/file_utils.go index 7e813c0c..cedea595 100644 --- a/packages/server/pkg/expression/file_utils.go +++ b/packages/server/pkg/expression/file_utils.go @@ -11,6 +11,12 @@ const ( FileRefPrefix = "#file:" // EnvRefPrefix is the prefix for environment variable references. EnvRefPrefix = "#env:" + // GCPRefPrefix is the prefix for GCP Secret Manager references. + GCPRefPrefix = "#gcp:" + // AWSRefPrefix is the prefix for AWS Secrets Manager references (future). + AWSRefPrefix = "#aws:" + // AzureRefPrefix is the prefix for Azure Key Vault references (future). + AzureRefPrefix = "#azure:" ) // IsFileReference checks if a string is a file reference (#file:/path). @@ -100,6 +106,38 @@ func IsVarPattern(s string) bool { strings.Count(s, "{{") == 1 && strings.Count(s, "}}") == 1 } +// IsSecretReference checks if a string is a cloud secret reference (#gcp:, #aws:, #azure:). +func IsSecretReference(s string) bool { + s = strings.TrimSpace(s) + return strings.HasPrefix(s, GCPRefPrefix) || + strings.HasPrefix(s, AWSRefPrefix) || + strings.HasPrefix(s, AzureRefPrefix) +} + +// ParseSecretReference parses a secret reference like "#gcp:path#fragment". +// Returns (provider, resourcePath, fragment). +func ParseSecretReference(s string) (provider, ref, fragment string) { + s = strings.TrimSpace(s) + + switch { + case strings.HasPrefix(s, GCPRefPrefix): + provider = "gcp" + s = strings.TrimPrefix(s, GCPRefPrefix) + case strings.HasPrefix(s, AWSRefPrefix): + provider = "aws" + s = strings.TrimPrefix(s, AWSRefPrefix) + case strings.HasPrefix(s, AzureRefPrefix): + provider = "azure" + s = strings.TrimPrefix(s, AzureRefPrefix) + } + + // Split on last '#' for fragment + if idx := strings.LastIndex(s, "#"); idx != -1 { + return provider, s[:idx], s[idx+1:] + } + return provider, s, "" +} + // ExtractVarKeysFromMultiple extracts all unique variable keys from multiple strings. func ExtractVarKeysFromMultiple(strs ...string) []string { seen := make(map[string]struct{}) diff --git a/packages/server/pkg/expression/interpolate.go b/packages/server/pkg/expression/interpolate.go index 17df1e4c..f410904d 100644 --- a/packages/server/pkg/expression/interpolate.go +++ b/packages/server/pkg/expression/interpolate.go @@ -33,18 +33,27 @@ func (e *UnifiedEnv) Interpolate(raw string) (string, error) { return result.Value, nil } -// InterpolateCtx is like Interpolate but accepts a context for cancellation. +// InterpolateCtx is like Interpolate but accepts a context for cancellation +// and cloud secret resolution. func (e *UnifiedEnv) InterpolateCtx(ctx context.Context, raw string) (string, error) { - // Check context before starting if ctx.Err() != nil { return "", ctx.Err() } - return e.Interpolate(raw) + result, err := e.interpolateWithResultCtx(ctx, raw) + if err != nil { + return "", err + } + return result.Value, nil } // InterpolateWithResult replaces {{ varKey }} patterns and returns both the result // and a map of all variables that were read during interpolation. func (e *UnifiedEnv) InterpolateWithResult(raw string) (InterpolationResult, error) { + return e.interpolateWithResultCtx(context.Background(), raw) +} + +// interpolateWithResultCtx is the context-aware implementation of InterpolateWithResult. +func (e *UnifiedEnv) interpolateWithResultCtx(ctx context.Context, raw string) (InterpolationResult, error) { if e == nil { return InterpolationResult{Value: raw, ReadVars: make(map[string]any)}, nil } @@ -75,7 +84,7 @@ func (e *UnifiedEnv) InterpolateWithResult(raw string) (InterpolationResult, err varRef = strings.TrimSpace(varRef) // Resolve the variable/expression - _, strVal, err := e.resolveVar(varRef, readVars) + _, strVal, err := e.resolveVar(ctx, varRef, readVars) if err != nil { return InterpolationResult{}, &InterpolationError{ Input: raw, @@ -114,7 +123,7 @@ func (e *UnifiedEnv) ResolveValue(raw string) (any, error) { if !strings.Contains(inner, menv.Prefix) && !strings.Contains(inner, menv.Suffix) { // Single expression - return typed value readVars := make(map[string]any) - val, _, err := e.resolveVar(strings.TrimSpace(inner), readVars) + val, _, err := e.resolveVar(context.Background(), strings.TrimSpace(inner), readVars) return val, err } } @@ -130,7 +139,7 @@ func (e *UnifiedEnv) ResolveValue(raw string) (any, error) { // resolveVar resolves a single variable/expression reference and tracks the read. // Returns the resolved value (typed) and its string representation. -func (e *UnifiedEnv) resolveVar(varRef string, readVars map[string]any) (any, string, error) { +func (e *UnifiedEnv) resolveVar(ctx context.Context, varRef string, readVars map[string]any) (any, string, error) { switch { case isEnvReference(varRef): str, err := e.resolveEnvVar(varRef, readVars) @@ -138,6 +147,9 @@ func (e *UnifiedEnv) resolveVar(varRef string, readVars map[string]any) (any, st case isFileReference(varRef): str, err := e.resolveFileVar(varRef, readVars) return str, str, err + case isSecretReference(varRef): + str, err := e.resolveSecretVar(ctx, varRef, readVars) + return str, str, err default: // Use expr-lang - supports paths AND expressions like now(), a > 5 val, err := e.resolveExprVar(varRef, readVars) @@ -218,6 +230,34 @@ func (e *UnifiedEnv) resolveExprVar(varRef string, readVars map[string]any) (any return output, nil } +// resolveSecretVar resolves a cloud secret reference (#gcp:path#fragment, #aws:name#key, etc.). +func (e *UnifiedEnv) resolveSecretVar(ctx context.Context, varRef string, readVars map[string]any) (string, error) { + if e.secretResolver == nil { + return "", &SecretReferenceError{ + Provider: "unknown", + Ref: varRef, + Cause: fmt.Errorf("no secret resolver configured; cloud secret references require a SecretResolver"), + } + } + + provider, ref, fragment := ParseSecretReference(varRef) + if ref == "" { + return "", &SecretReferenceError{Provider: provider, Ref: varRef, Cause: ErrEmptyPath} + } + + value, err := e.secretResolver.ResolveSecret(ctx, provider, ref, fragment) + if err != nil { + return "", &SecretReferenceError{Provider: provider, Ref: ref, Fragment: fragment, Cause: err} + } + + readVars[varRef] = value + if e.tracker != nil { + e.tracker.TrackRead(varRef, "***") // Mask secret values in tracking + } + + return value, nil +} + // isEnvReference checks if a variable reference is an environment variable. func isEnvReference(varRef string) bool { return strings.HasPrefix(strings.TrimSpace(varRef), EnvRefPrefix) @@ -228,6 +268,11 @@ func isFileReference(varRef string) bool { return strings.HasPrefix(strings.TrimSpace(varRef), FileRefPrefix) } +// isSecretReference checks if a variable reference is a cloud secret reference. +func isSecretReference(varRef string) bool { + return IsSecretReference(varRef) +} + // anyToString converts any value to its string representation. func anyToString(v any) string { if v == nil { diff --git a/packages/server/pkg/expression/unified_env.go b/packages/server/pkg/expression/unified_env.go index f8e15bec..e9fd5b5c 100644 --- a/packages/server/pkg/expression/unified_env.go +++ b/packages/server/pkg/expression/unified_env.go @@ -2,16 +2,19 @@ package expression import ( - "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/tracking" "maps" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/tracking" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/secretresolver" ) // UnifiedEnv provides a unified interface for variable resolution, expression evaluation, // and string interpolation. It operates on hierarchical (non-flattened) data. type UnifiedEnv struct { - data map[string]any // Hierarchical data (not flattened) - tracker *tracking.VariableTracker // Optional tracking - customFuncs map[string]any // Custom expr-lang functions + data map[string]any // Hierarchical data (not flattened) + tracker *tracking.VariableTracker // Optional tracking + customFuncs map[string]any // Custom expr-lang functions + secretResolver secretresolver.SecretResolver // Optional: cloud secret resolution } // NewUnifiedEnv creates a new UnifiedEnv with the given hierarchical data. @@ -41,6 +44,14 @@ func (e *UnifiedEnv) WithFunc(name string, fn any) *UnifiedEnv { return clone } +// WithSecretResolver returns a copy of the UnifiedEnv with a secret resolver +// for cloud secret references (#gcp:, #aws:, #azure:). +func (e *UnifiedEnv) WithSecretResolver(r secretresolver.SecretResolver) *UnifiedEnv { + clone := e.Clone() + clone.secretResolver = r + return clone +} + // Clone creates a deep copy of the UnifiedEnv. func (e *UnifiedEnv) Clone() *UnifiedEnv { if e == nil { @@ -54,9 +65,10 @@ func (e *UnifiedEnv) Clone() *UnifiedEnv { maps.Copy(newFuncs, e.customFuncs) return &UnifiedEnv{ - data: newData, - tracker: e.tracker, // Share tracker reference - customFuncs: newFuncs, + data: newData, + tracker: e.tracker, // Share tracker reference + customFuncs: newFuncs, + secretResolver: e.secretResolver, // Share resolver reference } } diff --git a/packages/server/pkg/expression/unified_env_test.go b/packages/server/pkg/expression/unified_env_test.go index ff4d257b..87f348d1 100644 --- a/packages/server/pkg/expression/unified_env_test.go +++ b/packages/server/pkg/expression/unified_env_test.go @@ -2,6 +2,7 @@ package expression import ( "context" + "fmt" "iter" "os" "path/filepath" @@ -9,8 +10,32 @@ import ( "github.com/stretchr/testify/require" "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/tracking" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/secretresolver" ) +// mockSecretResolver is a test double for secretresolver.SecretResolver. +type mockSecretResolver struct { + secrets map[string]string // "provider:ref#fragment" -> value + err error +} + +var _ secretresolver.SecretResolver = (*mockSecretResolver)(nil) + +func (m *mockSecretResolver) ResolveSecret(_ context.Context, provider, ref, fragment string) (string, error) { + if m.err != nil { + return "", m.err + } + key := provider + ":" + ref + if fragment != "" { + key += "#" + fragment + } + val, ok := m.secrets[key] + if !ok { + return "", fmt.Errorf("secret not found: %s", key) + } + return val, nil +} + // ============================================================================= // Path Resolution Tests // ============================================================================= @@ -1257,3 +1282,213 @@ func BenchmarkExtractExprPaths(b *testing.B) { } }) } + +// ============================================================================= +// Cloud Secret Reference Tests +// ============================================================================= + +func TestInterpolate_GCPSecret_SimpleValue(t *testing.T) { + resolver := &mockSecretResolver{ + secrets: map[string]string{ + "gcp:projects/p/secrets/s/versions/latest": "my-secret-value", + }, + } + env := NewUnifiedEnv(nil).WithSecretResolver(resolver) + + result, err := env.Interpolate("Secret: {{ #gcp:projects/p/secrets/s/versions/latest }}") + require.NoError(t, err) + require.Equal(t, "Secret: my-secret-value", result) +} + +func TestInterpolate_GCPSecret_WithFragment(t *testing.T) { + resolver := &mockSecretResolver{ + secrets: map[string]string{ + "gcp:projects/p/secrets/oauth/versions/latest#client_secret": "xyz-secret", + }, + } + env := NewUnifiedEnv(nil).WithSecretResolver(resolver) + + result, err := env.Interpolate("{{ #gcp:projects/p/secrets/oauth/versions/latest#client_secret }}") + require.NoError(t, err) + require.Equal(t, "xyz-secret", result) +} + +func TestInterpolate_GCPSecret_NoResolver(t *testing.T) { + env := NewUnifiedEnv(nil) // No secret resolver configured + + _, err := env.Interpolate("{{ #gcp:projects/p/secrets/s/versions/latest }}") + require.Error(t, err) + require.Contains(t, err.Error(), "no secret resolver configured") + + var secretErr *SecretReferenceError + require.ErrorAs(t, err, &secretErr) +} + +func TestInterpolate_GCPSecret_EmptyPath(t *testing.T) { + resolver := &mockSecretResolver{secrets: map[string]string{}} + env := NewUnifiedEnv(nil).WithSecretResolver(resolver) + + _, err := env.Interpolate("{{ #gcp: }}") + require.Error(t, err) +} + +func TestInterpolate_GCPSecret_MixedReferences(t *testing.T) { + t.Setenv("TEST_MIX_VAR", "env-value") + + resolver := &mockSecretResolver{ + secrets: map[string]string{ + "gcp:projects/p/secrets/s/versions/1": "secret-value", + }, + } + env := NewUnifiedEnv(map[string]any{ + "name": "John", + }).WithSecretResolver(resolver) + + result, err := env.Interpolate("{{ name }} {{ #env:TEST_MIX_VAR }} {{ #gcp:projects/p/secrets/s/versions/1 }}") + require.NoError(t, err) + require.Equal(t, "John env-value secret-value", result) +} + +func TestInterpolate_GCPSecret_TrackedAsMasked(t *testing.T) { + resolver := &mockSecretResolver{ + secrets: map[string]string{ + "gcp:projects/p/secrets/s/versions/latest": "super-secret", + }, + } + tracker := tracking.NewVariableTracker() + env := NewUnifiedEnv(nil).WithSecretResolver(resolver).WithTracking(tracker) + + result, err := env.Interpolate("{{ #gcp:projects/p/secrets/s/versions/latest }}") + require.NoError(t, err) + require.Equal(t, "super-secret", result) + + // Tracker should record masked value, not the actual secret + readVars := tracker.GetReadVars() + require.Contains(t, readVars, "#gcp:projects/p/secrets/s/versions/latest") + require.Equal(t, "***", readVars["#gcp:projects/p/secrets/s/versions/latest"]) +} + +func TestInterpolate_GCPSecret_ResolverError(t *testing.T) { + resolver := &mockSecretResolver{ + err: fmt.Errorf("permission denied"), + } + env := NewUnifiedEnv(nil).WithSecretResolver(resolver) + + _, err := env.Interpolate("{{ #gcp:projects/p/secrets/s/versions/latest }}") + require.Error(t, err) + require.Contains(t, err.Error(), "permission denied") +} + +func TestResolveValue_GCPSecret(t *testing.T) { + resolver := &mockSecretResolver{ + secrets: map[string]string{ + "gcp:projects/p/secrets/s/versions/latest": "typed-secret", + }, + } + env := NewUnifiedEnv(nil).WithSecretResolver(resolver) + + val, err := env.ResolveValue("{{ #gcp:projects/p/secrets/s/versions/latest }}") + require.NoError(t, err) + require.Equal(t, "typed-secret", val) +} + +func TestInterpolate_AWSSecret_NoResolver(t *testing.T) { + // AWS prefix is recognized but no resolver means clear error + env := NewUnifiedEnv(nil) + + _, err := env.Interpolate("{{ #aws:my-secret#key }}") + require.Error(t, err) + require.Contains(t, err.Error(), "no secret resolver configured") +} + +func TestInterpolate_AzureSecret_NoResolver(t *testing.T) { + // Azure prefix is recognized but no resolver means clear error + env := NewUnifiedEnv(nil) + + _, err := env.Interpolate("{{ #azure:vault/secret#key }}") + require.Error(t, err) + require.Contains(t, err.Error(), "no secret resolver configured") +} + +// ============================================================================= +// Secret Reference Helper Tests +// ============================================================================= + +func TestIsSecretReference(t *testing.T) { + require.True(t, IsSecretReference("#gcp:projects/p/secrets/s")) + require.True(t, IsSecretReference("#aws:my-secret")) + require.True(t, IsSecretReference("#azure:vault/secret")) + require.False(t, IsSecretReference("#env:VAR")) + require.False(t, IsSecretReference("#file:/path")) + require.False(t, IsSecretReference("plain")) +} + +func TestParseSecretReference(t *testing.T) { + tests := []struct { + name string + input string + expectedProvider string + expectedRef string + expectedFragment string + }{ + { + name: "gcp with fragment", + input: "#gcp:projects/p/secrets/s/versions/latest#client_secret", + expectedProvider: "gcp", + expectedRef: "projects/p/secrets/s/versions/latest", + expectedFragment: "client_secret", + }, + { + name: "gcp without fragment", + input: "#gcp:projects/p/secrets/s/versions/latest", + expectedProvider: "gcp", + expectedRef: "projects/p/secrets/s/versions/latest", + expectedFragment: "", + }, + { + name: "aws with fragment", + input: "#aws:secret-name#key", + expectedProvider: "aws", + expectedRef: "secret-name", + expectedFragment: "key", + }, + { + name: "azure with fragment", + input: "#azure:vault/secret-name#field", + expectedProvider: "azure", + expectedRef: "vault/secret-name", + expectedFragment: "field", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider, ref, fragment := ParseSecretReference(tt.input) + require.Equal(t, tt.expectedProvider, provider) + require.Equal(t, tt.expectedRef, ref) + require.Equal(t, tt.expectedFragment, fragment) + }) + } +} + +func TestWithSecretResolver_ClonePreservesResolver(t *testing.T) { + resolver := &mockSecretResolver{ + secrets: map[string]string{ + "gcp:projects/p/secrets/s/versions/latest": "value", + }, + } + env := NewUnifiedEnv(nil).WithSecretResolver(resolver) + + // Clone should preserve the resolver + clone := env.Clone() + result, err := clone.Interpolate("{{ #gcp:projects/p/secrets/s/versions/latest }}") + require.NoError(t, err) + require.Equal(t, "value", result) +} + +func TestExtractVarRefs_IncludesSecretRefs(t *testing.T) { + refs := ExtractVarRefs("{{ #gcp:projects/p/secrets/s/versions/latest#key }} and {{ name }}") + require.Len(t, refs, 2) + require.Contains(t, refs, "#gcp:projects/p/secrets/s/versions/latest#key") + require.Contains(t, refs, "name") +} diff --git a/packages/server/pkg/flow/node/nai/nai.go b/packages/server/pkg/flow/node/nai/nai.go index f328fdc3..53ea3c92 100644 --- a/packages/server/pkg/flow/node/nai/nai.go +++ b/packages/server/pkg/flow/node/nai/nai.go @@ -123,6 +123,9 @@ func (n NodeAI) RunSync(ctx context.Context, req *node.FlowNodeRequest) node.Flo // 4. Resolve prompt variables (supports expressions and AI marker functions) env := expression.NewUnifiedEnv(req.VarMap) + if req.SecretResolver != nil { + env = env.WithSecretResolver(req.SecretResolver) + } resolvedPrompt, err := env.Interpolate(n.Prompt) if err != nil { // Use raw prompt as fallback if variable resolution fails diff --git a/packages/server/pkg/flow/node/node.go b/packages/server/pkg/flow/node/node.go index 7435e829..88fc71bb 100644 --- a/packages/server/pkg/flow/node/node.go +++ b/packages/server/pkg/flow/node/node.go @@ -5,15 +5,17 @@ import ( "context" "encoding/json" "errors" - "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/runner" - "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/tracking" - "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" - "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" "log/slog" "sync" "time" "google.golang.org/protobuf/types/known/structpb" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/runner" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/tracking" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/secretresolver" ) var ErrNodeNotFound = errors.New("node not found") @@ -56,10 +58,11 @@ type FlowNodeRequest struct { Timeout time.Duration LogPushFunc LogPushFunc PendingAtmoicMap map[idwrap.IDWrap]uint32 - VariableTracker *tracking.VariableTracker // Optional tracking for input/output data - IterationContext *runner.IterationContext // For hierarchical execution naming in loops - ExecutionID idwrap.IDWrap // Unique ID for this specific execution of the node - Logger *slog.Logger // Optional structured logger for node diagnostics + VariableTracker *tracking.VariableTracker // Optional tracking for input/output data + IterationContext *runner.IterationContext // For hierarchical execution naming in loops + ExecutionID idwrap.IDWrap // Unique ID for this specific execution of the node + Logger *slog.Logger // Optional structured logger for node diagnostics + SecretResolver secretresolver.SecretResolver // Optional resolver for cloud secret references } type LogPushFunc func(status runner.FlowNodeStatus) diff --git a/packages/server/pkg/flow/node/nrequest/nrequest.go b/packages/server/pkg/flow/node/nrequest/nrequest.go index 9751c29d..3a52fe0b 100644 --- a/packages/server/pkg/flow/node/nrequest/nrequest.go +++ b/packages/server/pkg/flow/node/nrequest/nrequest.go @@ -149,7 +149,7 @@ func (nr *NodeRequest) RunSync(ctx context.Context, req *node.FlowNodeRequest) n varMapCopy := node.DeepCopyVarMap(req) prepareResult, err := request.PrepareHTTPRequestWithTracking(nr.HttpReq, nr.Headers, - nr.Params, nr.RawBody, nr.FormBody, nr.UrlBody, varMapCopy) + nr.Params, nr.RawBody, nr.FormBody, nr.UrlBody, varMapCopy, req.SecretResolver) if err != nil { result.Err = err return result @@ -344,7 +344,7 @@ func (nr *NodeRequest) RunAsync(ctx context.Context, req *node.FlowNodeRequest, varMapCopy := node.DeepCopyVarMap(req) prepareResult, err := request.PrepareHTTPRequestWithTracking(nr.HttpReq, nr.Headers, - nr.Params, nr.RawBody, nr.FormBody, nr.UrlBody, varMapCopy) + nr.Params, nr.RawBody, nr.FormBody, nr.UrlBody, varMapCopy, req.SecretResolver) if err != nil { result.Err = err resultChan <- result diff --git a/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go b/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go index 76056f84..18c6ad54 100644 --- a/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go +++ b/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go @@ -5,15 +5,17 @@ import ( "context" "errors" "fmt" + "log/slog" + "runtime" + "sync" + "time" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/node" "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/runner" "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/tracking" "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" - "log/slog" - "runtime" - "sync" - "time" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/secretresolver" ) // ExecutionMode controls how FlowLocalRunner schedules nodes. @@ -39,6 +41,14 @@ type FlowLocalRunner struct { selectedMode ExecutionMode enableDataTracking bool logger *slog.Logger + secretResolver secretresolver.SecretResolver // Optional resolver for cloud secret references +} + +// WithSecretResolver sets the secret resolver for cloud secret references +// in flow expressions (e.g., {{#gcp:...}}). +func (r *FlowLocalRunner) WithSecretResolver(resolver secretresolver.SecretResolver) *FlowLocalRunner { + r.secretResolver = resolver + return r } var _ runner.FlowRunner = (*FlowLocalRunner)(nil) @@ -500,6 +510,7 @@ func (r *FlowLocalRunner) RunWithEvents(ctx context.Context, channels runner.Flo Timeout: r.Timeout, PendingAtmoicMap: pendingAtmoicMap, Logger: r.logger, + SecretResolver: r.secretResolver, } predecessorMap := BuildPredecessorMap(r.EdgesMap) diff --git a/packages/server/pkg/http/request/request.go b/packages/server/pkg/http/request/request.go index 0c5f3b9e..0cea8990 100644 --- a/packages/server/pkg/http/request/request.go +++ b/packages/server/pkg/http/request/request.go @@ -26,6 +26,7 @@ import ( "github.com/the-dev-tools/dev-tools/packages/server/pkg/httpclient" "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mhttp" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/secretresolver" "github.com/the-dev-tools/dev-tools/packages/server/pkg/varsystem" ) @@ -58,6 +59,7 @@ type PrepareHTTPRequestResult struct { // - {{ a + b }} - Expressions // - {{ #env:VAR }} - Environment variables // - {{ #file:/path }} - File contents +// - {{ #gcp:path }} - Cloud secret references (requires secretResolver) func PrepareHTTPRequestWithTracking( httpReq mhttp.HTTP, headers []mhttp.HTTPHeader, @@ -66,9 +68,13 @@ func PrepareHTTPRequestWithTracking( formBody []mhttp.HTTPBodyForm, urlBody []mhttp.HTTPBodyUrlencoded, varMap map[string]any, + secretResolver ...secretresolver.SecretResolver, ) (*PrepareHTTPRequestResult, error) { // Create UnifiedEnv for expression interpolation env := expression.NewUnifiedEnv(varMap) + if len(secretResolver) > 0 && secretResolver[0] != nil { + env = env.WithSecretResolver(secretResolver[0]) + } readVars := make(map[string]any) // Helper to interpolate and collect reads diff --git a/packages/server/pkg/secretresolver/fragment.go b/packages/server/pkg/secretresolver/fragment.go new file mode 100644 index 00000000..22bebc7b --- /dev/null +++ b/packages/server/pkg/secretresolver/fragment.go @@ -0,0 +1,36 @@ +package secretresolver + +import ( + "encoding/json" + "fmt" +) + +// ExtractFragment extracts a JSON field from a value if fragment is non-empty. +// If fragment is empty, returns the raw value as-is. +// If fragment is specified but the value is not valid JSON or the key is missing, returns an error. +func ExtractFragment(value, fragment string) (string, error) { + if fragment == "" { + return value, nil + } + + var obj map[string]any + if err := json.Unmarshal([]byte(value), &obj); err != nil { + return "", fmt.Errorf("secret value is not valid JSON for fragment extraction: %w", err) + } + + v, ok := obj[fragment] + if !ok { + return "", fmt.Errorf("fragment key %q not found in secret JSON", fragment) + } + + switch val := v.(type) { + case string: + return val, nil + default: + data, err := json.Marshal(val) + if err != nil { + return "", fmt.Errorf("cannot marshal fragment value: %w", err) + } + return string(data), nil + } +} diff --git a/packages/server/pkg/secretresolver/fragment_test.go b/packages/server/pkg/secretresolver/fragment_test.go new file mode 100644 index 00000000..32a95286 --- /dev/null +++ b/packages/server/pkg/secretresolver/fragment_test.go @@ -0,0 +1,84 @@ +package secretresolver + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExtractFragment(t *testing.T) { + tests := []struct { + name string + value string + fragment string + expected string + expectError bool + }{ + { + name: "empty fragment returns raw value", + value: `{"key": "value"}`, + fragment: "", + expected: `{"key": "value"}`, + }, + { + name: "extract string field", + value: `{"client_id": "abc", "client_secret": "xyz"}`, + fragment: "client_secret", + expected: "xyz", + }, + { + name: "extract numeric field", + value: `{"port": 8080, "host": "localhost"}`, + fragment: "port", + expected: "8080", + }, + { + name: "extract boolean field", + value: `{"enabled": true}`, + fragment: "enabled", + expected: "true", + }, + { + name: "extract nested object field", + value: `{"config": {"nested": "value"}}`, + fragment: "config", + expected: `{"nested":"value"}`, + }, + { + name: "extract array field", + value: `{"items": [1, 2, 3]}`, + fragment: "items", + expected: `[1,2,3]`, + }, + { + name: "missing fragment key", + value: `{"key": "value"}`, + fragment: "missing", + expectError: true, + }, + { + name: "non-JSON value with fragment", + value: "plain-text-secret", + fragment: "key", + expectError: true, + }, + { + name: "plain text without fragment", + value: "plain-text-secret", + fragment: "", + expected: "plain-text-secret", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := ExtractFragment(tt.value, tt.fragment) + if tt.expectError { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tt.expected, result) + }) + } +} diff --git a/packages/server/pkg/secretresolver/gcpsecret/gcp.go b/packages/server/pkg/secretresolver/gcpsecret/gcp.go new file mode 100644 index 00000000..88bd41a6 --- /dev/null +++ b/packages/server/pkg/secretresolver/gcpsecret/gcp.go @@ -0,0 +1,95 @@ +// Package gcpsecret implements SecretResolver for Google Cloud Secret Manager. +package gcpsecret + +import ( + "context" + "fmt" + "sync" + "time" + + secretmanager "cloud.google.com/go/secretmanager/apiv1" + "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/secretresolver" +) + +type cacheEntry struct { + value string + fetchedAt time.Time +} + +// Resolver resolves secrets from Google Cloud Secret Manager. +// It uses Application Default Credentials (ADC) for authentication. +type Resolver struct { + client *secretmanager.Client + cache sync.Map // map[string]cacheEntry + ttl time.Duration +} + +// New creates a new GCP secret resolver. +func New(ctx context.Context, opts ...Option) (*Resolver, error) { + cfg := defaultConfig() + for _, o := range opts { + o(&cfg) + } + + client, err := secretmanager.NewClient(ctx) + if err != nil { + return nil, fmt.Errorf("creating GCP Secret Manager client: %w", err) + } + + return &Resolver{client: client, ttl: cfg.cacheTTL}, nil +} + +// ResolveSecret fetches a secret from GCP Secret Manager. +func (r *Resolver) ResolveSecret(ctx context.Context, provider, ref, fragment string) (string, error) { + if provider != "gcp" { + return "", fmt.Errorf("GCP resolver does not support provider %q", provider) + } + + cacheKey := ref + "#" + fragment + if entry, ok := r.cache.Load(cacheKey); ok { + e := entry.(cacheEntry) + if time.Since(e.fetchedAt) < r.ttl { + return e.value, nil + } + } + + result, err := r.client.AccessSecretVersion(ctx, &secretmanagerpb.AccessSecretVersionRequest{ + Name: ref, + }) + if err != nil { + return "", fmt.Errorf("accessing GCP secret %q: %w", ref, err) + } + + raw := string(result.Payload.Data) + + value, err := secretresolver.ExtractFragment(raw, fragment) + if err != nil { + return "", fmt.Errorf("extracting fragment from GCP secret %q: %w", ref, err) + } + + r.cache.Store(cacheKey, cacheEntry{value: value, fetchedAt: time.Now()}) + return value, nil +} + +// Close releases the underlying gRPC connection. +func (r *Resolver) Close() error { + return r.client.Close() +} + +type config struct { + cacheTTL time.Duration +} + +func defaultConfig() config { + return config{cacheTTL: 5 * time.Minute} +} + +// Option configures the GCP resolver. +type Option func(*config) + +// WithCacheTTL sets the cache TTL for resolved secrets. +func WithCacheTTL(d time.Duration) Option { + return func(c *config) { c.cacheTTL = d } +} diff --git a/packages/server/pkg/secretresolver/gcpsecret/integration_gcp_test.go b/packages/server/pkg/secretresolver/gcpsecret/integration_gcp_test.go new file mode 100644 index 00000000..5dc56ee8 --- /dev/null +++ b/packages/server/pkg/secretresolver/gcpsecret/integration_gcp_test.go @@ -0,0 +1,58 @@ +//go:build gcp_integration + +package gcpsecret + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGCPResolver_AccessSecret(t *testing.T) { + if os.Getenv("RUN_GCP_INTEGRATION_TESTS") != "true" { + t.Skip("Skipping GCP integration test: RUN_GCP_INTEGRATION_TESTS != true") + } + + secretName := os.Getenv("GCP_TEST_SECRET_NAME") + if secretName == "" { + t.Skip("GCP_TEST_SECRET_NAME not set") + } + + ctx := context.Background() + + resolver, err := New(ctx) + require.NoError(t, err) + defer resolver.Close() + + value, err := resolver.ResolveSecret(ctx, "gcp", secretName, "") + require.NoError(t, err) + require.NotEmpty(t, value, "expected non-empty secret value") +} + +func TestGCPResolver_AccessSecretWithFragment(t *testing.T) { + if os.Getenv("RUN_GCP_INTEGRATION_TESTS") != "true" { + t.Skip("Skipping GCP integration test: RUN_GCP_INTEGRATION_TESTS != true") + } + + secretName := os.Getenv("GCP_TEST_JSON_SECRET_NAME") + if secretName == "" { + t.Skip("GCP_TEST_JSON_SECRET_NAME not set") + } + + fragmentKey := os.Getenv("GCP_TEST_FRAGMENT_KEY") + if fragmentKey == "" { + t.Skip("GCP_TEST_FRAGMENT_KEY not set") + } + + ctx := context.Background() + + resolver, err := New(ctx) + require.NoError(t, err) + defer resolver.Close() + + value, err := resolver.ResolveSecret(ctx, "gcp", secretName, fragmentKey) + require.NoError(t, err) + require.NotEmpty(t, value, "expected non-empty fragment value") +} diff --git a/packages/server/pkg/secretresolver/multi.go b/packages/server/pkg/secretresolver/multi.go new file mode 100644 index 00000000..d6c60a72 --- /dev/null +++ b/packages/server/pkg/secretresolver/multi.go @@ -0,0 +1,35 @@ +package secretresolver + +import ( + "context" + "fmt" + "strings" +) + +// MultiResolver dispatches to provider-specific resolvers. +type MultiResolver struct { + providers map[string]SecretResolver +} + +// NewMultiResolver creates a resolver that dispatches by provider name. +func NewMultiResolver() *MultiResolver { + return &MultiResolver{providers: make(map[string]SecretResolver)} +} + +// Register adds a provider-specific resolver. +func (m *MultiResolver) Register(provider string, resolver SecretResolver) { + m.providers[provider] = resolver +} + +// ResolveSecret dispatches to the registered provider. +func (m *MultiResolver) ResolveSecret(ctx context.Context, provider, ref, fragment string) (string, error) { + r, ok := m.providers[provider] + if !ok { + available := make([]string, 0, len(m.providers)) + for k := range m.providers { + available = append(available, k) + } + return "", fmt.Errorf("unsupported secret provider %q (available: %s)", provider, strings.Join(available, ", ")) + } + return r.ResolveSecret(ctx, provider, ref, fragment) +} diff --git a/packages/server/pkg/secretresolver/multi_test.go b/packages/server/pkg/secretresolver/multi_test.go new file mode 100644 index 00000000..ab452af2 --- /dev/null +++ b/packages/server/pkg/secretresolver/multi_test.go @@ -0,0 +1,56 @@ +package secretresolver + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +type stubResolver struct { + value string + err error +} + +func (s *stubResolver) ResolveSecret(_ context.Context, _, _, _ string) (string, error) { + if s.err != nil { + return "", s.err + } + return s.value, nil +} + +func TestMultiResolver_Dispatch(t *testing.T) { + multi := NewMultiResolver() + multi.Register("gcp", &stubResolver{value: "gcp-secret"}) + multi.Register("aws", &stubResolver{value: "aws-secret"}) + + ctx := context.Background() + + t.Run("dispatches to gcp", func(t *testing.T) { + val, err := multi.ResolveSecret(ctx, "gcp", "ref", "") + require.NoError(t, err) + require.Equal(t, "gcp-secret", val) + }) + + t.Run("dispatches to aws", func(t *testing.T) { + val, err := multi.ResolveSecret(ctx, "aws", "ref", "") + require.NoError(t, err) + require.Equal(t, "aws-secret", val) + }) + + t.Run("unsupported provider returns error", func(t *testing.T) { + _, err := multi.ResolveSecret(ctx, "azure", "ref", "") + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported secret provider") + }) + + t.Run("provider error is forwarded", func(t *testing.T) { + multi2 := NewMultiResolver() + multi2.Register("gcp", &stubResolver{err: fmt.Errorf("permission denied")}) + + _, err := multi2.ResolveSecret(ctx, "gcp", "ref", "") + require.Error(t, err) + require.Contains(t, err.Error(), "permission denied") + }) +} diff --git a/packages/server/pkg/secretresolver/parse.go b/packages/server/pkg/secretresolver/parse.go new file mode 100644 index 00000000..480498fc --- /dev/null +++ b/packages/server/pkg/secretresolver/parse.go @@ -0,0 +1,14 @@ +package secretresolver + +import "strings" + +// ParseSecretRef splits a secret reference into its resource path and optional fragment. +// Input: "projects/p/secrets/s/versions/latest#client_secret" +// Returns: ("projects/p/secrets/s/versions/latest", "client_secret") +// If no fragment is present, fragment is empty. +func ParseSecretRef(ref string) (path, fragment string) { + if idx := strings.LastIndex(ref, "#"); idx != -1 { + return ref[:idx], ref[idx+1:] + } + return ref, "" +} diff --git a/packages/server/pkg/secretresolver/parse_test.go b/packages/server/pkg/secretresolver/parse_test.go new file mode 100644 index 00000000..9a8be1ab --- /dev/null +++ b/packages/server/pkg/secretresolver/parse_test.go @@ -0,0 +1,55 @@ +package secretresolver + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseSecretRef(t *testing.T) { + tests := []struct { + name string + input string + expectedPath string + expectedFrag string + }{ + { + name: "path with fragment", + input: "projects/p/secrets/s/versions/latest#client_secret", + expectedPath: "projects/p/secrets/s/versions/latest", + expectedFrag: "client_secret", + }, + { + name: "path without fragment", + input: "projects/p/secrets/s/versions/latest", + expectedPath: "projects/p/secrets/s/versions/latest", + expectedFrag: "", + }, + { + name: "empty string", + input: "", + expectedPath: "", + expectedFrag: "", + }, + { + name: "fragment only", + input: "#key", + expectedPath: "", + expectedFrag: "key", + }, + { + name: "multiple hash signs uses last", + input: "path#middle#last", + expectedPath: "path#middle", + expectedFrag: "last", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path, fragment := ParseSecretRef(tt.input) + require.Equal(t, tt.expectedPath, path) + require.Equal(t, tt.expectedFrag, fragment) + }) + } +} diff --git a/packages/server/pkg/secretresolver/resolver.go b/packages/server/pkg/secretresolver/resolver.go new file mode 100644 index 00000000..6e7281be --- /dev/null +++ b/packages/server/pkg/secretresolver/resolver.go @@ -0,0 +1,13 @@ +// Package secretresolver defines the interface and utilities for resolving +// cloud secret manager references (e.g., GCP Secret Manager, AWS Secrets Manager). +package secretresolver + +import "context" + +// SecretResolver resolves cloud secret manager references. +// The provider identifies the cloud platform ("gcp", "aws", "azure"). +// The ref is the provider-specific resource path. +// The fragment is an optional JSON field name to extract from the secret value. +type SecretResolver interface { + ResolveSecret(ctx context.Context, provider, ref, fragment string) (string, error) +}