diff --git a/.changeset/smart-files-brake.md b/.changeset/smart-files-brake.md new file mode 100644 index 000000000..e8d4d6820 --- /dev/null +++ b/.changeset/smart-files-brake.md @@ -0,0 +1,5 @@ +--- +"chainlink-deployments-framework": minor +--- + +feat(cld/runtime): add support to execute registered changesets from YAML input diff --git a/engine/cld/changeset/common.go b/engine/cld/changeset/common.go index 42bb3ee58..082a2e277 100644 --- a/engine/cld/changeset/common.go +++ b/engine/cld/changeset/common.go @@ -32,6 +32,7 @@ type Configurations struct { type internalChangeSet interface { noop() // unexported function to prevent arbitrary structs from implementing ChangeSet. Apply(env fdeployment.Environment) (fdeployment.ChangesetOutput, error) + applyWithInput(env fdeployment.Environment, inputStr string) (fdeployment.ChangesetOutput, error) Configurations() (Configurations, error) } @@ -77,6 +78,34 @@ type TypedJSON struct { ChainOverrides []uint64 `json:"chainOverrides"` // Optional field for chain overrides } +func parseTypedInput(inputStr string) (TypedJSON, error) { + if inputStr == "" { + return TypedJSON{}, errors.New("input is empty") + } + + var inputObject TypedJSON + if err := json.Unmarshal([]byte(inputStr), &inputObject); err != nil { + return TypedJSON{}, fmt.Errorf("JSON must be in JSON format with 'payload' fields: %w", err) + } + if len(inputObject.Payload) == 0 { + return TypedJSON{}, errors.New("'payload' field is required") + } + + return inputObject, nil +} + +func decodePayload[C any](payload json.RawMessage) (C, error) { + var config C + + payloadDecoder := json.NewDecoder(strings.NewReader(string(payload))) + payloadDecoder.DisallowUnknownFields() + if err := payloadDecoder.Decode(&config); err != nil { + return config, fmt.Errorf("failed to unmarshal payload: %w", err) + } + + return config, nil +} + // WithJSON returns a fully configured changeset, which pairs a [fdeployment.ChangeSet] with its configuration based // a JSON input. It also allows extensions, such as a PostProcessing function. // InputStr must be a JSON object with a "payload" field that contains the actual input data for a Durable Pipeline. @@ -92,30 +121,13 @@ type TypedJSON struct { // Note: Prefer WithEnvInput for durable_pipelines.go func (f WrappedChangeSet[C]) WithJSON(_ C, inputStr string) ConfiguredChangeSet { return ChangeSetImpl[C]{changeset: f, configProvider: func() (C, error) { - var config C - - if inputStr == "" { - return config, errors.New("input is empty") - } - - var inputObject TypedJSON - if err := json.Unmarshal([]byte(inputStr), &inputObject); err != nil { - return config, fmt.Errorf("JSON must be in JSON format with 'payload' fields: %w", err) - } - - // If payload is null, decode it as null (which will give zero value) - // If payload is missing, return an error - if len(inputObject.Payload) == 0 { - return config, errors.New("'payload' field is required") - } - - payloadDecoder := json.NewDecoder(strings.NewReader(string(inputObject.Payload))) - payloadDecoder.DisallowUnknownFields() - if err := payloadDecoder.Decode(&config); err != nil { - return config, fmt.Errorf("failed to unmarshal payload: %w", err) + inputObject, err := parseTypedInput(inputStr) + if err != nil { + var zero C + return zero, err } - return config, nil + return decodePayload[C](inputObject.Payload) }, inputChainOverrides: func() ([]uint64, error) { return loadInputChainOverrides(inputStr) @@ -151,41 +163,36 @@ func (f WrappedChangeSet[C]) WithEnvInput(opts ...EnvInputOption[C]) ConfiguredC inputStr := os.Getenv("DURABLE_PIPELINE_INPUT") - return ChangeSetImpl[C]{changeset: f, configProvider: func() (C, error) { - var config C - - if inputStr == "" { - return config, errors.New("input is empty") - } - - var inputObject TypedJSON - if err := json.Unmarshal([]byte(inputStr), &inputObject); err != nil { - return config, fmt.Errorf("JSON must be in JSON format with 'payload' fields: %w", err) - } + providerFromInput := func(rawInput string) (C, error) { + var zero C - // If payload is null, decode it as null (which will give zero value) - // If payload is missing, return an error - if len(inputObject.Payload) == 0 { - return config, errors.New("'payload' field is required") + inputObject, err := parseTypedInput(rawInput) + if err != nil { + return zero, err } - payloadDecoder := json.NewDecoder(strings.NewReader(string(inputObject.Payload))) - payloadDecoder.DisallowUnknownFields() - if err := payloadDecoder.Decode(&config); err != nil { - return config, fmt.Errorf("failed to unmarshal payload: %w", err) + config, err := decodePayload[C](inputObject.Payload) + if err != nil { + return zero, err } if options.inputModifier != nil { - conf, err := options.inputModifier(config) - if err != nil { - return conf, fmt.Errorf("failed to apply input modifier: %w", err) + conf, modifierErr := options.inputModifier(config) + if modifierErr != nil { + return conf, fmt.Errorf("failed to apply input modifier: %w", modifierErr) } return conf, nil } return config, nil - }, + } + + return ChangeSetImpl[C]{changeset: f, + configProvider: func() (C, error) { + return providerFromInput(inputStr) + }, + configProviderWithInput: providerFromInput, inputChainOverrides: func() ([]uint64, error) { return loadInputChainOverrides(inputStr) }, @@ -221,21 +228,17 @@ func (f WrappedChangeSet[C]) WithConfigResolver(resolver fresolvers.ConfigResolv // Read input from environment variable inputStr := os.Getenv("DURABLE_PIPELINE_INPUT") - configProvider := func() (C, error) { + configProviderFromInput := func(rawInput string) (C, error) { var zero C - if inputStr == "" { + if rawInput == "" { return zero, errors.New("input is empty") } - // Parse JSON input var inputObject TypedJSON - if err := json.Unmarshal([]byte(inputStr), &inputObject); err != nil { + if err := json.Unmarshal([]byte(rawInput), &inputObject); err != nil { return zero, fmt.Errorf("failed to parse resolver input as JSON: %w", err) } - - // If payload is null, pass it to the resolver (which will receive null) - // If payload field is missing, return an error if len(inputObject.Payload) == 0 { return zero, errors.New("'payload' field is required") } @@ -249,8 +252,12 @@ func (f WrappedChangeSet[C]) WithConfigResolver(resolver fresolvers.ConfigResolv return typedConfig, nil } - return ChangeSetImpl[C]{changeset: f, configProvider: configProvider, - ConfigResolver: resolver, + return ChangeSetImpl[C]{changeset: f, + configProvider: func() (C, error) { + return configProviderFromInput(inputStr) + }, + configProviderWithInput: configProviderFromInput, + ConfigResolver: resolver, inputChainOverrides: func() ([]uint64, error) { return loadInputChainOverrides(inputStr) }, @@ -260,9 +267,10 @@ func (f WrappedChangeSet[C]) WithConfigResolver(resolver fresolvers.ConfigResolv var _ ConfiguredChangeSet = ChangeSetImpl[any]{} type ChangeSetImpl[C any] struct { - changeset WrappedChangeSet[C] - configProvider func() (C, error) - inputChainOverrides func() ([]uint64, error) + changeset WrappedChangeSet[C] + configProvider func() (C, error) + configProviderWithInput func(inputStr string) (C, error) + inputChainOverrides func() ([]uint64, error) // Present only when the changeset was wired with // Configure(...).WithConfigResolver(...) @@ -287,6 +295,25 @@ func (ccs ChangeSetImpl[C]) Apply(env fdeployment.Environment) (fdeployment.Chan return ccs.changeset.operation.Apply(env, c) } +func (ccs ChangeSetImpl[C]) applyWithInput(env fdeployment.Environment, inputStr string) (fdeployment.ChangesetOutput, error) { + if inputStr == "" { + return ccs.Apply(env) + } + if ccs.configProviderWithInput == nil { + return ccs.Apply(env) + } + + c, err := ccs.configProviderWithInput(inputStr) + if err != nil { + return fdeployment.ChangesetOutput{}, err + } + if err := ccs.changeset.operation.VerifyPreconditions(env, c); err != nil { + return fdeployment.ChangesetOutput{}, err + } + + return ccs.changeset.operation.Apply(env, c) +} + func (ccs ChangeSetImpl[C]) Configurations() (Configurations, error) { var chainOverrides []uint64 var err error diff --git a/engine/cld/changeset/postprocess.go b/engine/cld/changeset/postprocess.go index 5119ea4d3..08a214407 100644 --- a/engine/cld/changeset/postprocess.go +++ b/engine/cld/changeset/postprocess.go @@ -35,6 +35,18 @@ func (ccs PostProcessingChangeSetImpl[C]) Apply(env fdeployment.Environment) (fd return ccs.postProcessor(env, output) } +func (ccs PostProcessingChangeSetImpl[C]) applyWithInput( + env fdeployment.Environment, inputStr string, +) (fdeployment.ChangesetOutput, error) { + env.Logger.Debugf("Post-processing ChangesetOutput from %T", ccs.changeset.changeset.operation) + output, err := ccs.changeset.applyWithInput(env, inputStr) + if err != nil { + return output, err + } + + return ccs.postProcessor(env, output) +} + func (ccs PostProcessingChangeSetImpl[C]) Configurations() (Configurations, error) { return ccs.changeset.Configurations() } diff --git a/engine/cld/changeset/registry.go b/engine/cld/changeset/registry.go index 62ffd1318..806f25ed5 100644 --- a/engine/cld/changeset/registry.go +++ b/engine/cld/changeset/registry.go @@ -172,6 +172,21 @@ func (r *ChangesetsRegistry) AddGlobalPostHooks(hooks ...PostHook) { // a failed Apply are logged but never mask the Apply error. func (r *ChangesetsRegistry) Apply( key string, e fdeployment.Environment, +) (fdeployment.ChangesetOutput, error) { + return r.applyWithInput(key, e, "") +} + +// ApplyWithInput applies a changeset with explicit input JSON for this apply invocation. +// Changesets configured to read durable pipeline input from environment variables +// consume this value directly when they support input-aware execution. +func (r *ChangesetsRegistry) ApplyWithInput( + key string, e fdeployment.Environment, inputStr string, +) (fdeployment.ChangesetOutput, error) { + return r.applyWithInput(key, e, inputStr) +} + +func (r *ChangesetsRegistry) applyWithInput( + key string, e fdeployment.Environment, inputStr string, ) (fdeployment.ChangesetOutput, error) { entry, globalPre, globalPost, err := r.getApplySnapshot(key) if err != nil { @@ -204,7 +219,9 @@ func (r *ChangesetsRegistry) Apply( } } - output, applyErr := entry.changeset.Apply(e) + var output fdeployment.ChangesetOutput + var applyErr error + output, applyErr = entry.changeset.applyWithInput(e, inputStr) postParams := PostHookParams{ Env: hookEnv, diff --git a/engine/cld/changeset/registry_test.go b/engine/cld/changeset/registry_test.go index 95c7e8718..a828e7de3 100644 --- a/engine/cld/changeset/registry_test.go +++ b/engine/cld/changeset/registry_test.go @@ -25,6 +25,10 @@ func (noopChangeset) Apply(e fdeployment.Environment) (fdeployment.ChangesetOutp return fdeployment.ChangesetOutput{}, nil } +func (n noopChangeset) applyWithInput(e fdeployment.Environment, _ string) (fdeployment.ChangesetOutput, error) { + return n.Apply(e) +} + func (n noopChangeset) Configurations() (Configurations, error) { return Configurations{ InputChainOverrides: n.chainOverrides, @@ -45,6 +49,10 @@ func (r *recordingChangeset) Apply(_ fdeployment.Environment) (fdeployment.Chang return r.output, r.err } +func (r *recordingChangeset) applyWithInput(e fdeployment.Environment, _ string) (fdeployment.ChangesetOutput, error) { + return r.Apply(e) +} + func (*recordingChangeset) Configurations() (Configurations, error) { return Configurations{}, nil } @@ -61,6 +69,10 @@ func (o *orderRecordingChangeset) Apply(_ fdeployment.Environment) (fdeployment. return fdeployment.ChangesetOutput{}, nil } +func (o *orderRecordingChangeset) applyWithInput(e fdeployment.Environment, _ string) (fdeployment.ChangesetOutput, error) { + return o.Apply(e) +} + func (*orderRecordingChangeset) Configurations() (Configurations, error) { return Configurations{}, nil } @@ -148,6 +160,63 @@ func Test_Changesets_Apply(t *testing.T) { } } +//nolint:paralleltest // Uses process environment for fallback behavior assertions. +func Test_Changesets_ApplyWithInput_WithEnvConfiguredChangeset(t *testing.T) { + type inputConfig struct { + Value int `json:"value"` + } + + t.Setenv("DURABLE_PIPELINE_INPUT", `{"payload":{"value":999}}`) + + var received int + cs := fdeployment.CreateChangeSet( + func(_ fdeployment.Environment, cfg inputConfig) (fdeployment.ChangesetOutput, error) { + received = cfg.Value + return fdeployment.ChangesetOutput{}, nil + }, + func(_ fdeployment.Environment, _ inputConfig) error { return nil }, + ) + + r := NewChangesetsRegistry() + r.Add("0001_test", Configure(cs).WithEnvInput()) + + _, err := r.ApplyWithInput("0001_test", fdeployment.Environment{}, `{"payload":{"value":1}}`) + require.NoError(t, err) + require.Equal(t, 1, received) +} + +//nolint:paralleltest // Uses process environment for fallback behavior assertions. +func Test_Changesets_ApplyWithInput_WithResolverConfiguredChangeset(t *testing.T) { + type resolverInput struct { + Base int `json:"base"` + } + type resolverOutput struct { + Value int `json:"value"` + } + + t.Setenv("DURABLE_PIPELINE_INPUT", `{"payload":{"base":100}}`) + + resolver := func(input resolverInput) (resolverOutput, error) { + return resolverOutput{Value: input.Base + 10}, nil + } + + var received int + cs := fdeployment.CreateChangeSet( + func(_ fdeployment.Environment, cfg resolverOutput) (fdeployment.ChangesetOutput, error) { + received = cfg.Value + return fdeployment.ChangesetOutput{}, nil + }, + func(_ fdeployment.Environment, _ resolverOutput) error { return nil }, + ) + + r := NewChangesetsRegistry() + r.Add("0001_test", Configure(cs).WithConfigResolver(resolver)) + + _, err := r.ApplyWithInput("0001_test", fdeployment.Environment{}, `{"payload":{"base":7}}`) + require.NoError(t, err) + require.Equal(t, 17, received) +} + func Test_Changesets_Add(t *testing.T) { t.Parallel() diff --git a/engine/cld/durablepipeline/yaml.go b/engine/cld/durablepipeline/yaml.go new file mode 100644 index 000000000..6af98fb7a --- /dev/null +++ b/engine/cld/durablepipeline/yaml.go @@ -0,0 +1,317 @@ +package durablepipeline + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "math/big" + "os" + "regexp" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +var decimalInteger = regexp.MustCompile(`^-?(0|[1-9][0-9]*)$`) + +// ParsedYAML represents the structure of a durable pipeline YAML input file. +type ParsedYAML struct { + Environment string `yaml:"environment"` + Domain string `yaml:"domain"` + Changesets any `yaml:"changesets"` // Must be []any (array format) +} + +// NamedChangeset is a changeset name with its corresponding YAML data. +type NamedChangeset struct { + Name string + Data any +} + +// ParseYAMLBytes parses and validates durable pipeline YAML content. +func ParseYAMLBytes(yamlData []byte) (*ParsedYAML, error) { + var root yaml.Node + if err := yaml.Unmarshal(yamlData, &root); err != nil { + return nil, fmt.Errorf("failed to parse YAML bytes: %w", err) + } + + rootMap, ok := yamlNodeToAny(&root).(map[string]any) + if !ok { + return nil, errors.New("expected a YAML object at the root") + } + + envRaw, hasEnv := rootMap["environment"] + domainRaw, hasDomain := rootMap["domain"] + changesetsRaw, hasChangesets := rootMap["changesets"] + + dpYAML := &ParsedYAML{ + Changesets: changesetsRaw, + } + if envStr, ok := envRaw.(string); ok { + dpYAML.Environment = envStr + } + if domainStr, ok := domainRaw.(string); ok { + dpYAML.Domain = domainStr + } + + if !hasEnv || dpYAML.Environment == "" { + return nil, errors.New("missing required 'environment' field") + } + if !hasDomain || dpYAML.Domain == "" { + return nil, errors.New("missing required 'domain' field") + } + if !hasChangesets || dpYAML.Changesets == nil { + return nil, errors.New("missing required 'changesets' field") + } + + return dpYAML, nil +} + +// FindChangesetInData finds a changeset in array format. +func FindChangesetInData(changesets any, changesetName string) (any, error) { + data, ok := changesets.([]any) + if !ok { + return nil, errors.New("invalid 'changesets' format, expected array format") + } + + if len(data) == 0 { + return nil, errors.New("empty 'changesets' array") + } + + for _, item := range data { + if itemMap, ok := item.(map[string]any); ok { + if changesetData, exists := itemMap[changesetName]; exists { + return changesetData, nil + } + } + } + + return nil, fmt.Errorf("changeset '%s' not found", changesetName) +} + +// GetAllChangesetsInOrder returns all changesets in order from array format changesets data. +func GetAllChangesetsInOrder(changesets any) ([]NamedChangeset, error) { + var result []NamedChangeset + + data, ok := changesets.([]any) + if !ok { + return nil, errors.New("invalid 'changesets' format for index access, expected array format") + } + + for _, item := range data { + if itemMap, ok := item.(map[string]any); ok { + for name, changesetData := range itemMap { + result = append(result, NamedChangeset{ + Name: name, + Data: changesetData, + }) + } + } + } + + return result, nil +} + +// SetChangesetEnvironmentVariable sets DURABLE_PIPELINE_INPUT from changeset data. +func SetChangesetEnvironmentVariable(changesetName string, changesetData any, inputName string) error { + inputJSON, err := BuildChangesetInputJSON(changesetName, changesetData) + if err != nil { + return fmt.Errorf("failed to build input for changeset %q in input file %s: %w", changesetName, inputName, err) + } + + if err := os.Setenv("DURABLE_PIPELINE_INPUT", inputJSON); err != nil { + return fmt.Errorf("failed to set DURABLE_PIPELINE_INPUT environment variable: %w", err) + } + + return nil +} + +// BuildChangesetInputJSON returns the DURABLE_PIPELINE_INPUT JSON string for changeset data. +func BuildChangesetInputJSON(changesetName string, changesetData any) (string, error) { + changesetMap, ok := changesetData.(map[string]any) + if !ok { + return "", fmt.Errorf("changeset %q is not a valid object", changesetName) + } + + payload, payloadExists := changesetMap["payload"] + if !payloadExists { + return "", fmt.Errorf("changeset %q is missing required 'payload' field", changesetName) + } + + jsonSafePayload, err := convertToJSONSafe(payload) + if err != nil { + return "", fmt.Errorf("failed to convert payload to JSON-safe format: %w", err) + } + + chainOverridesRaw, exists := changesetMap["chainOverrides"] + if exists && chainOverridesRaw != nil { + if chainOverridesList, ok := chainOverridesRaw.([]any); ok { + for _, override := range chainOverridesList { + switch v := override.(type) { + case int: + if v < 0 { + return "", fmt.Errorf("chain override value must be non-negative, got: %d", v) + } + case int64: + if v < 0 { + return "", fmt.Errorf("chain override value must be non-negative, got: %d", v) + } + case uint64: + case json.Number: + n, ok := new(big.Int).SetString(v.String(), 10) + if !ok { + return "", fmt.Errorf("chain override value must be an integer, got type %T with value: %v", override, override) + } + if n.Sign() < 0 { + return "", fmt.Errorf("chain override value must be non-negative, got: %s", v.String()) + } + default: + return "", fmt.Errorf("chain override value must be an integer, got type %T with value: %v", override, override) + } + } + } + } + + input := map[string]any{ + "payload": jsonSafePayload, + } + if exists { + input["chainOverrides"] = chainOverridesRaw + } + + jsonData, err := json.Marshal(input) + if err != nil { + return "", fmt.Errorf("failed to marshal payload to JSON: %w", err) + } + + return string(jsonData), nil +} + +// convertToJSONSafe recursively converts YAML-decoded objects to JSON-safe structures. +func convertToJSONSafe(data any) (any, error) { + switch v := data.(type) { + case map[interface{}]interface{}: + result := make(map[string]any) + for key, value := range v { + var keyStr string + switch k := key.(type) { + case string: + keyStr = k + case int: + keyStr = strconv.Itoa(k) + case int64: + keyStr = strconv.FormatInt(k, 10) + case uint64: + keyStr = strconv.FormatUint(k, 10) + case float64: + keyStr = strconv.FormatFloat(k, 'f', -1, 64) + default: + keyStr = fmt.Sprintf("%v", k) + } + + convertedValue, err := convertToJSONSafe(value) + if err != nil { + return nil, err + } + result[keyStr] = convertedValue + } + + return result, nil + case map[string]any: + result := make(map[string]any) + for key, value := range v { + convertedValue, err := convertToJSONSafe(value) + if err != nil { + return nil, err + } + result[key] = convertedValue + } + + return result, nil + case []any: + result := make([]any, len(v)) + for i, item := range v { + convertedItem, err := convertToJSONSafe(item) + if err != nil { + return nil, err + } + result[i] = convertedItem + } + + return result, nil + case float64: + if (v >= 1e15 || v <= -1e15) && v == math.Trunc(v) { + formatted := strconv.FormatFloat(v, 'f', 0, 64) + return json.Number(formatted), nil + } + + return v, nil + default: + return v, nil + } +} + +func yamlNodeToAny(node *yaml.Node) any { + if node == nil { + return nil + } + + switch node.Kind { + case yaml.DocumentNode: + if len(node.Content) == 0 { + return nil + } + + return yamlNodeToAny(node.Content[0]) + case yaml.MappingNode: + out := make(map[string]any, len(node.Content)/2) + for i := 0; i+1 < len(node.Content); i += 2 { + key := node.Content[i] + value := node.Content[i+1] + out[key.Value] = yamlNodeToAny(value) + } + + return out + case yaml.SequenceNode: + out := make([]any, 0, len(node.Content)) + for _, elem := range node.Content { + out = append(out, yamlNodeToAny(elem)) + } + + return out + case yaml.ScalarNode: + if node.Style == 0 && decimalInteger.MatchString(node.Value) { + return json.Number(node.Value) + } + + switch node.Tag { + case "!!int": + if decimalInteger.MatchString(node.Value) { + return json.Number(node.Value) + } + if n, ok := new(big.Int).SetString(strings.ReplaceAll(node.Value, "_", ""), 0); ok { + return json.Number(n.String()) + } + + return node.Value + case "!!float": + f, err := strconv.ParseFloat(node.Value, 64) + if err != nil { + return node.Value + } + + return f + case "!!null": + return nil + case "!!bool": + return strings.EqualFold(node.Value, "true") + default: + return node.Value + } + case yaml.AliasNode: + return yamlNodeToAny(node.Alias) + default: + return nil + } +} diff --git a/engine/cld/durablepipeline/yaml_test.go b/engine/cld/durablepipeline/yaml_test.go new file mode 100644 index 000000000..cf773f927 --- /dev/null +++ b/engine/cld/durablepipeline/yaml_test.go @@ -0,0 +1,258 @@ +package durablepipeline + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConvertToJSONSafe_NumberPreservation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input any + expectedType any + expectedJSON string + }{ + { + name: "large integer should be preserved as json.Number", + input: float64(2e+21), + expectedType: json.Number(""), + expectedJSON: "2000000000000000000000", + }, + { + name: "another large integer", + input: float64(1e+16), + expectedType: json.Number(""), + expectedJSON: "10000000000000000", + }, + { + name: "negative large integer", + input: float64(-5e+15), + expectedType: json.Number(""), + expectedJSON: "-5000000000000000", + }, + { + name: "normal integer should stay as float64", + input: float64(123), + expectedType: float64(0), + expectedJSON: "123", + }, + { + name: "normal float should stay as float64", + input: float64(3.14), + expectedType: float64(0), + expectedJSON: "3.14", + }, + { + name: "threshold boundary - exactly 1e15", + input: float64(1e15), + expectedType: json.Number(""), + expectedJSON: "1000000000000000", + }, + { + name: "just under threshold", + input: float64(9.99999e14), + expectedType: float64(0), + expectedJSON: "999999000000000", + }, + { + name: "string should pass through unchanged", + input: "hello world", + expectedType: "", + expectedJSON: `"hello world"`, + }, + { + name: "boolean should pass through unchanged", + input: true, + expectedType: true, + expectedJSON: "true", + }, + { + name: "nested map with large number", + input: map[string]any{ + "bigInt": float64(2e+21), + "normalInt": float64(123), + "message": "test", + }, + expectedType: map[string]any{}, + expectedJSON: `{"bigInt":2000000000000000000000,"message":"test","normalInt":123}`, + }, + { + name: "array with large numbers", + input: []any{float64(2e+21), float64(123), "test"}, + expectedType: []any{}, + expectedJSON: `[2000000000000000000000,123,"test"]`, + }, + { + name: "map with interface{} keys", + input: map[interface{}]any{ + "bigInt": float64(2e+21), + uint64(1601528660175782575): "ethereum-sepolia", + }, + expectedType: map[string]any{}, + expectedJSON: `{"1601528660175782575":"ethereum-sepolia","bigInt":2000000000000000000000}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + result, err := convertToJSONSafe(tt.input) + require.NoError(t, err) + require.IsType(t, tt.expectedType, result) + + jsonBytes, err := json.Marshal(result) + require.NoError(t, err) + require.JSONEq(t, tt.expectedJSON, string(jsonBytes)) + }) + } +} + +func TestFindChangesetInData(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + changesets any + changesetName string + expectError bool + expectedData any + errorContains string + }{ + { + name: "array format - changeset found", + changesets: []any{ + map[string]any{ + "test_changeset": map[string]any{ + "payload": map[string]any{"value": 123}, + }, + }, + map[string]any{ + "other_changeset": map[string]any{ + "payload": map[string]any{"value": 456}, + }, + }, + }, + changesetName: "test_changeset", + expectedData: map[string]any{ + "payload": map[string]any{"value": 123}, + }, + }, + { + name: "array format - changeset not found", + changesets: []any{ + map[string]any{ + "other_changeset": map[string]any{ + "payload": map[string]any{"value": 123}, + }, + }, + }, + changesetName: "test_changeset", + expectError: true, + }, + { + name: "array format - empty", + changesets: []any{}, + changesetName: "test_changeset", + expectError: true, + }, + { + name: "object format - should be rejected", + changesets: map[string]any{ + "test_changeset": map[string]any{ + "payload": map[string]any{"value": 123}, + }, + }, + changesetName: "test_changeset", + expectError: true, + errorContains: "expected array format", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + result, err := FindChangesetInData(tt.changesets, tt.changesetName) + if tt.expectError { + require.Error(t, err) + if tt.errorContains != "" { + require.ErrorContains(t, err, tt.errorContains) + } + + return + } + require.NoError(t, err) + require.Equal(t, tt.expectedData, result) + }) + } +} + +func TestGetAllChangesetsInOrder(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + changesets any + expectedNames []string + expectError bool + errorContains string + }{ + { + name: "object format - should return error", + changesets: map[string]any{ + "first": map[string]any{"payload": map[string]any{"value": 1}}, + "second": map[string]any{"payload": map[string]any{"value": 2}}, + }, + expectError: true, + errorContains: "expected array format", + }, + { + name: "array format", + changesets: []any{ + map[string]any{ + "first": map[string]any{"payload": map[string]any{"value": 1}}, + }, + map[string]any{ + "second": map[string]any{"payload": map[string]any{"value": 2}}, + }, + }, + expectedNames: []string{"first", "second"}, + }, + { + name: "array with invalid item", + changesets: []any{ + "not a map", + }, + expectedNames: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + result, err := GetAllChangesetsInOrder(tt.changesets) + if tt.expectError { + require.Error(t, err) + require.ErrorContains(t, err, tt.errorContains) + + return + } + + require.NoError(t, err) + var actualNames []string + for _, changeset := range result { + actualNames = append(actualNames, changeset.Name) + } + if len(tt.expectedNames) == 0 { + require.Empty(t, actualNames) + return + } + require.Equal(t, tt.expectedNames, actualNames) + }) + } +} diff --git a/engine/cld/legacy/cli/commands/durable-pipelines_test.go b/engine/cld/legacy/cli/commands/durable-pipelines_test.go index fd1bc1d88..ed454c43c 100644 --- a/engine/cld/legacy/cli/commands/durable-pipelines_test.go +++ b/engine/cld/legacy/cli/commands/durable-pipelines_test.go @@ -19,6 +19,7 @@ import ( fdeployment "github.com/smartcontractkit/chainlink-deployments-framework/deployment" "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/changeset" "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/domain" + "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/durablepipeline" "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/environment" "github.com/smartcontractkit/chainlink-deployments-framework/experimental/analyzer" ) @@ -974,104 +975,6 @@ changesets: require.Equal(t, "200000000000000000111", maxFee.String()) } -func TestFindChangesetInData(t *testing.T) { - t.Parallel() - tests := []struct { - name string - changesets any - changesetName string - expectError bool - expectedData any - errorContains string - description string - }{ - { - name: "array format - changeset found", - changesets: []any{ - map[string]any{ - "test_changeset": map[string]any{ - "payload": map[string]any{"value": 123}, - }, - }, - map[string]any{ - "other_changeset": map[string]any{ - "payload": map[string]any{"value": 456}, - }, - }, - }, - changesetName: "test_changeset", - expectError: false, - expectedData: map[string]any{ - "payload": map[string]any{"value": 123}, - }, - description: "Should find changeset in array format", - }, - { - name: "array format - changeset not found", - changesets: []any{ - map[string]any{ - "other_changeset": map[string]any{ - "payload": map[string]any{"value": 123}, - }, - }, - }, - changesetName: "test_changeset", - expectError: true, - description: "Should return error when changeset not found in array format", - }, - { - name: "array format - empty", - changesets: []any{}, - changesetName: "test_changeset", - expectError: true, - description: "Should return error for empty array", - }, - { - name: "object format - should be rejected", - changesets: map[string]any{ - "test_changeset": map[string]any{ - "payload": map[string]any{"value": 123}, - }, - }, - changesetName: "test_changeset", - expectError: true, - errorContains: "expected array format", - description: "Should return error when object format is provided (no longer supported)", - }, - { - name: "invalid format - string", - changesets: "invalid", - changesetName: "test_changeset", - expectError: true, - description: "Should return error for invalid format", - }, - { - name: "invalid format - nil", - changesets: nil, - changesetName: "test_changeset", - expectError: true, - description: "Should return error for nil", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - result, err := findChangesetInData(tt.changesets, tt.changesetName, "test-file.yaml") - - if tt.expectError { - require.Error(t, err, tt.description) - if tt.errorContains != "" { - require.ErrorContains(t, err, tt.errorContains, tt.description) - } - } else { - require.NoError(t, err, tt.description) - require.Equal(t, tt.expectedData, result, tt.description) - } - }) - } -} - //nolint:paralleltest func TestSetDurablePipelineInputFromYAML_ArrayFormat(t *testing.T) { testDomain := domain.NewDomain(t.TempDir(), "test") @@ -1592,89 +1495,6 @@ changesets: } } -func TestGetAllChangesetsInOrder(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - changesets any - expectedNames []string - expectError bool - errorContains string - }{ - { - name: "object format - should return error", - changesets: map[string]any{ - "first": map[string]any{"payload": map[string]any{"value": 1}}, - "second": map[string]any{"payload": map[string]any{"value": 2}}, - }, - expectError: true, - errorContains: "expected array format", - }, - { - name: "array format", - changesets: []any{ - map[string]any{ - "first": map[string]any{"payload": map[string]any{"value": 1}}, - }, - map[string]any{ - "second": map[string]any{"payload": map[string]any{"value": 2}}, - }, - }, - expectedNames: []string{"first", "second"}, - }, - { - name: "invalid format - string", - changesets: "invalid", - expectError: true, - errorContains: "has invalid 'changesets' format", - }, - { - name: "invalid format - number", - changesets: 123, - expectError: true, - errorContains: "has invalid 'changesets' format", - }, - { - name: "array with invalid item", - changesets: []any{ - "not a map", - }, - expectedNames: []string{}, // Should skip invalid items - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - result, err := getAllChangesetsInOrder(tt.changesets, "test-file.yaml") - - if tt.expectError { - require.Error(t, err) - require.Contains(t, err.Error(), tt.errorContains) - - return - } - - require.NoError(t, err) - - // Extract names from result - var actualNames []string - for _, changeset := range result { - actualNames = append(actualNames, changeset.name) - } - - // Check if the expected names match - if len(tt.expectedNames) > 0 { - require.Equal(t, tt.expectedNames, actualNames) - } else { - require.Empty(t, actualNames) - } - }) - } -} - //nolint:paralleltest // This test uses os.Chdir which changes global state func TestParseDurablePipelineYAML(t *testing.T) { testDomain := domain.NewDomain(t.TempDir(), "test") @@ -1800,7 +1620,7 @@ func TestParseDurablePipelineYAML_YamlNodeConversion(t *testing.T) { expectError bool errorContains string expectedJSON string - assertResult func(t *testing.T, result *durablePipelineYAML) + assertResult func(t *testing.T, result *durablepipeline.ParsedYAML) }{ { name: "preserves dynamic payload fields and large integers", @@ -1824,7 +1644,7 @@ changesets: 16015286601757825753: maxgas: 5000000`, expectedJSON: `[{"test_changeset":{"payload":{"chainconfig":{"16015286601757825753":{"maxgas":5000000}},"enabled":true,"feequoterparams":{"defaulttokenfeeusdcents":1200,"maxfeejuelspermsg":200000000000000000001,"premiummultiplierwei":1000000000000000000000000},"multiplier":1.25,"note":"hello-world","optional":null,"tags":["alpha","beta"]}}}]`, - assertResult: func(t *testing.T, result *durablePipelineYAML) { + assertResult: func(t *testing.T, result *durablepipeline.ParsedYAML) { t.Helper() changesets, ok := result.Changesets.([]any) diff --git a/engine/cld/legacy/cli/commands/durable_pipeline_yaml.go b/engine/cld/legacy/cli/commands/durable_pipeline_yaml.go index ba9c9e170..80ed482fe 100644 --- a/engine/cld/legacy/cli/commands/durable_pipeline_yaml.go +++ b/engine/cld/legacy/cli/commands/durable_pipeline_yaml.go @@ -1,29 +1,13 @@ package commands import ( - "encoding/json" "fmt" - "math" - "math/big" "os" - "regexp" - "strconv" - "strings" - - "gopkg.in/yaml.v3" "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/domain" + "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/durablepipeline" ) -var decimalInteger = regexp.MustCompile(`^-?(0|[1-9][0-9]*)$`) - -// durablePipelineYAML represents the structure of a durable pipeline YAML input file -type durablePipelineYAML struct { - Environment string `yaml:"environment"` - Domain string `yaml:"domain"` - Changesets any `yaml:"changesets"` // Must be []any (array format) -} - // setDurablePipelineInputFromYAML reads a YAML file, extracts the payload for the specified changeset, // and sets it as the DURABLE_PIPELINE_INPUT environment variable in JSON format. // If inputFileName is just a filename (no path separators), it will be resolved relative to the @@ -34,121 +18,12 @@ func setDurablePipelineInputFromYAML(inputFileName, changesetName string, domain return err } - changesetData, err := findChangesetInData(dpYAML.Changesets, changesetName, inputFileName) + changesetData, err := durablepipeline.FindChangesetInData(dpYAML.Changesets, changesetName) if err != nil { - return err + return fmt.Errorf("input file %s: %w", inputFileName, err) } - // Use the shared logic to set the environment variable - return setChangesetEnvironmentVariable(changesetName, changesetData, inputFileName) -} - -// findChangesetInData finds a changeset in array format -func findChangesetInData(changesets any, changesetName, inputFileName string) (any, error) { - // Array format: [{"changeset1": {...}}, {"changeset2": {...}}] - data, ok := changesets.([]any) - if !ok { - return nil, fmt.Errorf("input file %s has invalid 'changesets' format, expected array format", inputFileName) - } - - if len(data) == 0 { - return nil, fmt.Errorf("input file %s has empty 'changesets' array", inputFileName) - } - - // Search through array items for the changeset - for _, item := range data { - if itemMap, ok := item.(map[string]any); ok { - if changesetData, exists := itemMap[changesetName]; exists { - return changesetData, nil - } - } - } - - return nil, fmt.Errorf("changeset '%s' not found in input file %s", changesetName, inputFileName) -} - -// convertToJSONSafe recursively converts map[interface{}]interface{} to map[string]any -// and handles other YAML types that need conversion for JSON marshaling. -// This is because the JSON marshaling library does not support map[interface{}]interface{}. -func convertToJSONSafe(data any) (any, error) { - switch v := data.(type) { - case map[interface{}]interface{}: - // Convert map[interface{}]interface{} to map[string]any - result := make(map[string]any) - for key, value := range v { - // Convert key to string - handle both string and numeric keys - var keyStr string - switch k := key.(type) { - case string: - keyStr = k - case int: - keyStr = strconv.Itoa(k) - case int64: - keyStr = strconv.FormatInt(k, 10) - case uint64: - keyStr = strconv.FormatUint(k, 10) - case float64: - keyStr = strconv.FormatFloat(k, 'f', -1, 64) - default: - keyStr = fmt.Sprintf("%v", k) - } - - // Recursively convert the value - convertedValue, err := convertToJSONSafe(value) - if err != nil { - return nil, err - } - result[keyStr] = convertedValue - } - - return result, nil - - case map[string]any: - // Already the right type, but recursively convert values - result := make(map[string]any) - for key, value := range v { - convertedValue, err := convertToJSONSafe(value) - if err != nil { - return nil, err - } - result[key] = convertedValue - } - - return result, nil - - case []any: - // Convert slice elements recursively - result := make([]any, len(v)) - for i, item := range v { - convertedItem, err := convertToJSONSafe(item) - if err != nil { - return nil, err - } - result[i] = convertedItem - } - - return result, nil - - case float64: - // Convert large numbers that would become scientific notation to json.Number - // as it can cause issues to big.Int when it tries to unmarshal it. - // Only convert if it's actually an integer (no fractional part) - if v >= 1e15 || v <= -1e15 { - // Check if this is truly an integer (no fractional part) - if v == math.Trunc(v) { - // This is a large integer that would be in scientific notation - // Convert to json.Number to preserve exact representation - formatted := strconv.FormatFloat(v, 'f', 0, 64) - return json.Number(formatted), nil - } - } - - return v, nil - - default: - // For primitive types (string, int, bool, etc.), return as-is - return v, nil - } + return durablepipeline.SetChangesetEnvironmentVariable(changesetName, changesetData, inputFileName) } // setDurablePipelineInputFromYAMLByIndex sets the DURABLE_PIPELINE_INPUT environment variable @@ -166,9 +41,9 @@ func setDurablePipelineInputFromYAMLByIndex(inputFileName string, index int, dom } // Get all changesets in order - changesets, err := getAllChangesetsInOrder(dpYAML.Changesets, inputFileName) + changesets, err := durablepipeline.GetAllChangesetsInOrder(dpYAML.Changesets) if err != nil { - return "", err + return "", fmt.Errorf("input file %s: %w", inputFileName, err) } if index < 0 || index >= len(changesets) { @@ -178,89 +53,15 @@ func setDurablePipelineInputFromYAMLByIndex(inputFileName string, index int, dom selectedChangeset := changesets[index] // Use the existing logic to set the environment variable - if err := setChangesetEnvironmentVariable(selectedChangeset.name, selectedChangeset.data, inputFileName); err != nil { + if err := durablepipeline.SetChangesetEnvironmentVariable(selectedChangeset.Name, selectedChangeset.Data, inputFileName); err != nil { return "", err } - return selectedChangeset.name, nil -} - -// setChangesetEnvironmentVariable sets the DURABLE_PIPELINE_INPUT environment variable -// from changeset data (shared logic for both by-name and by-index approaches) -func setChangesetEnvironmentVariable(changesetName string, changesetData any, inputFileName string) error { - // Convert changeset data to map to access fields - changesetMap, ok := changesetData.(map[string]any) - if !ok { - return fmt.Errorf("changeset '%s' in input file %s is not a valid object", changesetName, inputFileName) - } - - payload, payloadExists := changesetMap["payload"] - if !payloadExists { - return fmt.Errorf("changeset '%s' in input file %s is missing required 'payload' field", changesetName, inputFileName) - } - - // Convert payload to JSON-safe format to handle map[interface{}]interface{} types - jsonSafePayload, err := convertToJSONSafe(payload) - if err != nil { - return fmt.Errorf("failed to convert payload to JSON-safe format: %w", err) - } - - chainOverridesRaw, exists := changesetMap["chainOverrides"] - if exists && chainOverridesRaw != nil { - if chainOverridesList, ok := chainOverridesRaw.([]any); ok { - for _, override := range chainOverridesList { - switch v := override.(type) { - case int: - if v < 0 { - return fmt.Errorf("chain override value must be non-negative, got: %d", v) - } - case int64: - if v < 0 { - return fmt.Errorf("chain override value must be non-negative, got: %d", v) - } - case uint64: - // no need to do any checks here - case json.Number: - // yaml.Node conversion preserves integers as json.Number. - // Validate it's a non-negative integer without precision loss. - n, ok := new(big.Int).SetString(v.String(), 10) - if !ok { - return fmt.Errorf("chain override value must be an integer, got type %T with value: %v", override, override) - } - if n.Sign() < 0 { - return fmt.Errorf("chain override value must be non-negative, got: %s", v.String()) - } - default: - return fmt.Errorf("chain override value must be an integer, got type %T with value: %v", override, override) - } - } - } - } - - // Create the JSON structure that WithEnvInput expects - inputJSON := map[string]any{ - "payload": jsonSafePayload, - } - if exists { - inputJSON["chainOverrides"] = chainOverridesRaw - } - - // Convert to JSON - jsonData, err := json.Marshal(inputJSON) - if err != nil { - return fmt.Errorf("failed to marshal payload to JSON: %w", err) - } - - // Set the environment variable - if err := os.Setenv("DURABLE_PIPELINE_INPUT", string(jsonData)); err != nil { - return fmt.Errorf("failed to set DURABLE_PIPELINE_INPUT environment variable: %w", err) - } - - return nil + return selectedChangeset.Name, nil } -// parseDurablePipelineYAML parses and validates a durable pipeline YAML file (shared logic) -func parseDurablePipelineYAML(inputFileName string, domain domain.Domain, envKey string) (*durablePipelineYAML, error) { +// parseDurablePipelineYAML parses and validates a durable pipeline YAML file +func parseDurablePipelineYAML(inputFileName string, domain domain.Domain, envKey string) (*durablepipeline.ParsedYAML, error) { resolvedPath, err := resolveDurablePipelineYamlPath(inputFileName, domain, envKey) if err != nil { return nil, fmt.Errorf("failed to resolve input file path: %w", err) @@ -271,137 +72,10 @@ func parseDurablePipelineYAML(inputFileName string, domain domain.Domain, envKey return nil, fmt.Errorf("failed to read input file %s: %w", resolvedPath, err) } - var root yaml.Node - if err = yaml.Unmarshal(yamlData, &root); err != nil { + parsed, err := durablepipeline.ParseYAMLBytes(yamlData) + if err != nil { return nil, fmt.Errorf("failed to parse input file %s: %w", inputFileName, err) } - rootMap, ok := yamlNodeToAny(&root).(map[string]any) - if !ok { - return nil, fmt.Errorf("failed to parse input file %s: expected a YAML object at the root", inputFileName) - } - - envRaw, hasEnv := rootMap["environment"] - domainRaw, hasDomain := rootMap["domain"] - changesetsRaw, hasChangesets := rootMap["changesets"] - - dpYAML := &durablePipelineYAML{ - Changesets: changesetsRaw, - } - if envStr, ok := envRaw.(string); ok { - dpYAML.Environment = envStr - } - if domainStr, ok := domainRaw.(string); ok { - dpYAML.Domain = domainStr - } - - if !hasEnv || dpYAML.Environment == "" { - return nil, fmt.Errorf("input file %s is missing required 'environment' field", inputFileName) - } - if !hasDomain || dpYAML.Domain == "" { - return nil, fmt.Errorf("input file %s is missing required 'domain' field", inputFileName) - } - if !hasChangesets || dpYAML.Changesets == nil { - return nil, fmt.Errorf("input file %s is missing required 'changesets' field", inputFileName) - } - - return dpYAML, nil -} - -func yamlNodeToAny(node *yaml.Node) any { - if node == nil { - return nil - } - - switch node.Kind { - case yaml.DocumentNode: - if len(node.Content) == 0 { - return nil - } - - return yamlNodeToAny(node.Content[0]) - case yaml.MappingNode: - out := make(map[string]any, len(node.Content)/2) - for i := 0; i+1 < len(node.Content); i += 2 { - key := node.Content[i] - value := node.Content[i+1] - out[key.Value] = yamlNodeToAny(value) - } - - return out - case yaml.SequenceNode: - out := make([]any, 0, len(node.Content)) - for _, elem := range node.Content { - out = append(out, yamlNodeToAny(elem)) - } - - return out - case yaml.ScalarNode: - // Plain decimal integers are preserved as JSON numbers, which allows - // downstream big.Int unmarshal without float64 precision loss. - if node.Style == 0 && decimalInteger.MatchString(node.Value) { - return json.Number(node.Value) - } - - switch node.Tag { - case "!!int": - if decimalInteger.MatchString(node.Value) { - return json.Number(node.Value) - } - if n, ok := new(big.Int).SetString(strings.ReplaceAll(node.Value, "_", ""), 0); ok { - return json.Number(n.String()) - } - - return node.Value - case "!!float": - f, err := strconv.ParseFloat(node.Value, 64) - if err != nil { - return node.Value - } - - return f - case "!!null": - return nil - case "!!bool": - return strings.EqualFold(node.Value, "true") - default: - return node.Value - } - case yaml.AliasNode: - return yamlNodeToAny(node.Alias) - default: - return nil - } -} - -// getAllChangesetsInOrder returns all changesets in order from array format changesets data -// This function only supports array format, not object format -func getAllChangesetsInOrder(changesets any, inputFileName string) ([]struct { - name string - data any -}, error) { - var result []struct { - name string - data any - } - - // Only support array format for index-based access - data, ok := changesets.([]any) - if !ok { - return nil, fmt.Errorf("input file %s has invalid 'changesets' format for index access, expected array format", inputFileName) - } - - // Array format: [{"changeset1": {...}}, {"changeset2": {...}}] - for _, item := range data { - if itemMap, ok := item.(map[string]any); ok { - for name, changesetData := range itemMap { - result = append(result, struct { - name string - data any - }{name, changesetData}) - } - } - } - - return result, nil + return parsed, nil } diff --git a/engine/cld/legacy/cli/commands/durable_pipeline_yaml_test.go b/engine/cld/legacy/cli/commands/durable_pipeline_yaml_test.go deleted file mode 100644 index 41a11349b..000000000 --- a/engine/cld/legacy/cli/commands/durable_pipeline_yaml_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package commands - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestConvertToJSONSafe_NumberPreservation(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - input any - expectedType any // Use actual type for comparison - expectedJSON string - description string - }{ - { - name: "large integer should be preserved as json.Number", - input: float64(2e+21), // This is how YAML parses 2000000000000000000000 - expectedType: json.Number(""), - expectedJSON: "2000000000000000000000", - description: "Large integers from YAML should preserve exact representation", - }, - { - name: "another large integer", - input: float64(1e+16), - expectedType: json.Number(""), - expectedJSON: "10000000000000000", - description: "Another large integer case", - }, - { - name: "negative large integer", - input: float64(-5e+15), - expectedType: json.Number(""), - expectedJSON: "-5000000000000000", - description: "Negative large integers should also be preserved", - }, - { - name: "normal integer should stay as float64", - input: float64(123), - expectedType: float64(0), - expectedJSON: "123", - description: "Small integers don't need special handling", - }, - { - name: "normal float should stay as float64", - input: float64(3.14), - expectedType: float64(0), - expectedJSON: "3.14", - description: "Regular floats should not be converted", - }, - { - name: "threshold boundary - exactly 1e15", - input: float64(1e15), - expectedType: json.Number(""), - expectedJSON: "1000000000000000", - description: "Numbers at the threshold should be converted", - }, - { - name: "just under threshold", - input: float64(9.99999e14), - expectedType: float64(0), - expectedJSON: "999999000000000", - description: "Numbers just under threshold should not be converted", - }, - { - name: "string should pass through unchanged", - input: "hello world", - expectedType: "", - expectedJSON: `"hello world"`, - description: "Non-numeric types should pass through", - }, - { - name: "boolean should pass through unchanged", - input: true, - expectedType: true, - expectedJSON: "true", - description: "Booleans should pass through unchanged", - }, - { - name: "nested map with large number", - input: map[string]any{ - "bigInt": float64(2e+21), - "normalInt": float64(123), - "message": "test", - }, - expectedType: map[string]any{}, - expectedJSON: `{"bigInt":2000000000000000000000,"message":"test","normalInt":123}`, - description: "Nested structures should preserve large numbers", - }, - { - name: "array with large numbers", - input: []any{float64(2e+21), float64(123), "test"}, - expectedType: []any{}, - expectedJSON: `[2000000000000000000000,123,"test"]`, - description: "Arrays should preserve large numbers", - }, - { - name: "map with interface{} keys (YAML parsing artifact)", - input: map[interface{}]any{ - "bigInt": float64(2e+21), - uint64(1601528660175782575): "ethereum-sepolia", // Chain selector as key (smaller number) - }, - expectedType: map[string]any{}, - expectedJSON: `{"1601528660175782575":"ethereum-sepolia","bigInt":2000000000000000000000}`, - description: "Should handle map[interface{}]any from YAML parsing", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - result, err := convertToJSONSafe(tt.input) - require.NoError(t, err, tt.description) - - // Check the type - require.IsType(t, tt.expectedType, result, "Result should be of expected type") - - // Marshal to JSON and check the output - jsonBytes, err := json.Marshal(result) - require.NoError(t, err, "Should be able to marshal result to JSON") - - require.JSONEq(t, tt.expectedJSON, string(jsonBytes), "JSON output should match expected") - }) - } -} diff --git a/engine/test/runtime/doc.go b/engine/test/runtime/doc.go index c68f31a9a..210b70345 100644 --- a/engine/test/runtime/doc.go +++ b/engine/test/runtime/doc.go @@ -222,6 +222,51 @@ // // The runtime provides specialized tasks for handling Multi-Chain Multi-Sig (MCMS) proposals. // +// # Registered Changesets From YAML +// +// Runtime can execute durable-pipeline changesets directly from YAML using a registry provider +// factory. This is useful for domain tests that already define a changeset registry (for example +// `domains///pipelines.go`) and want to run the exact same registered changesets in +// unit or integration tests. +// +// The input YAML must include: +// - `environment` +// - `domain` +// - `changesets` as an ordered array +// +// Each changeset entry should follow: +// +// changesets: +// - deploy_link_token: +// payload: +// chains: +// - linea_sepolia +// chainOverrides: [1, 2, 3] # optional +// +// Example: +// +// func TestExecuteRegisteredChangesetsFromYAML(t *testing.T) { +// rt, err := runtime.New(t.Context()) +// require.NoError(t, err) +// +// input := []byte(`environment: testnet +// domain: opdev +// changesets: +// - deploy_link_token: +// payload: +// chains: +// - linea_sepolia +// `) +// +// err = rt.ExecRegisteredChangesetsFromYAML( +// func() changeset.RegistryProvider { +// return testnet.NewPipelinesRegistryProvider() +// }, +// input, +// ) +// require.NoError(t, err) +// } +// // ## Available MCMS Tasks // // **SignProposalTask(proposalID, signingKeys...)** - Signs MCMS or Timelock proposals with diff --git a/engine/test/runtime/runtime_registered_changesets.go b/engine/test/runtime/runtime_registered_changesets.go new file mode 100644 index 000000000..d2a5b180c --- /dev/null +++ b/engine/test/runtime/runtime_registered_changesets.go @@ -0,0 +1,74 @@ +package runtime + +import ( + "errors" + "fmt" + + "github.com/segmentio/ksuid" + + "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/changeset" + "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/durablepipeline" +) + +// ExecRegisteredChangesetsFromYAML executes registered changesets from durable-pipeline YAML input. +// +// For each changeset entry in YAML order: +// 1. Build per-entry durable-pipeline JSON input from payload/chainOverrides. +// 2. Create and initialize a fresh registry provider. +// 3. Apply the named changeset against the current runtime environment with explicit input. +// 4. Merge output into runtime state and regenerate environment for the next step. +func (r *Runtime) ExecRegisteredChangesetsFromYAML( + providerFactory func() changeset.RegistryProvider, + inputYAML []byte, +) error { + r.mu.Lock() + defer r.mu.Unlock() + + if providerFactory == nil { + return errors.New("provider factory is required") + } + + parsed, err := durablepipeline.ParseYAMLBytes(inputYAML) + if err != nil { + return fmt.Errorf("failed to parse input file %s: %w", "runtime-input.yaml", err) + } + + ordered, err := durablepipeline.GetAllChangesetsInOrder(parsed.Changesets) + if err != nil { + return fmt.Errorf("input file %s: %w", "runtime-input.yaml", err) + } + if len(ordered) == 0 { + return fmt.Errorf("input file %s has empty 'changesets' array", "runtime-input.yaml") + } + + for _, cs := range ordered { + inputJSON, err := durablepipeline.BuildChangesetInputJSON(cs.Name, cs.Data) + if err != nil { + return fmt.Errorf("failed to build input for changeset %q in input file %s: %w", cs.Name, "runtime-input.yaml", err) + } + + provider := providerFactory() + if provider == nil { + return errors.New("provider factory returned nil") + } + if initErr := provider.Init(); initErr != nil { + return fmt.Errorf("failed to init registry provider: %w", initErr) + } + + out, err := provider.Registry().ApplyWithInput(cs.Name, r.currentEnv, inputJSON) + if err != nil { + return err + } + + if err := r.state.MergeChangesetOutput( + fmt.Sprintf("registered-%s-%s", cs.Name, ksuid.New().String()), + out, + ); err != nil { + return err + } + + r.currentEnv = r.generateNewEnvironment() + } + + return nil +} diff --git a/engine/test/runtime/runtime_registered_changesets_test.go b/engine/test/runtime/runtime_registered_changesets_test.go new file mode 100644 index 000000000..987c492f9 --- /dev/null +++ b/engine/test/runtime/runtime_registered_changesets_test.go @@ -0,0 +1,213 @@ +package runtime + +import ( + "testing" + + "github.com/stretchr/testify/require" + + fdeployment "github.com/smartcontractkit/chainlink-deployments-framework/deployment" + "github.com/smartcontractkit/chainlink-deployments-framework/engine/cld/changeset" +) + +type runtimeYAMLInput struct { + Value int `json:"value"` +} + +type runtimeResolverInput struct { + Base int `json:"base"` +} + +type recordedCall struct { + Name string + Value int +} + +type testRegistryProvider struct { + *changeset.BaseRegistryProvider + register func(registry *changeset.ChangesetsRegistry) +} + +func newTestRegistryProvider(register func(registry *changeset.ChangesetsRegistry)) *testRegistryProvider { + return &testRegistryProvider{ + BaseRegistryProvider: changeset.NewBaseRegistryProvider(), + register: register, + } +} + +func (p *testRegistryProvider) Init() error { + if p.register != nil { + p.register(p.Registry()) + } + + return nil +} + +func (p *testRegistryProvider) Archive() {} + +func makeInputChangeset(name string, out *[]recordedCall) fdeployment.ChangeSetV2[runtimeYAMLInput] { + return fdeployment.CreateChangeSet( + func(e fdeployment.Environment, cfg runtimeYAMLInput) (fdeployment.ChangesetOutput, error) { + *out = append(*out, recordedCall{Name: name, Value: cfg.Value}) + return fdeployment.ChangesetOutput{}, nil + }, + func(e fdeployment.Environment, cfg runtimeYAMLInput) error { + return nil + }, + ) +} + +func makeResolverChangeset(name string, out *[]recordedCall) fdeployment.ChangeSetV2[runtimeYAMLInput] { + return fdeployment.CreateChangeSet( + func(e fdeployment.Environment, cfg runtimeYAMLInput) (fdeployment.ChangesetOutput, error) { + *out = append(*out, recordedCall{Name: name, Value: cfg.Value}) + return fdeployment.ChangesetOutput{}, nil + }, + func(e fdeployment.Environment, cfg runtimeYAMLInput) error { + return nil + }, + ) +} + +//nolint:paralleltest // One subtest intentionally uses process env for precedence assertions. +func TestRuntime_ExecRegisteredChangesetsFromYAML(t *testing.T) { + t.Run("executes changesets in YAML order with per-entry input", func(t *testing.T) { + t.Parallel() + + rt, err := New(t.Context()) + require.NoError(t, err) + + var calls []recordedCall + providerFactory := func() changeset.RegistryProvider { + return newTestRegistryProvider(func(registry *changeset.ChangesetsRegistry) { + registry.SetValidate(false) + registry.Add( + "first", + changeset.Configure(makeInputChangeset("first", &calls)).WithEnvInput(), + ) + registry.Add( + "second", + changeset.Configure(makeInputChangeset("second", &calls)).WithEnvInput(), + ) + }) + } + + inputYAML := []byte(`environment: testnet +domain: opdev +changesets: + - first: + payload: + value: 1 + - second: + payload: + value: 2 +`) + + err = rt.ExecRegisteredChangesetsFromYAML(providerFactory, inputYAML) + require.NoError(t, err) + require.Equal(t, []recordedCall{ + {Name: "first", Value: 1}, + {Name: "second", Value: 2}, + }, calls) + require.Len(t, rt.State().Outputs, 2) + }) + + t.Run("supports resolver-backed registered changesets", func(t *testing.T) { + t.Parallel() + + rt, err := New(t.Context()) + require.NoError(t, err) + + var calls []recordedCall + providerFactory := func() changeset.RegistryProvider { + return newTestRegistryProvider(func(registry *changeset.ChangesetsRegistry) { + registry.SetValidate(false) + resolver := func(input runtimeResolverInput) (runtimeYAMLInput, error) { + return runtimeYAMLInput{Value: input.Base + 10}, nil + } + registry.Add( + "resolved", + changeset.Configure(makeResolverChangeset("resolved", &calls)). + WithConfigResolver(resolver), + ) + }) + } + + inputYAML := []byte(`environment: testnet +domain: opdev +changesets: + - resolved: + payload: + base: 7 +`) + + err = rt.ExecRegisteredChangesetsFromYAML(providerFactory, inputYAML) + require.NoError(t, err) + require.Equal(t, []recordedCall{ + {Name: "resolved", Value: 17}, + }, calls) + }) + + t.Run("returns error for invalid YAML changeset payload", func(t *testing.T) { + t.Parallel() + + rt, err := New(t.Context()) + require.NoError(t, err) + + inputYAML := []byte(`environment: testnet +domain: opdev +changesets: + - first: + notPayload: + value: 1 +`) + + err = rt.ExecRegisteredChangesetsFromYAML(func() changeset.RegistryProvider { + return newTestRegistryProvider(nil) + }, inputYAML) + require.ErrorContains(t, err, "is missing required 'payload' field") + }) + + t.Run("returns error when provider factory is nil", func(t *testing.T) { + t.Parallel() + + rt, err := New(t.Context()) + require.NoError(t, err) + + err = rt.ExecRegisteredChangesetsFromYAML(nil, []byte(`environment: testnet +domain: opdev +changesets: + - first: + payload: + value: 1 +`)) + require.ErrorContains(t, err, "provider factory is required") + }) + + t.Run("explicit YAML input takes precedence over env var", func(t *testing.T) { + t.Setenv("DURABLE_PIPELINE_INPUT", `{"payload":{"value":999}}`) + + rt, err := New(t.Context()) + require.NoError(t, err) + var calls []recordedCall + + providerFactory := func() changeset.RegistryProvider { + return newTestRegistryProvider(func(registry *changeset.ChangesetsRegistry) { + registry.SetValidate(false) + registry.Add( + "first", + changeset.Configure(makeInputChangeset("first", &calls)).WithEnvInput(), + ) + }) + } + + err = rt.ExecRegisteredChangesetsFromYAML(providerFactory, []byte(`environment: testnet +domain: opdev +changesets: + - first: + payload: + value: 1 +`)) + require.NoError(t, err) + require.Equal(t, []recordedCall{{Name: "first", Value: 1}}, calls) + }) +}