From 6b94291a2f51fc1c187b310f731b683e5f8aae74 Mon Sep 17 00:00:00 2001 From: Alexandr Kitaev Date: Thu, 23 Jul 2026 20:26:40 +0300 Subject: [PATCH] Add --ignore-unsupported-specs flag to validate command --- CHANGELOG.md | 1 + cmd/sloth/commands/validate.go | 54 +++++++++++++------ pkg/common/errors/errors.go | 4 ++ pkg/lib/gen.go | 3 +- .../validate_unsupported/good-sloth.yaml | 42 +++++++++++++++ .../unsupported-other-tool.yaml | 16 ++++++ test/integration/prometheus/validate_test.go | 15 ++++++ 7 files changed, 117 insertions(+), 18 deletions(-) create mode 100644 test/integration/prometheus/testdata/validate_unsupported/good-sloth.yaml create mode 100644 test/integration/prometheus/testdata/validate_unsupported/unsupported-other-tool.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 263ae17d..95625e79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- `validate`: Support ignoring discovered specs that are not Sloth supported spec types with `--ignore-unsupported-specs`. - `server`: Support HTTP request/response MCP with the `--mcp-enabled` and `--mcp-path` flags. - MCP: Add the `context`, `list_slos` and `get_slo` tools. diff --git a/cmd/sloth/commands/validate.go b/cmd/sloth/commands/validate.go index 1fbef826..c05fb6e2 100644 --- a/cmd/sloth/commands/validate.go +++ b/cmd/sloth/commands/validate.go @@ -2,6 +2,7 @@ package commands import ( "context" + "errors" "fmt" "io" "io/fs" @@ -30,6 +31,7 @@ type validateCommand struct { sloPlugins []string disableDefaultSLOPlugins bool ignoreSloDuplicates bool + ignoreUnsupportedSpecs bool } // NewValidateCommand returns the validate command. @@ -46,6 +48,7 @@ func NewValidateCommand(app *kingpin.Application) Command { cmd.Flag("slo-plugins", `SLO plugins chain declaration in JSON format '{"id": "foo","priority": 0,"config": "{}"}' (Can be repeated).`).Short('s').StringsVar(&c.sloPlugins) cmd.Flag("disable-default-slo-plugins", `Disables the default SLO plugins, normally used along with custom SLO plugins to fully customize Sloth behavior`).BoolVar(&c.disableDefaultSLOPlugins) cmd.Flag("ignore-slo-duplicates", "Flag to ignore SLO duplicates in specs (service and name used as an SLO/SLI identifier).").Default("false").BoolVar(&c.ignoreSloDuplicates) + cmd.Flag("ignore-unsupported-specs", "Flag to ignore the discovered specs that are not any of the supported Sloth spec types instead of failing (e.g other tools specs living in the same directory).").Default("false").BoolVar(&c.ignoreUnsupportedSpecs) return c } @@ -120,6 +123,7 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error { // For every file load the data and start the validation process: validations := []*fileValidation{} totalValidations := 0 + totalIgnoredSpecs := 0 sloIDs := make(map[string]string) for _, input := range sloPaths { // Get SLO spec data. @@ -136,8 +140,6 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error { validation := &fileValidation{File: input} validations = append(validations, validation) for _, data := range splittedSLOsData { - totalValidations++ - _ = data genTarget := generateTarget{ SLOData: data, Out: io.Discard, @@ -146,21 +148,33 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error { // Generate SLOs. sloGroupResult, err := genService.GenerateFromRaw(ctx, []byte(genTarget.SLOData)) if err != nil { + // The discovered files may contain specs that don't belong to Sloth (e.g other tools + // specs living in the same directory), on demand we ignore them instead of failing. + if v.ignoreUnsupportedSpecs && errors.Is(err, commonerrors.ErrUnsupportedSpecType) { + totalIgnoredSpecs++ + logger.WithValues(log.Kv{"file": validation.File}).Debugf("Ignored spec, not a Sloth supported spec type") + continue + } + + totalValidations++ validation.Errs = append(validation.Errs, fmt.Errorf("invalid SLO: %w", err)) - } else { - // Check for SLO duplicates - if !v.ignoreSloDuplicates { - for _, sloResult := range sloGroupResult.SLOResults { - slo := sloResult.SLO - if sloFile, exists := sloIDs[slo.ID]; !exists { - sloIDs[slo.ID] = validation.File - } else { - err := fmt.Errorf( - "SLO duplicated. SLO{service=%s, name=%s}, ID=%s already exists in a file: %s: %w", - slo.Service, slo.Name, slo.ID, sloFile, commonerrors.ErrAlreadyExists, - ) - validation.Errs = append(validation.Errs, err) - } + continue + } + + totalValidations++ + + // Check for SLO duplicates. + if !v.ignoreSloDuplicates { + for _, sloResult := range sloGroupResult.SLOResults { + slo := sloResult.SLO + if sloFile, exists := sloIDs[slo.ID]; !exists { + sloIDs[slo.ID] = validation.File + } else { + err := fmt.Errorf( + "SLO duplicated. SLO{service=%s, name=%s}, ID=%s already exists in a file: %s: %w", + slo.Service, slo.Name, slo.ID, sloFile, commonerrors.ErrAlreadyExists, + ) + validation.Errs = append(validation.Errs, err) } } } @@ -181,7 +195,13 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error { } } - logger.WithValues(log.Kv{"slo-specs": totalValidations}).Infof("Validation succeeded") + // All the discovered specs could have been ignored, we don't want to succeed silently on + // setups where the user thinks Sloth specs are being validated. + if totalValidations == 0 { + return fmt.Errorf("0 slo specs have been validated") + } + + logger.WithValues(log.Kv{"slo-specs": totalValidations, "ignored-specs": totalIgnoredSpecs}).Infof("Validation succeeded") return nil } diff --git a/pkg/common/errors/errors.go b/pkg/common/errors/errors.go index 506fc743..ae6267f8 100644 --- a/pkg/common/errors/errors.go +++ b/pkg/common/errors/errors.go @@ -15,4 +15,8 @@ var ( // ErrAlreadyExists will be used when a resource already exists. ErrAlreadyExists = fmt.Errorf("already exists") + + // ErrUnsupportedSpecType will be used when a spec doesn't match any of the known Sloth spec + // types. + ErrUnsupportedSpecType = fmt.Errorf("unsupported spec type") ) diff --git a/pkg/lib/gen.go b/pkg/lib/gen.go index d518e1db..9751d594 100644 --- a/pkg/lib/gen.go +++ b/pkg/lib/gen.go @@ -15,6 +15,7 @@ import ( k8stransformpromopv1 "github.com/slok/sloth/internal/plugin/k8stransform/prom_operator_prometheus_rule_v1" storagefs "github.com/slok/sloth/internal/storage/fs" storageio "github.com/slok/sloth/internal/storage/io" + commonerrors "github.com/slok/sloth/pkg/common/errors" "github.com/slok/sloth/pkg/common/model" utilsdata "github.com/slok/sloth/pkg/common/utils/data" kubernetesv1 "github.com/slok/sloth/pkg/kubernetes/api/sloth/v1" @@ -188,7 +189,7 @@ func (p PrometheusSLOGenerator) GenerateFromRaw(ctx context.Context, data []byte return p.GenerateFromOpenSLOV1Alpha(ctx, *apiSpec) default: - return nil, fmt.Errorf("invalid spec, could not load with any of the supported spec types") + return nil, fmt.Errorf("invalid spec, could not load with any of the supported spec types: %w", commonerrors.ErrUnsupportedSpecType) } } diff --git a/test/integration/prometheus/testdata/validate_unsupported/good-sloth.yaml b/test/integration/prometheus/testdata/validate_unsupported/good-sloth.yaml new file mode 100644 index 00000000..5454f71f --- /dev/null +++ b/test/integration/prometheus/testdata/validate_unsupported/good-sloth.yaml @@ -0,0 +1,42 @@ +version: "prometheus/v1" +service: "svc01" +labels: + global01k1: global01v1 +slos: + - name: "slo1" + objective: 99.9 + description: "This is SLO 01." + labels: + global02k1: global02v1 + sli: + events: + error_query: sum(rate(http_request_duration_seconds_count{job="myservice",code=~"(5..|429)"}[{{.window}}])) + total_query: sum(rate(http_request_duration_seconds_count{job="myservice"}[{{.window}}])) + alerting: + name: myServiceAlert + labels: + alert01k1: "alert01v1" + annotations: + alert02k1: "alert02k2" + pageAlert: + labels: + alert03k1: "alert03v1" + ticketAlert: + labels: + alert04k1: "alert04v1" + - name: "slo02" + objective: 95 + description: "This is SLO 02." + labels: + global03k1: global03v1 + sli: + raw: + error_ratio_query: | + sum(rate(http_request_duration_seconds_count{job="myservice",code=~"(5..|429)"}[{{.window}}])) + / + sum(rate(http_request_duration_seconds_count{job="myservice"}[{{.window}}])) + alerting: + page_alert: + disable: true + ticket_alert: + disable: true diff --git a/test/integration/prometheus/testdata/validate_unsupported/unsupported-other-tool.yaml b/test/integration/prometheus/testdata/validate_unsupported/unsupported-other-tool.yaml new file mode 100644 index 00000000..d87eaff9 --- /dev/null +++ b/test/integration/prometheus/testdata/validate_unsupported/unsupported-other-tool.yaml @@ -0,0 +1,16 @@ +# Spec that doesn't belong to Sloth (e.g other SLO tools specs living in the same directory). +apiVersion: pyrra.dev/v1alpha1 +kind: ServiceLevelObjective +metadata: + name: svc01-slo1 + labels: + prometheus: k8s +spec: + target: "99.9" + window: 30d + indicator: + ratio: + errors: + metric: http_requests_total{job="svc01",code=~"5.."} + total: + metric: http_requests_total{job="svc01"} diff --git a/test/integration/prometheus/validate_test.go b/test/integration/prometheus/validate_test.go index 911000fc..e4eaf703 100644 --- a/test/integration/prometheus/validate_test.go +++ b/test/integration/prometheus/validate_test.go @@ -49,6 +49,21 @@ func TestPrometheusValidate(t *testing.T) { valCmdArgs: "--input ./testdata/validate --fs-exclude bad --fs-include .*-aa.* --ignore-slo-duplicates", }, + "Discovery of specs with unsupported spec types should validate with failures.": { + valCmdArgs: "--input ./testdata/validate_unsupported", + expErr: true, + }, + + "Discovery of specs with unsupported spec types and ignore flag should validate correctly.": { + valCmdArgs: "--input ./testdata/validate_unsupported --ignore-unsupported-specs", + expErr: false, + }, + + "Discovery of only unsupported spec types with ignore flag should fail because 0 specs validated.": { + valCmdArgs: "--input ./testdata/validate_unsupported --fs-include unsupported --ignore-unsupported-specs", + expErr: true, + }, + "Discovery of specs with duplicates should validate with failures.": { valCmdArgs: "--input ./testdata/validate_with_duplicates", expErr: true,