Add --ignore-unsupported-specs flag to validate command

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Added
 
+- `validate`: Support ignoring discovered specs that are not Sloth supported spec types with `--ignore-unsupported-specs`.
 - `server`: Support HTTP request/response MCP with the `--mcp-enabled` and `--mcp-path` flags.
 - MCP: Add the `context`, `list_slos` and `get_slo` tools.
 

cmd/sloth/commands/validate.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"io/fs"
@@ -30,6 +31,7 @@ type validateCommand struct {
 	sloPlugins               []string
 	disableDefaultSLOPlugins bool
 	ignoreSloDuplicates      bool
+	ignoreUnsupportedSpecs   bool
 }
 
 // NewValidateCommand returns the validate command.
@@ -46,6 +48,7 @@ func NewValidateCommand(app *kingpin.Application) Command {
 	cmd.Flag("slo-plugins", `SLO plugins chain declaration in JSON format '{"id": "foo","priority": 0,"config": "{}"}' (Can be repeated).`).Short('s').StringsVar(&c.sloPlugins)
 	cmd.Flag("disable-default-slo-plugins", `Disables the default SLO plugins, normally used along with custom SLO plugins to fully customize Sloth behavior`).BoolVar(&c.disableDefaultSLOPlugins)
 	cmd.Flag("ignore-slo-duplicates", "Flag to ignore SLO duplicates in specs (service and name used as an SLO/SLI identifier).").Default("false").BoolVar(&c.ignoreSloDuplicates)
+	cmd.Flag("ignore-unsupported-specs", "Flag to ignore the discovered specs that are not any of the supported Sloth spec types instead of failing (e.g other tools specs living in the same directory).").Default("false").BoolVar(&c.ignoreUnsupportedSpecs)
 
 	return c
 }
@@ -120,6 +123,7 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error {
 	// For every file load the data and start the validation process:
 	validations := []*fileValidation{}
 	totalValidations := 0
+	totalIgnoredSpecs := 0
 	sloIDs := make(map[string]string)
 	for _, input := range sloPaths {
 		// Get SLO spec data.
@@ -136,8 +140,6 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error {
 		validation := &fileValidation{File: input}
 		validations = append(validations, validation)
 		for _, data := range splittedSLOsData {
-			totalValidations++
-			_ = data
 			genTarget := generateTarget{
 				SLOData: data,
 				Out:     io.Discard,
@@ -146,21 +148,33 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error {
 			// Generate SLOs.
 			sloGroupResult, err := genService.GenerateFromRaw(ctx, []byte(genTarget.SLOData))
 			if err != nil {
+				// The discovered files may contain specs that don't belong to Sloth (e.g other tools
+				// specs living in the same directory), on demand we ignore them instead of failing.
+				if v.ignoreUnsupportedSpecs && errors.Is(err, commonerrors.ErrUnsupportedSpecType) {
+					totalIgnoredSpecs++
+					logger.WithValues(log.Kv{"file": validation.File}).Debugf("Ignored spec, not a Sloth supported spec type")
+					continue
+				}
+
+				totalValidations++
 				validation.Errs = append(validation.Errs, fmt.Errorf("invalid SLO: %w", err))
-			} else {
-				// Check for SLO duplicates
-				if !v.ignoreSloDuplicates {
-					for _, sloResult := range sloGroupResult.SLOResults {
-						slo := sloResult.SLO
-						if sloFile, exists := sloIDs[slo.ID]; !exists {
-							sloIDs[slo.ID] = validation.File
-						} else {
-							err := fmt.Errorf(
-								"SLO duplicated. SLO{service=%s, name=%s}, ID=%s already exists in a file: %s: %w",
-								slo.Service, slo.Name, slo.ID, sloFile, commonerrors.ErrAlreadyExists,
-							)
-							validation.Errs = append(validation.Errs, err)
-						}
+				continue
+			}
+
+			totalValidations++
+
+			// Check for SLO duplicates.
+			if !v.ignoreSloDuplicates {
+				for _, sloResult := range sloGroupResult.SLOResults {
+					slo := sloResult.SLO
+					if sloFile, exists := sloIDs[slo.ID]; !exists {
+						sloIDs[slo.ID] = validation.File
+					} else {
+						err := fmt.Errorf(
+							"SLO duplicated. SLO{service=%s, name=%s}, ID=%s already exists in a file: %s: %w",
+							slo.Service, slo.Name, slo.ID, sloFile, commonerrors.ErrAlreadyExists,
+						)
+						validation.Errs = append(validation.Errs, err)
 					}
 				}
 			}
@@ -181,7 +195,13 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error {
 		}
 	}
 
-	logger.WithValues(log.Kv{"slo-specs": totalValidations}).Infof("Validation succeeded")
+	// All the discovered specs could have been ignored, we don't want to succeed silently on
+	// setups where the user thinks Sloth specs are being validated.
+	if totalValidations == 0 {
+		return fmt.Errorf("0 slo specs have been validated")
+	}
+
+	logger.WithValues(log.Kv{"slo-specs": totalValidations, "ignored-specs": totalIgnoredSpecs}).Infof("Validation succeeded")
 	return nil
 }
 

pkg/common/errors/errors.go

@@ -15,4 +15,8 @@ var (
 
 	// ErrAlreadyExists will be used when a resource already exists.
 	ErrAlreadyExists = fmt.Errorf("already exists")
+
+	// ErrUnsupportedSpecType will be used when a spec doesn't match any of the known Sloth spec
+	// types.
+	ErrUnsupportedSpecType = fmt.Errorf("unsupported spec type")
 )

pkg/lib/gen.go

@@ -15,6 +15,7 @@ import (
 	k8stransformpromopv1 "github.com/slok/sloth/internal/plugin/k8stransform/prom_operator_prometheus_rule_v1"
 	storagefs "github.com/slok/sloth/internal/storage/fs"
 	storageio "github.com/slok/sloth/internal/storage/io"
+	commonerrors "github.com/slok/sloth/pkg/common/errors"
 	"github.com/slok/sloth/pkg/common/model"
 	utilsdata "github.com/slok/sloth/pkg/common/utils/data"
 	kubernetesv1 "github.com/slok/sloth/pkg/kubernetes/api/sloth/v1"
@@ -188,7 +189,7 @@ func (p PrometheusSLOGenerator) GenerateFromRaw(ctx context.Context, data []byte
 		return p.GenerateFromOpenSLOV1Alpha(ctx, *apiSpec)
 
 	default:
-		return nil, fmt.Errorf("invalid spec, could not load with any of the supported spec types")
+		return nil, fmt.Errorf("invalid spec, could not load with any of the supported spec types: %w", commonerrors.ErrUnsupportedSpecType)
 	}
 }
 

test/integration/prometheus/testdata/validate_unsupported/good-sloth.yaml

@@ -0,0 +1,42 @@
+version: "prometheus/v1"
+service: "svc01"
+labels:
+  global01k1: global01v1
+slos:
+  - name: "slo1"
+    objective: 99.9
+    description: "This is SLO 01."
+    labels:
+      global02k1: global02v1
+    sli:
+      events:
+        error_query: sum(rate(http_request_duration_seconds_count{job="myservice",code=~"(5..|429)"}[{{.window}}]))
+        total_query: sum(rate(http_request_duration_seconds_count{job="myservice"}[{{.window}}]))
+    alerting:
+      name: myServiceAlert
+      labels:
+        alert01k1: "alert01v1"
+      annotations:
+        alert02k1: "alert02k2"
+      pageAlert:
+        labels:
+          alert03k1: "alert03v1"
+      ticketAlert:
+        labels:
+          alert04k1: "alert04v1"
+  - name: "slo02"
+    objective: 95
+    description: "This is SLO 02."
+    labels:
+      global03k1: global03v1
+    sli:
+      raw:
+        error_ratio_query: |
+          sum(rate(http_request_duration_seconds_count{job="myservice",code=~"(5..|429)"}[{{.window}}]))
+          /
+          sum(rate(http_request_duration_seconds_count{job="myservice"}[{{.window}}]))
+    alerting:
+      page_alert:
+        disable: true
+      ticket_alert:
+        disable: true

test/integration/prometheus/testdata/validate_unsupported/unsupported-other-tool.yaml

@@ -0,0 +1,16 @@
+# Spec that doesn't belong to Sloth (e.g other SLO tools specs living in the same directory).
+apiVersion: pyrra.dev/v1alpha1
+kind: ServiceLevelObjective
+metadata:
+  name: svc01-slo1
+  labels:
+    prometheus: k8s
+spec:
+  target: "99.9"
+  window: 30d
+  indicator:
+    ratio:
+      errors:
+        metric: http_requests_total{job="svc01",code=~"5.."}
+      total:
+        metric: http_requests_total{job="svc01"}

test/integration/prometheus/validate_test.go

@@ -49,6 +49,21 @@ func TestPrometheusValidate(t *testing.T) {
 			valCmdArgs: "--input ./testdata/validate --fs-exclude bad --fs-include .*-aa.* --ignore-slo-duplicates",
 		},
 
+		"Discovery of specs with unsupported spec types should validate with failures.": {
+			valCmdArgs: "--input ./testdata/validate_unsupported",
+			expErr:     true,
+		},
+
+		"Discovery of specs with unsupported spec types and ignore flag should validate correctly.": {
+			valCmdArgs: "--input ./testdata/validate_unsupported --ignore-unsupported-specs",
+			expErr:     false,
+		},
+
+		"Discovery of only unsupported spec types with ignore flag should fail because 0 specs validated.": {
+			valCmdArgs: "--input ./testdata/validate_unsupported --fs-include unsupported --ignore-unsupported-specs",
+			expErr:     true,
+		},
+
 		"Discovery of specs with duplicates should validate with failures.": {
 			valCmdArgs: "--input ./testdata/validate_with_duplicates",
 			expErr:     true,