diff --git a/documentdb-kubectl-plugin/cmd/promote.go b/documentdb-kubectl-plugin/cmd/promote.go index e058adfe..cdb7d6a5 100644 --- a/documentdb-kubectl-plugin/cmd/promote.go +++ b/documentdb-kubectl-plugin/cmd/promote.go @@ -11,11 +11,14 @@ import ( "github.com/spf13/cobra" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + + "github.com/documentdb/documentdb-operator/api/preview" ) const ( @@ -31,6 +34,7 @@ type promoteOptions struct { targetCluster string targetContext string skipWait bool + failover bool waitTimeout time.Duration pollInterval time.Duration } @@ -55,6 +59,7 @@ func newPromoteCommand() *cobra.Command { cmd.Flags().StringVar(&opts.targetCluster, "target-cluster", opts.targetCluster, "Name of the cluster that should become primary (required)") cmd.Flags().StringVar(&opts.targetContext, "cluster-context", opts.targetContext, "Kubeconfig context for verifying member status (defaults to current context)") cmd.Flags().BoolVar(&opts.skipWait, "skip-wait", opts.skipWait, "Return immediately after submitting the promotion request") + cmd.Flags().BoolVar(&opts.failover, "failover", opts.failover, "Perform a failover promotion (may result in data loss)") cmd.Flags().DurationVar(&opts.waitTimeout, "wait-timeout", 10*time.Minute, "Maximum time to wait for the promotion to complete") cmd.Flags().DurationVar(&opts.pollInterval, "poll-interval", 10*time.Second, "Polling interval while waiting for the promotion to complete") @@ -148,11 +153,41 @@ func (o *promoteOptions) run(ctx context.Context, cmd *cobra.Command) error { func (o *promoteOptions) patchDocumentDB(ctx context.Context, dyn dynamic.Interface) error { gvr := schema.GroupVersionResource{Group: documentDBGVRGroup, Version: documentDBGVRVersion, Resource: documentDBGVRResource} + clusterReplicationPatch := map[string]any{ + "primary": o.targetCluster, + } + + // If failover is true, remove the old primary from clusterList + if o.failover { + unstructuredDoc, err := dyn.Resource(gvr).Namespace(o.namespace).Get(ctx, o.documentDBName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get DocumentDB %q: %w", o.documentDBName, err) + } + + var doc preview.DocumentDB + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredDoc.Object, &doc); err != nil { + return fmt.Errorf("failed to convert DocumentDB %q to typed object: %w", o.documentDBName, err) + } + + if doc.Spec.ClusterReplication == nil { + return fmt.Errorf("DocumentDB %q does not have clusterReplication configured", o.documentDBName) + } + + oldPrimary := doc.Spec.ClusterReplication.Primary + if oldPrimary != "" && len(doc.Spec.ClusterReplication.ClusterList) > 0 { + var newClusterList []preview.MemberCluster + for _, cluster := range doc.Spec.ClusterReplication.ClusterList { + if cluster.Name != oldPrimary { + newClusterList = append(newClusterList, cluster) + } + } + clusterReplicationPatch["clusterList"] = newClusterList + } + } + patch := map[string]any{ "spec": map[string]any{ - "clusterReplication": map[string]any{ - "primary": o.targetCluster, - }, + "clusterReplication": clusterReplicationPatch, }, } diff --git a/documentdb-kubectl-plugin/cmd/promote_test.go b/documentdb-kubectl-plugin/cmd/promote_test.go index 7e777756..b4a4258f 100644 --- a/documentdb-kubectl-plugin/cmd/promote_test.go +++ b/documentdb-kubectl-plugin/cmd/promote_test.go @@ -95,6 +95,88 @@ func TestPatchDocumentDB(t *testing.T) { } } +func TestPatchDocumentDBFailover(t *testing.T) { + t.Parallel() + gvr := documentDBGVR() + + namespace := defaultDocumentDBNamespace + docName := "sample" + + doc := newDocumentWithClusterList(docName, namespace, "cluster-a", "Ready", []string{"cluster-a", "cluster-b", "cluster-c"}) + + client := newFakeDynamicClient(doc.DeepCopy()) + + opts := &promoteOptions{ + documentDBName: docName, + namespace: namespace, + targetCluster: "cluster-b", + failover: true, + } + + if err := opts.patchDocumentDB(context.Background(), client); err != nil { + t.Fatalf("patchDocumentDB returned error: %v", err) + } + + patched, err := client.Resource(gvr).Namespace(namespace).Get(context.Background(), docName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to fetch patched document: %v", err) + } + + primary, _, err := unstructured.NestedString(patched.Object, "spec", "clusterReplication", "primary") + if err != nil { + t.Fatalf("failed to read patched primary: %v", err) + } + if primary != "cluster-b" { + t.Fatalf("expected primary cluster-b, got %q", primary) + } + + clusterList, _, err := unstructured.NestedSlice(patched.Object, "spec", "clusterReplication", "clusterList") + if err != nil { + t.Fatalf("failed to read patched clusterList: %v", err) + } + + // Verify old primary (cluster-a) was removed from clusterList + for _, cluster := range clusterList { + clusterMap, ok := cluster.(map[string]any) + if !ok { + continue + } + name, _, _ := unstructured.NestedString(clusterMap, "name") + if name == "cluster-a" { + t.Fatal("expected old primary cluster-a to be removed from clusterList") + } + } + + // Verify remaining clusters are still present + if len(clusterList) != 2 { + t.Fatalf("expected 2 clusters in clusterList after failover, got %d", len(clusterList)) + } +} + +func newDocumentWithClusterList(name, namespace, primary, phase string, clusters []string) *unstructured.Unstructured { + clusterList := make([]any, 0, len(clusters)) + for _, c := range clusters { + clusterList = append(clusterList, map[string]any{"name": c}) + } + + doc := &unstructured.Unstructured{Object: map[string]any{ + "spec": map[string]any{ + "clusterReplication": map[string]any{ + "primary": primary, + "clusterList": clusterList, + }, + }, + "status": map[string]any{ + "status": phase, + }, + }} + gvk := schema.GroupVersionKind{Group: documentDBGVRGroup, Version: documentDBGVRVersion, Kind: "DocumentDB"} + doc.SetGroupVersionKind(gvk) + doc.SetName(name) + doc.SetNamespace(namespace) + return doc +} + func setDocumentState(ctx context.Context, client dynamic.Interface, gvr schema.GroupVersionResource, namespace, name, primary, phase string) error { for { obj, err := client.Resource(gvr).Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) diff --git a/documentdb-kubectl-plugin/go.mod b/documentdb-kubectl-plugin/go.mod index 5a5574f9..a863b095 100644 --- a/documentdb-kubectl-plugin/go.mod +++ b/documentdb-kubectl-plugin/go.mod @@ -3,50 +3,90 @@ module github.com/documentdb/documentdb-operator/documentdb-kubectl-plugin go 1.25.7 require ( - github.com/spf13/cobra v1.9.1 - k8s.io/api v0.32.2 - k8s.io/apimachinery v0.32.2 - k8s.io/client-go v0.32.2 + github.com/documentdb/documentdb-operator v0.0.0 + github.com/spf13/cobra v1.10.2 + k8s.io/api v0.35.0 + k8s.io/apimachinery v0.35.0 + k8s.io/client-go v0.35.0 ) +replace github.com/documentdb/documentdb-operator => ../operator/src + require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudnative-pg/barman-cloud v0.4.1-0.20260108104508-ced266c145f5 // indirect + github.com/cloudnative-pg/cloudnative-pg v1.28.1 // indirect + github.com/cloudnative-pg/cnpg-i v0.3.1 // indirect + github.com/cloudnative-pg/machinery v0.3.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.23.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.22.4 // indirect + github.com/go-openapi/jsonreference v0.21.4 // indirect + github.com/go-openapi/swag v0.25.4 // indirect + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/fileutils v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/mangling v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/gnostic-models v0.7.1 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0 // indirect + github.com/lib/pq v1.11.1 // indirect + github.com/moby/spdystream v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.87.1 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.4 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/robfig/cron v1.2.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 // indirect - golang.org/x/time v0.7.0 // indirect - google.golang.org/protobuf v1.35.1 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/term v0.39.0 // indirect + golang.org/x/text v0.33.0 // indirect + golang.org/x/time v0.14.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect + google.golang.org/grpc v1.78.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.35.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e // indirect + k8s.io/utils v0.0.0-20260108192941-914a6e750570 // indirect + sigs.k8s.io/controller-runtime v0.22.4 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/documentdb-kubectl-plugin/go.sum b/documentdb-kubectl-plugin/go.sum index 64cf7b81..79414213 100644 --- a/documentdb-kubectl-plugin/go.sum +++ b/documentdb-kubectl-plugin/go.sum @@ -1,160 +1,240 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudnative-pg/barman-cloud v0.4.1-0.20260108104508-ced266c145f5 h1:wPB7VTNgTv6t9sl4QYOBakmVTqHnOdKUht7Q3aL+uns= +github.com/cloudnative-pg/barman-cloud v0.4.1-0.20260108104508-ced266c145f5/go.mod h1:qD0NtJOllNQbRB0MaleuHsZjFYaXtXfdg0HbFTbuHn0= +github.com/cloudnative-pg/cloudnative-pg v1.28.1 h1:HdOUWgFhta558uHfXeO/199qCApxaj5yi05x6nWNmgs= +github.com/cloudnative-pg/cloudnative-pg v1.28.1/go.mod h1:yhRa4GqJAjNd0tT9AiRgk1KdqLhMjo/JmGGoASRl2CU= +github.com/cloudnative-pg/cnpg-i v0.3.1 h1:fKj8NoToWI11HUL2UWYJBpkVzmaTvbs3kDMo7wQF8RU= +github.com/cloudnative-pg/cnpg-i v0.3.1/go.mod h1:glRDiJLJY51FY8ScJIv/OkaGJxFnojJkkNAqSy5XC6s= +github.com/cloudnative-pg/machinery v0.3.3 h1:CaqXqLTJH9RrVv3R/YU0NmFaI/F18HLg2JfH3mQLcDk= +github.com/cloudnative-pg/machinery v0.3.3/go.mod h1:RYAYlVKBF5pH4mg+Q8wHjNDyENV9ajbkG41zOEf8DEs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= -github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.22.4 h1:dZtK82WlNpVLDW2jlA1YCiVJFVqkED1MegOUy9kR5T4= +github.com/go-openapi/jsonpointer v0.22.4/go.mod h1:elX9+UgznpFhgBuaMQ7iu4lvvX1nvNsesQ3oxmYTw80= +github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmGjjySRCHtC8= +github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4= +github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= +github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= +github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= +github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= +github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= +github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= +github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= +github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= +github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= +github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= +github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= +github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= +github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= +github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= +github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= +github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= +github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= +github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= +github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= +github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= +github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= +github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= +github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= +github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= +github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= +github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= +github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0 h1:bMqrb3UHgHbP+PW9VwiejfDJU1R0PpXVZNMdeH8WYKI= +github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI= +github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= +github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= +github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.87.1 h1:wyKanf+IFdbIqbDNYGt+f1dabLErLWtBaxd0KaAx4aM= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.87.1/go.mod h1:WHiLZmOWVop/MoYvRD58LfnPeyE+dcITby/jQjg83Hw= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= +github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= +github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= +github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= -golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= +gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw= -k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y= -k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= -k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/client-go v0.32.2 h1:4dYCD4Nz+9RApM2b/3BtVvBHw54QjMFUl1OLcJG5yOA= -k8s.io/client-go v0.32.2/go.mod h1:fpZ4oJXclZ3r2nDOv+Ux3XcJutfrwjKTCHz2H3sww94= +k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= +k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= +k8s.io/apiextensions-apiserver v0.35.0 h1:3xHk2rTOdWXXJM+RDQZJvdx0yEOgC0FgQ1PlJatA5T4= +k8s.io/apiextensions-apiserver v0.35.0/go.mod h1:E1Ahk9SADaLQ4qtzYFkwUqusXTcaV2uw3l14aqpL2LU= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE= +k8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e h1:iW9ChlU0cU16w8MpVYjXk12dqQ4BPFBEgif+ap7/hqQ= +k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20260108192941-914a6e750570 h1:JT4W8lsdrGENg9W+YwwdLJxklIuKWdRm+BC+xt33FOY= +k8s.io/utils v0.0.0-20260108192941-914a6e750570/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= +sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= +sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/documentdb-playground/aks-fleet-deployment/deploy-fleet-bicep.sh b/documentdb-playground/aks-fleet-deployment/deploy-fleet-bicep.sh index eab2d86f..9900fc91 100755 --- a/documentdb-playground/aks-fleet-deployment/deploy-fleet-bicep.sh +++ b/documentdb-playground/aks-fleet-deployment/deploy-fleet-bicep.sh @@ -129,7 +129,7 @@ echo "Pods ($HUB_CLUSTER):" kubectl get pods -n cert-manager -o wide || true export REGISTRY="ghcr.io/kubefleet-dev/kubefleet" -export TAG=$(curl "https://api.github.com/repos/kubefleet-dev/kubefleet/tags" | jq -r '.[0].name') # Gets latest tag +export TAG="v0.2" # Install the helm chart for running Fleet agents on the hub cluster. helm upgrade --install hub-agent ./charts/hub-agent/ \ --set image.pullPolicy=Always \ @@ -156,7 +156,7 @@ fleetNetworkingDir=$(mktemp -d) git clone https://github.com/Azure/fleet-networking.git $fleetNetworkingDir pushd $fleetNetworkingDir # Set up HUB_CLUSTER as the hub -NETWORKING_TAG=$(curl "https://api.github.com/repos/Azure/fleet-networking/tags" | jq -r '.[0].name') # Gets latest tag +NETWORKING_TAG="v0.3.28" # Install the helm chart for running Fleet agents on the hub cluster. kubectl config use-context $HUB_CLUSTER @@ -233,6 +233,9 @@ else fi rm -f "$ALIASES_TMP" +echo "Tag the HUB/MEMBER cluster" +kubectl --context $HUB_CLUSTER label membercluster $HUB_CLUSTER "documentdb.io/fleet-hub"=true + echo "" echo "✅ Deployment completed successfully!" echo "" diff --git a/documentdb-playground/aks-fleet-deployment/documentdb-resource-crp.yaml b/documentdb-playground/aks-fleet-deployment/documentdb-resource-crp.yaml index fdd53ce3..6b6df247 100644 --- a/documentdb-playground/aks-fleet-deployment/documentdb-resource-crp.yaml +++ b/documentdb-playground/aks-fleet-deployment/documentdb-resource-crp.yaml @@ -79,5 +79,19 @@ spec: name: documentdb-credentials policy: placementType: PickAll + affinity: + clusterAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + clusterSelectorTerms: + - labelSelector: + matchExpressions: + - key: documentdb.io/fleet-hub + operator: DoesNotExist + - labelSelector: + matchExpressions: + - key: documentdb.io/fleet-hub + operator: NotIn + values: + - "true" strategy: type: RollingUpdate diff --git a/documentdb-playground/aks-fleet-deployment/insert_read_test.py b/documentdb-playground/aks-fleet-deployment/insert_read_test.py new file mode 100644 index 00000000..3c62988c --- /dev/null +++ b/documentdb-playground/aks-fleet-deployment/insert_read_test.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import sys +import time +import uuid +from datetime import datetime, timezone +from pymongo import MongoClient +from pymongo.errors import PyMongoError + + +def build_connection_string(ip: str, password: str, port: int = 10260) -> str: + return f"mongodb://default_user:{password}@{ip}:{port}/?authMechanism=SCRAM-SHA-256&tls=true&tlsAllowInvalidCertificates=true" + + +def main() -> int: + if len(sys.argv) != 5: + print("Usage: python insert_read_test.py ") + return 1 + + insert_ip, read_ip_1, read_ip_2, password = sys.argv[1:5] + + insert_client = MongoClient(build_connection_string(insert_ip, password)) + read_client_1 = MongoClient(build_connection_string(read_ip_1, password)) + read_client_2 = MongoClient(build_connection_string(read_ip_2, password)) + + collection_name = f"testcollection_{uuid.uuid4().hex}" + insert_collection = insert_client.testdb[collection_name] + read_collection_1 = read_client_1.testdb[collection_name] + read_collection_2 = read_client_2.testdb[collection_name] + + print(f"Using collection: {collection_name}") + + print(f"{'Inserted Document':<30} {'Insert Count':<15} {'Read1 Count':<15} {'Read2 Count':<15}") + print("-" * 85) + + start_time = time.time() + end_time = start_time + (10 * 60) # 10 minutes + count = 0 + + while time.time() < end_time: + try: + doc = { + "count": count, + "message": f"Insert operation {count}", + "timestamp": datetime.now(timezone.utc), + } + result = insert_collection.insert_one(doc) + count += 1 + + read_count_1 = read_collection_1.count_documents({}) + read_count_2 = read_collection_2.count_documents({}) + + print(f"{str(result.inserted_id):<30} {count:<15} {read_count_1:<15} {read_count_2:<15}") + except PyMongoError as exc: + print(f"Mongo error: {exc}") + except Exception as exc: + print(f"Unexpected error: {exc}") + + time.sleep(1) + + print(f"Completed {count} insert operations in 10 minutes") + final_read_count_1 = read_collection_1.count_documents({}) + final_read_count_2 = read_collection_2.count_documents({}) + print(f"Final read count (read_ip_1): {final_read_count_1}") + print(f"Final read count (read_ip_2): {final_read_count_2}") + + insert_client.close() + read_client_1.close() + read_client_2.close() + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/documentdb-playground/bcdr-testing/README.md b/documentdb-playground/bcdr-testing/README.md new file mode 100644 index 00000000..8cc82f35 --- /dev/null +++ b/documentdb-playground/bcdr-testing/README.md @@ -0,0 +1,61 @@ +# BCDR Testing Playground + +This playground focuses on exercising DocumentDB Business Continuity and +Disaster Recovery (BCDR) scenarios. It builds on the AKS Fleet Deployment +playground and adds Chaos Mesh fault injection and a continuous insert/read +workload to measure downtime, data loss, and recovery behavior under HA and +regional failure conditions. + +## Contents + +- `deploy.sh`: Deploys a three-region AKS fleet (westus3, uksouth, eastus2), + installs cert-manager, the DocumentDB operator, a multi-region DocumentDB + cluster, and Chaos Mesh on every member cluster. +- `failure_insert_test.py`: Python script that continuously inserts documents + into the primary and reads from two replicas, tracking insert counts, read + counts, and downtime. +- `run_ha_failure_test.sh`: Runs the failure test and kills a primary cluster + pod to test HA failover time. (uses `ha-failure.yaml`) +- `run_regional_failure_test.sh`: Runs the failure test and kills all the pods + on the primary cluster, then initiates a regional failover (uses regional-failure.yaml) + +## Prerequisites + +- Azure CLI (`az`), `kubectl`, `jq`, `helm` +- Python 3 with packages from `requirements.txt` (`pip install -r requirements.txt`) +- An Azure subscription with permissions to create AKS clusters and DNS zones + +## Reading results + +Results will be of the form + +```text +Final read count (read_host_1): 100 +Final read count (read_host_2): 100 +Data loss (read_host_1): 0 +Data loss (read_host_2): 0 +Downtime (s): 11.08 +HA Primary Pod changed from documentdb-preview-bb8b4c62e10c285b-1 to documentdb-preview-bb8b4c62e10c285b-2 +``` + +The issues you'll want to note are downtimes that are too long: + +* More than 1 minute for HA +* More than 5 for Regional + +Any data loss. Ideally the count for missing inserts will always be zero. + +## Environment Variables + +| Variable | Default | Description | +| --- | --- | --- | +| `RESOURCE_GROUP` | `documentdb-bcdr-test-rg` | Azure resource group for the fleet | +| `DOCUMENTDB_NAME` | `documentdb-preview` | Name of the DocumentDB custom resource | +| `DOCUMENTDB_NAMESPACE` | `documentdb-preview-ns` | Kubernetes namespace for DocumentDB | +| `TOTAL_DURATION_SECONDS` | `60` (HA) / `360` (regional) | How long the insert workload runs | +| `CHAOS_DELAY_SECONDS` | `15` (HA) / `30` (regional) | Seconds to wait before injecting chaos | +| `FAILOVER_DELAY_SECONDS` | `10` | REGIONAL ONLY Seconds to wait before performing regional failover | +| `USE_DNS_ENDPOINTS` | `false` | Use DNS SRV records instead of LoadBalancer IPs | +| `DNS_ZONE_FQDN` | *(empty)* | Azure DNS zone FQDN (required when `USE_DNS_ENDPOINTS=true`) | +| `HUB_REGION` | `westus3` | Hub region used for fleet management | +| `PRIMARY_CONTEXT` | *(auto-detected)* | Override the kubectl context of the primary cluster | diff --git a/documentdb-playground/bcdr-testing/deploy.sh b/documentdb-playground/bcdr-testing/deploy.sh new file mode 100755 index 00000000..5d01e7c3 --- /dev/null +++ b/documentdb-playground/bcdr-testing/deploy.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -euo pipefail + +export MEMBER_REGIONS="westus3,uksouth,eastus2" +export RESOURCE_GROUP="${RESOURCE_GROUP:-documentdb-bcdr-test-rg}" +SCRIPT_DIR="$(dirname "$0")" + +# Deploy the AKS fleet with three regions +$SCRIPT_DIR/../aks-fleet-deployment/deploy-fleet-bicep.sh +$SCRIPT_DIR/../aks-fleet-deployment/install-cert-manager.sh +$SCRIPT_DIR/../aks-fleet-deployment/install-documentdb-operator.sh +$SCRIPT_DIR/../aks-fleet-deployment/deploy-multi-region.sh + +# Install Chaos Mesh on each cluster +helm repo add chaos-mesh https://charts.chaos-mesh.org +MEMBER_CLUSTERS=$(az aks list -g "$RESOURCE_GROUP" -o json | jq -r '.[] | select(.name|startswith("member-")) | .name' | sort) +CLUSTER_ARRAY=($MEMBER_CLUSTERS) +for cluster in "${CLUSTER_ARRAY[@]}"; do + kubectl create ns chaos-mesh --context "$cluster" + # Default to /var/run/docker.sock + helm install chaos-mesh chaos-mesh/chaos-mesh \ + -n=chaos-mesh \ + --set dashboard.create=false \ + --version 2.8.1 \ + --kube-context "$cluster" +done \ No newline at end of file diff --git a/documentdb-playground/bcdr-testing/failure_insert_test.py b/documentdb-playground/bcdr-testing/failure_insert_test.py new file mode 100755 index 00000000..61797ae6 --- /dev/null +++ b/documentdb-playground/bcdr-testing/failure_insert_test.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +import argparse +import sys +import time +import uuid +from datetime import datetime, timezone + +from pymongo import MongoClient +from pymongo.errors import PyMongoError + + +def build_connection_string(host: str, username: str, password: str, port: int, use_srv=False) -> str: + if use_srv: + return ( + f"mongodb+srv://{username}:{password}@{host}/" + "?authMechanism=SCRAM-SHA-256&tls=true&tlsAllowInvalidCertificates=true" + ) + else: + return ( + f"mongodb://{username}:{password}@{host}:{port}/" + "?authMechanism=SCRAM-SHA-256&tls=true&tlsAllowInvalidCertificates=true" + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="failure insert/read test") + parser.add_argument("insert_host") + parser.add_argument("read_host_1") + parser.add_argument("read_host_2") + parser.add_argument("username") + parser.add_argument("password") + parser.add_argument("--use-srv", action="store_true", help="Use srv connection string") + parser.add_argument("--duration-seconds", type=int, default=600) + parser.add_argument("--sleep-seconds", type=float, default=0.2) + parser.add_argument("--port", type=int, default=10260) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + insert_client = MongoClient( + build_connection_string(args.insert_host, args.username, args.password, args.port, args.use_srv) + ) + read_client_1 = MongoClient( + build_connection_string(args.read_host_1, args.username, args.password, args.port) + ) + read_client_2 = MongoClient( + build_connection_string(args.read_host_2, args.username, args.password, args.port) + ) + + collection_name = f"testcollection_{uuid.uuid4().hex}" + insert_collection = insert_client.testdb[collection_name] + read_collection_1 = read_client_1.testdb[collection_name] + read_collection_2 = read_client_2.testdb[collection_name] + + print(f"Using collection: {collection_name}") + + start_time = time.time() + end_time = start_time + args.duration_seconds + + successful_inserts = 0 + last_success_time = None + last_success_before_failure = None + first_failure_time = None + recovery_time = None + + print(f"{'Inserted Document':<30} {'Insert Count':<15} {'Read1 Count':<15} {'Read2 Count':<15}") + print("-" * 85) + + while time.time() < end_time: + try: + now = datetime.now(timezone.utc) + doc = { + "count": successful_inserts, + "message": f"Insert operation {successful_inserts}", + "timestamp": now, + } + result = insert_collection.insert_one(doc) + successful_inserts += 1 + last_success_time = time.time() + if first_failure_time is not None and recovery_time is None: + recovery_time = last_success_time + + try: + read_count_1 = read_collection_1.count_documents({}) + read_count_2 = read_collection_2.count_documents({}) + except PyMongoError as exc: + print(f"Read error: {exc}") + read_count_1 = -1 + read_count_2 = -1 + + print( + f"{str(result.inserted_id):<30} {successful_inserts:<15} {read_count_1:<15} {read_count_2:<15}" + ) + except PyMongoError as exc: + if first_failure_time is None: + first_failure_time = time.time() + last_success_before_failure = last_success_time + print(f"Mongo error: {exc}") + except Exception as exc: + if first_failure_time is None: + first_failure_time = time.time() + last_success_before_failure = last_success_time + print(f"Unexpected error: {exc}") + + time.sleep(args.sleep_seconds) + + final_read_count_1 = read_collection_1.count_documents({}) + final_read_count_2 = read_collection_2.count_documents({}) + + print(f"Completed {successful_inserts} insert operations") + print(f"Final read count (read_host_1): {final_read_count_1}") + print(f"Final read count (read_host_2): {final_read_count_2}") + + data_loss_1 = max(0, successful_inserts - final_read_count_1) + data_loss_2 = max(0, successful_inserts - final_read_count_2) + print(f"Data loss (read_host_1): {data_loss_1}") + print(f"Data loss (read_host_2): {data_loss_2}") + + if first_failure_time is not None and recovery_time is not None and last_success_before_failure is not None: + downtime = recovery_time - last_success_before_failure + print(f"Downtime (s): {downtime:.2f}") + else: + print("Downtime (s): N/A") + + insert_client.close() + read_client_1.close() + read_client_2.close() + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/documentdb-playground/bcdr-testing/ha-failure.yaml b/documentdb-playground/bcdr-testing/ha-failure.yaml new file mode 100644 index 00000000..d9990a09 --- /dev/null +++ b/documentdb-playground/bcdr-testing/ha-failure.yaml @@ -0,0 +1,13 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: pod-kill-documentdb +spec: + action: pod-kill + mode: one + gracePeriod: 0 + selector: + namespaces: + - documentdb-preview-ns + labelSelectors: + "cnpg.io/instanceRole": "primary" diff --git a/documentdb-playground/bcdr-testing/regional-failure.yaml b/documentdb-playground/bcdr-testing/regional-failure.yaml new file mode 100644 index 00000000..bffc13c6 --- /dev/null +++ b/documentdb-playground/bcdr-testing/regional-failure.yaml @@ -0,0 +1,13 @@ +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: pod-failure-documentdb +spec: + action: pod-failure + mode: all + duration: "5m" + selector: + namespaces: + - documentdb-preview-ns + - documentdb-operator + - cnpg-system diff --git a/documentdb-playground/bcdr-testing/requirements.txt b/documentdb-playground/bcdr-testing/requirements.txt new file mode 100644 index 00000000..741cb86d --- /dev/null +++ b/documentdb-playground/bcdr-testing/requirements.txt @@ -0,0 +1,2 @@ +pymongo>=4.0 +dnspython>=2.0 diff --git a/documentdb-playground/bcdr-testing/run_ha_failure_test.sh b/documentdb-playground/bcdr-testing/run_ha_failure_test.sh new file mode 100755 index 00000000..8911f461 --- /dev/null +++ b/documentdb-playground/bcdr-testing/run_ha_failure_test.sh @@ -0,0 +1,136 @@ +#!/bin/bash +set -euo pipefail + +# Usage: +# RESOURCE_GROUP=... ./run_ha_failure_test.sh + +RESOURCE_GROUP="${RESOURCE_GROUP:-documentdb-bcdr-test-rg}" +DOCUMENTDB_NAME="${DOCUMENTDB_NAME:-documentdb-preview}" +DOCUMENTDB_NAMESPACE="${DOCUMENTDB_NAMESPACE:-documentdb-preview-ns}" +SERVICE_NAME="${SERVICE_NAME:-documentdb-service-documentdb-preview}" +CHAOS_NAMESPACE="${CHAOS_NAMESPACE:-chaos-mesh}" +TOTAL_DURATION_SECONDS="${TOTAL_DURATION_SECONDS:-60}" +CHAOS_DELAY_SECONDS="${CHAOS_DELAY_SECONDS:-15}" +PYTHON_BIN="${PYTHON_BIN:-python3}" +PRIMARY_CONTEXT="${PRIMARY_CONTEXT:-}" +USE_DNS_ENDPOINTS="${USE_DNS_ENDPOINTS:-false}" +DNS_ZONE_FQDN="${DNS_ZONE_FQDN:-}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHAOS_FILE="$SCRIPT_DIR/ha-failure.yaml" + +fail() { + echo "Error: $1" >&2 + exit 1 +} + +get_service_endpoint() { + local context="$1" + local ip + local host + ip=$(kubectl --context "$context" get svc "$SERVICE_NAME" -n "$DOCUMENTDB_NAMESPACE" -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + if [[ -z "$ip" ]]; then + host=$(kubectl --context "$context" get svc "$SERVICE_NAME" -n "$DOCUMENTDB_NAMESPACE" -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') + if [[ -n "$host" ]]; then + echo "$host" + return 0 + fi + else + echo "$ip" + return 0 + fi + + return 1 +} + +if ! command -v az >/dev/null 2>&1; then + fail "az is required" +fi +if ! command -v jq >/dev/null 2>&1; then + fail "jq is required" +fi +if ! command -v kubectl >/dev/null 2>&1; then + fail "kubectl is required" +fi + +MEMBER_CLUSTERS=$(az aks list -g "$RESOURCE_GROUP" -o json | jq -r '.[] | select(.name|startswith("member-")) | .name' | sort) +if [[ -z "$MEMBER_CLUSTERS" ]]; then + fail "No member-* clusters found in resource group $RESOURCE_GROUP" +fi +CLUSTER_ARRAY=($MEMBER_CLUSTERS) + +primary_context="$PRIMARY_CONTEXT" +if [[ -z "$primary_context" ]]; then + if kubectl --context "${CLUSTER_ARRAY[0]}" get documentdb "$DOCUMENTDB_NAME" -n "$DOCUMENTDB_NAMESPACE" >/dev/null 2>&1; then + primary_context=$(kubectl --context "${CLUSTER_ARRAY[0]}" get documentdb "$DOCUMENTDB_NAME" -n "$DOCUMENTDB_NAMESPACE" -o jsonpath='{.spec.clusterReplication.primary}') + fi +fi + +read_clusters=() +for cluster in "${CLUSTER_ARRAY[@]}"; do + if [[ "$cluster" != "$primary_context" ]]; then + read_clusters+=("$cluster") + fi +done + +if [[ ${#read_clusters[@]} -lt 2 ]]; then + fail "Need at least two non-primary clusters for read endpoints" +fi + +use_srv="" +if [[ "$USE_DNS_ENDPOINTS" == "true" ]]; then + if [[ -z "$DNS_ZONE_FQDN" ]]; then + fail "DNS_ZONE_FQDN must be set when USE_DNS_ENDPOINTS is true" + fi + insert_endpoint="${DNS_ZONE_FQDN}" + read_endpoint_1="${read_clusters[0]}.${DNS_ZONE_FQDN}" + read_endpoint_2="${read_clusters[1]}.${DNS_ZONE_FQDN}" + use_srv="--use-srv" +else + insert_endpoint=$(get_service_endpoint "$primary_context") || fail "Failed to resolve insert endpoint on $primary_context" + read_endpoint_1=$(get_service_endpoint "${read_clusters[0]}") || fail "Failed to resolve read endpoint on ${read_clusters[0]}" + read_endpoint_2=$(get_service_endpoint "${read_clusters[1]}") || fail "Failed to resolve read endpoint on ${read_clusters[1]}" +fi + +username=$(kubectl --context "$primary_context" get secret documentdb-credentials -n "$DOCUMENTDB_NAMESPACE" -o jsonpath='{.data.username}' | base64 -d 2>/dev/null || true) +password=$(kubectl --context "$primary_context" get secret documentdb-credentials -n "$DOCUMENTDB_NAMESPACE" -o jsonpath='{.data.password}' | base64 -d 2>/dev/null || true) +if [[ -z "$username" ]]; then + echo "Warning: username not found in secret, defaulting to default_user" >&2 + username="default_user" +fi +if [[ -z "$password" ]]; then + fail "Password not found in secret documentdb-credentials" +fi + +echo "Primary context: $primary_context" +echo "Read contexts: ${read_clusters[0]}, ${read_clusters[1]}" +if [[ "$USE_DNS_ENDPOINTS" == "true" ]]; then + echo "DNS zone: $DNS_ZONE_FQDN" +fi +echo "Insert endpoint: $insert_endpoint" +echo "Read endpoint 1: $read_endpoint_1" +echo "Read endpoint 2: $read_endpoint_2" +echo "Total duration (s): $TOTAL_DURATION_SECONDS" +echo "Chaos delay (s): $CHAOS_DELAY_SECONDS" + +initial_primary_pod=$(kubectl --context "$primary_context" get pods -n "$DOCUMENTDB_NAMESPACE" -l "cnpg.io/instanceRole"="primary" -o jsonpath='{.items[0].metadata.name}') + +echo "Starting workload..." +"$PYTHON_BIN" "$SCRIPT_DIR/failure_insert_test.py" \ + "$insert_endpoint" "$read_endpoint_1" "$read_endpoint_2" \ + "$username" "$password" \ + --duration-seconds "$TOTAL_DURATION_SECONDS" \ + $use_srv & +python_pid=$! + +sleep "$CHAOS_DELAY_SECONDS" + +echo "Applying chaos..." +kubectl --context "$primary_context" apply -n "$CHAOS_NAMESPACE" -f "$CHAOS_FILE" + +wait "$python_pid" + +final_primary_pod=$(kubectl --context "$primary_context" get pods -n "$DOCUMENTDB_NAMESPACE" -l "cnpg.io/instanceRole"="primary" -o jsonpath='{.items[0].metadata.name}') +echo "HA Primary Pod changed from $initial_primary_pod to $final_primary_pod" + +kubectl --context "$primary_context" delete -n "$CHAOS_NAMESPACE" -f "$CHAOS_FILE" diff --git a/documentdb-playground/bcdr-testing/run_regional_failure_test.sh b/documentdb-playground/bcdr-testing/run_regional_failure_test.sh new file mode 100755 index 00000000..82a9c67d --- /dev/null +++ b/documentdb-playground/bcdr-testing/run_regional_failure_test.sh @@ -0,0 +1,176 @@ +#!/bin/bash +set -euo pipefail + +# Usage: +# RESOURCE_GROUP=... ./run_ha_failure_test.sh + +RESOURCE_GROUP="${RESOURCE_GROUP:-documentdb-bcdr-test-rg}" +DOCUMENTDB_NAME="${DOCUMENTDB_NAME:-documentdb-preview}" +DOCUMENTDB_NAMESPACE="${DOCUMENTDB_NAMESPACE:-documentdb-preview-ns}" +SERVICE_NAME="${SERVICE_NAME:-documentdb-service-documentdb-preview}" +CHAOS_NAMESPACE="${CHAOS_NAMESPACE:-chaos-mesh}" +TOTAL_DURATION_SECONDS="${TOTAL_DURATION_SECONDS:-360}" +CHAOS_DELAY_SECONDS="${CHAOS_DELAY_SECONDS:-30}" +FAILOVER_DELAY_SECONDS="${FAILOVER_DELAY_SECONDS:-10}" +PYTHON_BIN="${PYTHON_BIN:-python3}" +PRIMARY_CONTEXT="${PRIMARY_CONTEXT:-}" +USE_DNS_ENDPOINTS="${USE_DNS_ENDPOINTS:-false}" +DNS_ZONE_FQDN="${DNS_ZONE_FQDN:-}" +HUB_REGION="${HUB_REGION:-westus3}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHAOS_FILE="$SCRIPT_DIR/regional-failure.yaml" + +fail() { + echo "Error: $1" >&2 + exit 1 +} + +get_service_endpoint() { + local context="$1" + local ip + local host + ip=$(kubectl --context "$context" get svc "$SERVICE_NAME" -n "$DOCUMENTDB_NAMESPACE" -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + if [[ -z "$ip" ]]; then + host=$(kubectl --context "$context" get svc "$SERVICE_NAME" -n "$DOCUMENTDB_NAMESPACE" -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') + if [[ -n "$host" ]]; then + echo "$host" + return 0 + fi + else + echo "$ip" + return 0 + fi + + return 1 +} + +if ! command -v az >/dev/null 2>&1; then + fail "az is required" +fi +if ! command -v jq >/dev/null 2>&1; then + fail "jq is required" +fi +if ! command -v kubectl >/dev/null 2>&1; then + fail "kubectl is required" +fi + +MEMBER_CLUSTERS=$(az aks list -g "$RESOURCE_GROUP" -o json | jq -r '.[] | select(.name|startswith("member-")) | .name' | sort) +if [[ -z "$MEMBER_CLUSTERS" ]]; then + fail "No member-* clusters found in resource group $RESOURCE_GROUP" +fi +CLUSTER_ARRAY=($MEMBER_CLUSTERS) + +primary_context="$PRIMARY_CONTEXT" +if [[ -z "$primary_context" ]]; then + if kubectl --context "${CLUSTER_ARRAY[0]}" get documentdb "$DOCUMENTDB_NAME" -n "$DOCUMENTDB_NAMESPACE" >/dev/null 2>&1; then + primary_context=$(kubectl --context "${CLUSTER_ARRAY[0]}" get documentdb "$DOCUMENTDB_NAME" -n "$DOCUMENTDB_NAMESPACE" -o jsonpath='{.spec.clusterReplication.primary}') + fi +fi + +read_clusters=() +hub_context="" +for cluster in "${CLUSTER_ARRAY[@]}"; do + if [[ "$cluster" != "$primary_context" ]]; then + read_clusters+=("$cluster") + fi + if [[ "$cluster" == *"$HUB_REGION"* ]]; then + hub_context="$cluster" + fi +done + +if [[ ${#read_clusters[@]} -lt 2 ]]; then + fail "Need at least two non-primary clusters for read endpoints" +fi + +use_srv="" +if [[ "$USE_DNS_ENDPOINTS" == "true" ]]; then + if [[ -z "$DNS_ZONE_FQDN" ]]; then + fail "DNS_ZONE_FQDN must be set when USE_DNS_ENDPOINTS is true" + fi + insert_endpoint="${DNS_ZONE_FQDN}" + read_endpoint_1="${read_clusters[0]}.${DNS_ZONE_FQDN}" + read_endpoint_2="${read_clusters[1]}.${DNS_ZONE_FQDN}" + use_srv="--use-srv" +else + insert_endpoint=$(get_service_endpoint "$primary_context") || fail "Failed to resolve insert endpoint on $primary_context" + read_endpoint_1=$(get_service_endpoint "${read_clusters[0]}") || fail "Failed to resolve read endpoint on ${read_clusters[0]}" + read_endpoint_2=$(get_service_endpoint "${read_clusters[1]}") || fail "Failed to resolve read endpoint on ${read_clusters[1]}" +fi + +username=$(kubectl --context "$primary_context" get secret documentdb-credentials -n "$DOCUMENTDB_NAMESPACE" -o jsonpath='{.data.username}' | base64 -d 2>/dev/null || true) +password=$(kubectl --context "$primary_context" get secret documentdb-credentials -n "$DOCUMENTDB_NAMESPACE" -o jsonpath='{.data.password}' | base64 -d 2>/dev/null || true) +if [[ -z "$username" ]]; then + echo "Warning: username not found in secret, defaulting to default_user" >&2 + username="default_user" +fi +if [[ -z "$password" ]]; then + fail "Password not found in secret documentdb-credentials" +fi + +# Create plugin and add to path +pushd "$SCRIPT_DIR/../../operator/src" +make build-kubectl-plugin +export PATH="$(pwd)/bin:$PATH" +popd + +echo "Primary context: $primary_context" +echo "Read contexts: ${read_clusters[0]}, ${read_clusters[1]}" +if [[ "$USE_DNS_ENDPOINTS" == "true" ]]; then + echo "DNS zone: $DNS_ZONE_FQDN" +fi +echo "Insert endpoint: $insert_endpoint" +echo "Read endpoint 1: $read_endpoint_1" +echo "Read endpoint 2: $read_endpoint_2" +echo "Total duration (s): $TOTAL_DURATION_SECONDS" +echo "Chaos delay (s): $CHAOS_DELAY_SECONDS" + +echo "Starting workload..." +"$PYTHON_BIN" "$SCRIPT_DIR/failure_insert_test.py" \ + "$insert_endpoint" "$read_endpoint_1" "$read_endpoint_2" \ + "$username" "$password" \ + --duration-seconds "$TOTAL_DURATION_SECONDS" \ + "$use_srv" & +python_pid=$! + +sleep "$CHAOS_DELAY_SECONDS" + +echo "Applying chaos..." +kubectl --context "$primary_context" apply -n "$CHAOS_NAMESPACE" -f "$CHAOS_FILE" + +sleep "$FAILOVER_DELAY_SECONDS" + +# Perform manual failover +kubectl documentdb promote \ + --documentdb documentdb-preview \ + --namespace documentdb-preview-ns \ + --hub-context $hub_context \ + --target-cluster ${read_clusters[0]} \ + --cluster-context ${read_clusters[0]} \ + --skip-wait \ + --failover + +if [[ "$USE_DNS_ENDPOINTS" == "true" ]]; then + az network dns record-set srv remove-record \ + --record-set-name "_mongodb._tcp" \ + --zone-name "$DNS_ZONE_FQDN" \ + --resource-group "$RESOURCE_GROUP" \ + --priority 0 \ + --weight 0 \ + --port 10260 \ + --target "$primary_context.$DNS_ZONE_FQDN" \ + --keep-empty-record-set > /dev/null + + az network dns record-set srv add-record \ + --record-set-name "_mongodb._tcp" \ + --zone-name "$DNS_ZONE_FQDN" \ + --resource-group "$RESOURCE_GROUP" \ + --priority 0 \ + --weight 0 \ + --port 10260 \ + --target "${read_clusters[0]}.$DNS_ZONE_FQDN" > /dev/null +fi + +wait "$python_pid" + +kubectl --context "$primary_context" delete -n "$CHAOS_NAMESPACE" -f "$CHAOS_FILE" diff --git a/operator/src/internal/controller/physical_replication.go b/operator/src/internal/controller/physical_replication.go index ea045c03..66af6fad 100644 --- a/operator/src/internal/controller/physical_replication.go +++ b/operator/src/internal/controller/physical_replication.go @@ -360,6 +360,15 @@ func (r *DocumentDBReconciler) TryUpdateCluster(ctx context.Context, current, de } func (r *DocumentDBReconciler) getPrimaryChangePatchOps(ctx context.Context, patchOps *[]util.JSONPatch, current, desired *cnpgv1.Cluster, documentdb *dbpreview.DocumentDB, replicationContext *util.ReplicationContext) (error, time.Duration) { + + // Remove old bootstrap method if present + if current.Spec.Bootstrap != nil { + *patchOps = append(*patchOps, util.JSONPatch{ + Op: util.JSON_PATCH_OP_REMOVE, + Path: util.JSON_PATCH_PATH_BOOTSTRAP, + }) + } + if current.Spec.ReplicaCluster.Primary == current.Spec.ReplicaCluster.Self { // Primary => replica // demote diff --git a/operator/src/internal/utils/constants.go b/operator/src/internal/utils/constants.go index e4e67267..c67da524 100644 --- a/operator/src/internal/utils/constants.go +++ b/operator/src/internal/utils/constants.go @@ -53,6 +53,7 @@ const ( JSON_PATCH_PATH_EXTERNAL_CLUSTERS = "/spec/externalClusters" JSON_PATCH_PATH_MANAGED_SERVICES = "/spec/managed/services/additional" JSON_PATCH_PATH_SYNCHRONOUS = "/spec/postgresql/synchronous" + JSON_PATCH_PATH_BOOTSTRAP = "/spec/bootstrap" // JSON Patch operations JSON_PATCH_OP_REPLACE = "replace"