Skip to content

[eno] Make eno-reconciler's concurrency configurable #603

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ruinan-liu merged 6 commits into from
May 18, 2026
+87 −87
Merged

[eno] Make eno-reconciler's concurrency configurable #603

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d4ca150
Initial commit for adding multiple concurrent runs
ruinan-liu May 16, 2026
0e54d61
Updating kind ordering and making maxreconcile configurable
ruinan-liu May 18, 2026
dd9b036
update comment
ruinan-liu May 18, 2026
5ce340b
move logging location
ruinan-liu May 18, 2026
9a981c6
Updating the flag
ruinan-liu May 18, 2026
4e6a594
Fix test
ruinan-liu May 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions cmd/eno-reconciler/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ func run() error {
flag.BoolVar(&recOpts.FailOpen, "fail-open", false, "Report that resources are reconciled once they've been seen, even if reconciliation failed. Overridden by individual resources with 'eno.azure.io/fail-open: true|false'")
flag.StringVar(&migratingFieldManagers, "migrating-field-managers", os.Getenv("MIGRATING_FIELD_MANAGERS"), "Comma-separated list of Kubernetes SSA field manager names to take ownership from during migrations")
flag.StringVar(&migratingFields, "migrating-fields", os.Getenv("MIGRATING_FIELDS"), "Comma-seperated list of fields Kubernetes fields(metadata.labels, spec, stringData...) to migrate the ownership to eno")
flag.IntVar(&recOpts.MaxConcurrentReconciles, "max-concurrent-reconciles", 1, "Maximum number of concurrent reconciles for the reconciliation controller")
mgrOpts.Bind(flag.CommandLine)
flag.Parse()

Expand Down Expand Up @@ -149,5 +150,25 @@ func run() error {
return fmt.Errorf("constructing reconciliation controller: %w", err)
}

logger.Info("reconciler configuration",
"debugLogging", debugLogging,
"remoteKubeconfigFile", remoteKubeconfigFile,
"remoteQPS", remoteQPS,
"timeout", recOpts.Timeout,
"readinessPollInterval", recOpts.ReadinessPollInterval,
"minReconcileInterval", recOpts.MinReconcileInterval,
"disableServerSideApply", recOpts.DisableServerSideApply,
"compositionLabelSelector", compositionSelector,
"compositionNamespace", compositionNamespace,
"resourceFilter", resourceFilter,
"namespaceCreationGracePeriod", namespaceCreationGracePeriod,
"namespaceCleanup", namespaceCleanup,
"failOpen", recOpts.FailOpen,
"migratingFieldManagers", migratingFieldManagers,
"migratingFields", migratingFields,
"maxConcurrentReconciles", recOpts.MaxConcurrentReconciles,
"enoBuildVersion", enoBuildVersion,
)

return mgr.Start(ctx)
}
55 changes: 30 additions & 25 deletions internal/controllers/reconciliation/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,21 +43,24 @@ type Options struct {
Timeout time.Duration
ReadinessPollInterval time.Duration
MinReconcileInterval time.Duration

MaxConcurrentReconciles int
}

type Controller struct {
client client.Client
writeBuffer *flowcontrol.ResourceSliceWriteBuffer
resourceClient *resource.Cache
resourceFilter cel.Program
timeout time.Duration
readinessPollInterval time.Duration
upstreamClient client.Client
minReconcileInterval time.Duration
disableSSA bool
failOpen bool
migratingFieldManagers []string
migratingFields []string
client client.Client
writeBuffer *flowcontrol.ResourceSliceWriteBuffer
resourceClient *resource.Cache
resourceFilter cel.Program
timeout time.Duration
readinessPollInterval time.Duration
upstreamClient client.Client
minReconcileInterval time.Duration
disableSSA bool
failOpen bool
migratingFieldManagers []string
migratingFields []string
maxConcurrentReconciles int
}

func New(mgr ctrl.Manager, opts Options) error {
Expand All @@ -74,18 +77,19 @@ func New(mgr ctrl.Manager, opts Options) error {
}

c := &Controller{
client: opts.Manager.GetClient(),
writeBuffer: opts.WriteBuffer,
resourceClient: cache,
resourceFilter: opts.ResourceFilter,
timeout: opts.Timeout,
readinessPollInterval: opts.ReadinessPollInterval,
upstreamClient: upstreamClient,
minReconcileInterval: opts.MinReconcileInterval,
disableSSA: opts.DisableServerSideApply,
failOpen: opts.FailOpen,
migratingFieldManagers: opts.MigratingFieldManagers,
migratingFields: opts.MigratingFields,
client: opts.Manager.GetClient(),
writeBuffer: opts.WriteBuffer,
resourceClient: cache,
resourceFilter: opts.ResourceFilter,
timeout: opts.Timeout,
readinessPollInterval: opts.ReadinessPollInterval,
upstreamClient: upstreamClient,
minReconcileInterval: opts.MinReconcileInterval,
disableSSA: opts.DisableServerSideApply,
failOpen: opts.FailOpen,
migratingFieldManagers: opts.MigratingFieldManagers,
migratingFields: opts.MigratingFields,
maxConcurrentReconciles: opts.MaxConcurrentReconciles,
}

return builder.TypedControllerManagedBy[resource.Request](mgr).
Expand All @@ -97,7 +101,8 @@ func New(mgr ctrl.Manager, opts Options) error {
//
// This rate limiter uses the same per-item rate limiter as the default, but without
// the additional shared/global/non-item-scoped limiter.
RateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[resource.Request](5*time.Millisecond, 1000*time.Second),
RateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[resource.Request](5*time.Millisecond, 1000*time.Second),
MaxConcurrentReconciles: c.maxConcurrentReconciles,
}).
WatchesRawSource(src).
Complete(c)
Expand Down
46 changes: 24 additions & 22 deletions internal/resource/kind_ordering.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,26 +13,28 @@ package resource
var managedCreateOrder = map[string]int{
"PriorityClass": -100,
"Namespace": -100,
"NetworkPolicy": -99,
"ResourceQuota": -99,
"LimitRange": -99,
"PodSecurityPolicy": -98,
"PodDisruptionBudget": -98,
"ServiceAccount": -97,
"Secret": -96,
"SecretList": -96,
"ConfigMap": -96,
"StorageClass": -95,
"PersistentVolume": -94,
"PersistentVolumeClaim": -93,
"CustomResourceDefinition": -92,
"ClusterRole": -91,
"ClusterRoleList": -91,
"ClusterRoleBinding": -91,
"ClusterRoleBindingList": -91,
"Role": -90,
"RoleList": -90,
"RoleBinding": -90,
"RoleBindingList": -90,
"Service": -89,
"NetworkPolicy": -100,
"ResourceQuota": -100,
"LimitRange": -100,
"PodSecurityPolicy": -100,
"PodDisruptionBudget": -100,
"ServiceAccount": -100,
"Secret": -100,
"SecretList": -100,
"ConfigMap": -100,
"StorageClass": -100,
"PersistentVolume": -100,
"PersistentVolumeClaim": -100,
"CustomResourceDefinition": -100,
"ClusterRole": -100,
"ClusterRoleList": -100,
"ClusterRoleBinding": -100,
"ClusterRoleBindingList": -100,
"Role": -100,
"RoleList": -100,
"RoleBinding": -100,
"RoleBindingList": -100,
"Service": -100,
// TO-DO: Cleanup this once ETCD will reconcile on its own
"EtcdCluster": -100,
}
52 changes: 12 additions & 40 deletions internal/resource/kind_ordering_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func TestManagedCreateOrderCoversExpectedKinds(t *testing.T) {
func TestManagedCreateOrderGroupRange(t *testing.T) {
for kind, grp := range managedCreateOrder {
assert.GreaterOrEqual(t, grp, -100, "kind %q group %d below minimum", kind, grp)
assert.LessOrEqual(t, grp, -81, "kind %q group %d above reserved max", kind, grp)
assert.LessOrEqual(t, grp, -60, "kind %q group %d above reserved max", kind, grp)
}
}

Expand Down Expand Up @@ -58,23 +58,23 @@ func TestNewResource_DefaultOrderingForManagedKind(t *testing.T) {
name: "managed kind without annotations gets defaults",
manifest: `{"apiVersion":"v1","kind":"Secret",
"metadata":{"name":"s","namespace":"default"}}`,
wantReadiness: -96,
wantDeletion: intPtr(96),
wantReadiness: -100,
wantDeletion: intPtr(100),
},
{
name: "user readiness annotation wins; deletion still defaulted",
manifest: `{"apiVersion":"v1","kind":"Secret",
"metadata":{"name":"s","namespace":"default",
"annotations":{"eno.azure.io/readiness-group":"5"}}}`,
wantReadiness: 5,
wantDeletion: intPtr(96),
wantDeletion: intPtr(100),
},
{
name: "user deletion annotation wins; readiness still defaulted",
manifest: `{"apiVersion":"v1","kind":"Secret",
"metadata":{"name":"s","namespace":"default",
"annotations":{"eno.azure.io/deletion-group":"10"}}}`,
wantReadiness: -96,
wantReadiness: -100,
wantDeletion: intPtr(10),
},
{
Expand Down Expand Up @@ -112,39 +112,11 @@ func TestNewResource_DefaultOrderingForManagedKind(t *testing.T) {

func intPtr(i int) *int { return &i }

func TestManagedOrderingPrecedence(t *testing.T) {
// Namespace/PriorityClass (-100) before everything
assert.Less(t, managedCreateOrder["Namespace"], managedCreateOrder["ServiceAccount"])
// ServiceAccount (-97) before Secret/ConfigMap (-96)
assert.Less(t, managedCreateOrder["ServiceAccount"], managedCreateOrder["Secret"])
assert.Less(t, managedCreateOrder["ServiceAccount"], managedCreateOrder["ConfigMap"])
// StorageClass (-95) before PV (-94) before PVC (-93)
assert.Less(t, managedCreateOrder["StorageClass"], managedCreateOrder["PersistentVolume"])
assert.Less(t, managedCreateOrder["PersistentVolume"], managedCreateOrder["PersistentVolumeClaim"])
// PVC (-93) before CRD (-92)
assert.Less(t, managedCreateOrder["PersistentVolumeClaim"], managedCreateOrder["CustomResourceDefinition"])
// CRD (-92) before ClusterRole (-91)
assert.Less(t, managedCreateOrder["CustomResourceDefinition"], managedCreateOrder["ClusterRole"])
// ClusterRole (-91) before Role (-90)
assert.Less(t, managedCreateOrder["ClusterRole"], managedCreateOrder["Role"])
// Role (-90) before Service (-89)
assert.Less(t, managedCreateOrder["Role"], managedCreateOrder["Service"])
}

func TestManagedOrderingDeletionPrecedence(t *testing.T) {
// Deletion group is negation of create group, so deletion precedence is reversed.
delGrp := func(kind string) int { return -managedCreateOrder[kind] }

// Service (+89) < Role (+90) < ClusterRole (+91) < CRD (+92) < ... < Namespace (+100)
assert.Less(t, delGrp("Service"), delGrp("Role"))
assert.Less(t, delGrp("Role"), delGrp("ClusterRole"))
assert.Less(t, delGrp("ClusterRole"), delGrp("CustomResourceDefinition"))
assert.Less(t, delGrp("CustomResourceDefinition"), delGrp("PersistentVolumeClaim"))
assert.Less(t, delGrp("PersistentVolumeClaim"), delGrp("PersistentVolume"))
assert.Less(t, delGrp("PersistentVolume"), delGrp("StorageClass"))
assert.Less(t, delGrp("StorageClass"), delGrp("ConfigMap"))
assert.Less(t, delGrp("ConfigMap"), delGrp("ServiceAccount"))
assert.Less(t, delGrp("ServiceAccount"), delGrp("Namespace"))
func TestManagedOrderingFlat(t *testing.T) {
// All managed kinds share the same reserved create group (-100) so they
// are reconciled together as infrastructure, ahead of user resources
// (whose groups must be >= -60). Deletion groups are the negation.
for kind, grp := range managedCreateOrder {
assert.Equal(t, -100, grp, "managed kind %q should be at reserved group -100", kind)
}
}


Loading