From 6d598df4e27c852cb657db0de21bdeded937c619 Mon Sep 17 00:00:00 2001 From: David Keegan Date: Wed, 10 Dec 2025 17:26:47 -0700 Subject: [PATCH] Use exponential backoff --- ruby/lib/ci/queue/redis/worker.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ruby/lib/ci/queue/redis/worker.rb b/ruby/lib/ci/queue/redis/worker.rb index 6dbfa9d..3117a78 100644 --- a/ruby/lib/ci/queue/redis/worker.rb +++ b/ruby/lib/ci/queue/redis/worker.rb @@ -10,10 +10,13 @@ module Redis class << self attr_accessor :requeue_offset + attr_accessor :max_sleep_time end self.requeue_offset = 42 + self.max_sleep_time = 2 class Worker < Base + DEFAULT_SLEEP_SECONDS = 0.5 attr_reader :total def initialize(redis, config) @@ -81,8 +84,10 @@ def poll end idle_since = nil idle_state_printed = false + attempt = 0 until shutdown_required? || config.circuit_breakers.any?(&:open?) || exhausted? || max_test_failed? if id = reserve + attempt = 0 idle_since = nil executable = resolve_executable(id) @@ -111,7 +116,11 @@ def poll end idle_state_printed = true end - sleep 0.05 + # Adding exponential backoff to avoid hammering Redis + # we just stay online here in case a test gets retried or times out so we can afford to wait + sleep_time = [DEFAULT_SLEEP_SECONDS * (2 ** attempt), Redis.max_sleep_time].min + attempt += 1 + sleep sleep_time end end redis.pipelined do |pipeline|