Skip to content

Commit 722c84a

Browse files
committed
Reduce Redis key size
Cherry-picked from Shopify commit b534f89 with conflict resolution. Changes: - Add KeyShortener module to reduce Redis key sizes by ~76% - Update all key() methods to use KeyShortener - Preserve Figma's test suite (no merge of Shopify's new tests) - Keep Figma's moving average imports
1 parent 832415b commit 722c84a

File tree

7 files changed

+60
-6
lines changed

7 files changed

+60
-6
lines changed

ruby/lib/ci/queue/redis.rb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
require 'ci/queue/redis/test_time_record'
1313
require 'ci/queue/redis/test_duration_moving_averages'
1414
require 'ci/queue/redis/update_test_duration_moving_average'
15+
require 'ci/queue/redis/key_shortener'
1516

1617
module CI
1718
module Queue

ruby/lib/ci/queue/redis/base.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ def master_worker_id
102102
attr_reader :redis, :redis_url
103103

104104
def key(*args)
105-
['build', build_id, *args].join(':')
105+
KeyShortener.key(config.build_id, *args)
106106
end
107107

108108
def build_id

ruby/lib/ci/queue/redis/build_record.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ def record_stats(stats, pipeline: redis)
124124
end
125125

126126
def key(*args)
127-
['build', config.build_id, *args].join(':')
127+
KeyShortener.key(config.build_id, *args)
128128
end
129129
end
130130
end

ruby/lib/ci/queue/redis/grind_record.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ def pop_warnings
5252
attr_reader :redis, :config
5353

5454
def key(*args)
55-
['build', config.build_id, *args].join(':')
55+
KeyShortener.key(config.build_id, *args)
5656
end
5757

5858
def record_stats(stats, pipeline: redis)
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# frozen_string_literal: true
2+
require 'digest/md5'
3+
4+
module CI
5+
module Queue
6+
module Redis
7+
module KeyShortener
8+
# Suffix mapping for common key patterns
9+
SUFFIX_ALIASES = {
10+
'running' => 'r',
11+
'processed' => 'p',
12+
'queue' => 'q',
13+
'owners' => 'o',
14+
'error-reports' => 'e',
15+
'requeues-count' => 'rc',
16+
'assertions' => 'a',
17+
'errors' => 'er',
18+
'failures' => 'f',
19+
'skips' => 's',
20+
'requeues' => 'rq',
21+
'total_time' => 't',
22+
'test_failed_count' => 'fc',
23+
'completed' => 'c',
24+
'master-status' => 'm',
25+
'created-at' => 'ca',
26+
'workers' => 'w',
27+
'worker' => 'w',
28+
'warnings' => 'wn',
29+
'worker-errors' => 'we',
30+
'flaky-reports' => 'fl',
31+
}.freeze
32+
33+
# We're transforming the key to a shorter format to minimize network traffic.
34+
#
35+
# Strategy:
36+
# - Shorten prefix: 'b' instead of 'build'
37+
# - Hash UUID: 8-char MD5 instead of 36-char UUID
38+
# - Alias suffixes: single letters instead of full words
39+
#
40+
# Example:
41+
# build:unit:019aef0e-c010-433e-b706-c658d3c16372:running (55 bytes)
42+
# -> b:f03d3bef:r (13 bytes, 76% reduction)
43+
44+
def self.key(build_id, *args)
45+
digest = Digest::MD5.hexdigest(build_id)[0..7]
46+
shortened_args = args.map { |arg| SUFFIX_ALIASES[arg] || arg }
47+
48+
['b', digest, *shortened_args].join(':')
49+
end
50+
end
51+
end
52+
end
53+
end

ruby/test/ci/queue/redis_test.rb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -203,9 +203,9 @@ def test_acknowledge_returns_false_if_the_test_was_picked_up_by_another_worker
203203
end
204204

205205
def test_workers_register
206-
assert_equal 1, @redis.scard('build:42:workers')
206+
assert_equal 1, @redis.scard(CI::Queue::Redis::KeyShortener.key('42', 'workers'))
207207
worker(2)
208-
assert_equal 2, @redis.scard('build:42:workers')
208+
assert_equal 2, @redis.scard(CI::Queue::Redis::KeyShortener.key('42', 'workers'))
209209
end
210210

211211
def test_timeout_warning

ruby/test/integration/minitest_redis_test.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ def test_retry_fails_when_test_run_is_expired
240240
assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output
241241

242242
one_day = 60 * 60 * 24
243-
key = ['build', "1", "created-at"].join(':')
243+
key = CI::Queue::Redis::KeyShortener.key("1", "created-at")
244244
@redis.set(key, Time.now - one_day)
245245

246246
out, err = capture_subprocess_io do

0 commit comments

Comments
 (0)