Skip to content

Commit 9f6667c

Browse files
chillaqchillaq
committed
Updated uniqeKeysTracker
1 parent 6de4559 commit 9f6667c

File tree

2 files changed

+108
-3
lines changed

2 files changed

+108
-3
lines changed

‎client/src/main/java/io/split/client/impressions/UniqueKeysTrackerImp.java‎

Lines changed: 56 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package io.split.client.impressions;
22

3+
import com.google.common.collect.Lists;
34
import io.split.client.dtos.UniqueKeys;
45
import io.split.client.impressions.filters.BloomFilterImp;
56
import io.split.client.impressions.filters.Filter;
@@ -25,7 +26,7 @@
2526
public class UniqueKeysTrackerImp implements UniqueKeysTracker{
2627
private static final Logger _log = LoggerFactory.getLogger(UniqueKeysTrackerImp.class);
2728
private static final double MARGIN_ERROR = 0.01;
28-
private static final int MAX_AMOUNT_OF_TRACKED_UNIQUE_KEYS = 30000;
29+
private static final int MAX_UNIQUE_KEYS_POST_SIZE = 5000;
2930
private static final int MAX_AMOUNT_OF_KEYS = 10000000;
3031
private FilterAdapter filterAdapter;
3132
private final TelemetrySynchronizer _telemetrySynchronizer;
@@ -62,7 +63,7 @@ public boolean track(String featureFlagName, String key) {
6263
return keysByFeature;
6364
});
6465
_logger.debug("The feature flag " + featureFlagName + " and key " + key + " was added");
65-
if (uniqueKeysTracker.size() >= MAX_AMOUNT_OF_TRACKED_UNIQUE_KEYS){
66+
if (getTrackerKeysSize() >= MAX_UNIQUE_KEYS_POST_SIZE){
6667
_logger.warn("The UniqueKeysTracker size reached the maximum limit");
6768
try {
6869
sendUniqueKeys();
@@ -115,23 +116,75 @@ private void sendUniqueKeys(){
115116
_log.debug("SendUniqueKeys already running");
116117
return;
117118
}
119+
118120
try {
119121
if (uniqueKeysTracker.size() == 0) {
120122
_log.debug("The Unique Keys Tracker is empty");
121123
return;
122124
}
125+
123126
HashMap<String, HashSet<String>> uniqueKeysHashMap = popAll();
124127
List<UniqueKeys.UniqueKey> uniqueKeysFromPopAll = new ArrayList<>();
125128
for (Map.Entry<String, HashSet<String>> uniqueKeyEntry : uniqueKeysHashMap.entrySet()) {
126129
UniqueKeys.UniqueKey uniqueKey = new UniqueKeys.UniqueKey(uniqueKeyEntry.getKey(), new ArrayList<>(uniqueKeyEntry.getValue()));
127130
uniqueKeysFromPopAll.add(uniqueKey);
128131
}
129-
_telemetrySynchronizer.synchronizeUniqueKeys(new UniqueKeys(uniqueKeysFromPopAll));
132+
uniqueKeysFromPopAll = capChunksToMaxSize(uniqueKeysFromPopAll);
133+
134+
for (List<UniqueKeys.UniqueKey> chunk : getChunks(uniqueKeysFromPopAll)) {
135+
_telemetrySynchronizer.synchronizeUniqueKeys(new UniqueKeys(chunk));
136+
}
130137
} finally {
131138
sendGuard.set(false);
132139
}
133140
}
134141

142+
private List<UniqueKeys.UniqueKey> capChunksToMaxSize(List<UniqueKeys.UniqueKey> uniqeKeys) {
143+
List<UniqueKeys.UniqueKey> finalChunk = new ArrayList<>();
144+
for (UniqueKeys.UniqueKey uniqueKey : uniqeKeys) {
145+
if (uniqueKey.keysDto.size() > MAX_UNIQUE_KEYS_POST_SIZE) {
146+
for(List<String> subChunk : Lists.partition(uniqueKey.keysDto, MAX_UNIQUE_KEYS_POST_SIZE)) {
147+
finalChunk.add(new UniqueKeys.UniqueKey(uniqueKey.featureName, subChunk));
148+
}
149+
continue;
150+
}
151+
finalChunk.add(uniqueKey);
152+
}
153+
return finalChunk;
154+
}
155+
156+
private List<List<UniqueKeys.UniqueKey>> getChunks(List<UniqueKeys.UniqueKey> uniqeKeys) {
157+
List<List<UniqueKeys.UniqueKey>> chunks = new ArrayList<>();
158+
List<UniqueKeys.UniqueKey> intermediateChunk = new ArrayList<>();
159+
for (UniqueKeys.UniqueKey uniqueKey : uniqeKeys) {
160+
if ((getChunkSize(intermediateChunk) + uniqueKey.keysDto.size()) > MAX_UNIQUE_KEYS_POST_SIZE) {
161+
chunks.add(intermediateChunk);
162+
intermediateChunk = new ArrayList<>();
163+
}
164+
intermediateChunk.add(uniqueKey);
165+
}
166+
if (!intermediateChunk.isEmpty()) {
167+
chunks.add(intermediateChunk);
168+
}
169+
return chunks;
170+
}
171+
172+
private int getChunkSize(List<UniqueKeys.UniqueKey> uniqueKeysChunk) {
173+
int totalSize = 0;
174+
for (UniqueKeys.UniqueKey uniqueKey : uniqueKeysChunk) {
175+
totalSize += uniqueKey.keysDto.size();
176+
}
177+
return totalSize;
178+
}
179+
180+
private int getTrackerKeysSize() {
181+
int totalSize = 0;
182+
for (String key : uniqueKeysTracker.keySet()) {
183+
totalSize += uniqueKeysTracker.get(key).size();
184+
}
185+
return totalSize;
186+
}
187+
135188
private interface ExecuteUniqueKeysAction{
136189
void execute();
137190
}

‎client/src/test/java/io/split/client/impressions/UniqueKeysTrackerImpTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
11
package io.split.client.impressions;
22

3+
import io.split.client.dtos.UniqueKeys;
34
import io.split.telemetry.synchronizer.TelemetryInMemorySubmitter;
45
import io.split.telemetry.synchronizer.TelemetrySynchronizer;
56
import org.junit.Assert;
67
import org.junit.Test;
78
import org.mockito.Mockito;
89

10+
import java.lang.reflect.InvocationTargetException;
11+
import java.lang.reflect.Method;
912
import java.util.HashMap;
1013
import java.util.HashSet;
14+
import java.util.List;
15+
import java.util.Map;
16+
import java.util.ArrayList;
1117

1218
public class UniqueKeysTrackerImpTest {
1319
private static TelemetrySynchronizer _telemetrySynchronizer = Mockito.mock(TelemetryInMemorySubmitter.class);
@@ -100,4 +106,50 @@ public void testStopSynchronization() throws Exception {
100106
uniqueKeysTrackerImp.stop();
101107
Mockito.verify(telemetrySynchronizer, Mockito.times(1)).synchronizeUniqueKeys(Mockito.anyObject());
102108
}
109+
110+
@Test
111+
public void testUniqueKeysChunks() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
112+
UniqueKeysTrackerImp uniqueKeysTrackerImp = new UniqueKeysTrackerImp(_telemetrySynchronizer, 10000, 10000, null);
113+
for (Integer i=1; i<6000; i++) {
114+
if (i <= 1000) {
115+
Assert.assertTrue(uniqueKeysTrackerImp.track("feature1", "key" + i.toString()));
116+
}
117+
if (i <= 2000) {
118+
Assert.assertTrue(uniqueKeysTrackerImp.track("feature2", "key" + i.toString()));
119+
}
120+
if (i <= 3000) {
121+
Assert.assertTrue(uniqueKeysTrackerImp.track("feature3", "key" + i.toString()));
122+
}
123+
if (i <= 4000) {
124+
Assert.assertTrue(uniqueKeysTrackerImp.track("feature4", "key" + i.toString()));
125+
}
126+
Assert.assertTrue(uniqueKeysTrackerImp.track("feature5", "key" + i.toString()));
127+
}
128+
129+
Method methodTrackerSize = uniqueKeysTrackerImp.getClass().getDeclaredMethod("getTrackerKeysSize");
130+
methodTrackerSize.setAccessible(true);
131+
int totalSize = (int) methodTrackerSize.invoke(uniqueKeysTrackerImp);
132+
Assert.assertTrue(totalSize == 15999);
133+
134+
HashMap<String, HashSet<String>> uniqueKeysHashMap = uniqueKeysTrackerImp.popAll();
135+
List<UniqueKeys.UniqueKey> uniqueKeysFromPopAll = new ArrayList<>();
136+
for (Map.Entry<String, HashSet<String>> uniqueKeyEntry : uniqueKeysHashMap.entrySet()) {
137+
UniqueKeys.UniqueKey uniqueKey = new UniqueKeys.UniqueKey(uniqueKeyEntry.getKey(), new ArrayList<>(uniqueKeyEntry.getValue()));
138+
uniqueKeysFromPopAll.add(uniqueKey);
139+
}
140+
Method methodCapChunks = uniqueKeysTrackerImp.getClass().getDeclaredMethod("capChunksToMaxSize", List.class);
141+
methodCapChunks.setAccessible(true);
142+
uniqueKeysFromPopAll = (List<UniqueKeys.UniqueKey>)methodCapChunks.invoke(uniqueKeysTrackerImp, uniqueKeysFromPopAll);
143+
144+
Method methodGetChunks = uniqueKeysTrackerImp.getClass().getDeclaredMethod("getChunks", List.class);
145+
methodGetChunks.setAccessible(true);
146+
List<List<UniqueKeys.UniqueKey>> keysChunks = (List<List<UniqueKeys.UniqueKey>>) methodGetChunks.invoke(uniqueKeysTrackerImp, uniqueKeysFromPopAll);
147+
for (List<UniqueKeys.UniqueKey> chunk : keysChunks) {
148+
int chunkSize = 0;
149+
for (UniqueKeys.UniqueKey keys : chunk) {
150+
chunkSize += keys.keysDto.size();
151+
}
152+
Assert.assertTrue(chunkSize <= 5000);
153+
}
154+
}
103155
}

0 commit comments

Comments
 (0)