Skip to content

Commit 40aa744

Browse files
meiravgrimeiravgri
authored
[0.7] [MOD-11237] Block Size Boundary Oscillation Benchmark (#768) (#772)
* initial backport * align with 0.7 api * move UpdateAtBlockSize_Single to the end
1 parent 0e1e454 commit 40aa744

File tree

4 files changed

+112
-10
lines changed

4 files changed

+112
-10
lines changed

‎tests/benchmark/bm_common.h‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,17 +60,19 @@ void BM_VecSimCommon<index_type_t>::Memory_FLAT(benchmark::State &st, unsigned s
6060
for (auto _ : st) {
6161
// Do nothing...
6262
}
63-
st.counters["memory"] =
64-
(double)VecSimIndex_StatsInfo(INDICES[VecSimAlgo_BF + index_offset]).memory;
63+
st.counters["memory"] = benchmark::Counter(
64+
(double)VecSimIndex_StatsInfo(INDICES[VecSimAlgo_BF + index_offset]).memory,
65+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
6566
}
6667
template <typename index_type_t>
6768
void BM_VecSimCommon<index_type_t>::Memory_HNSW(benchmark::State &st, unsigned short index_offset) {
6869

6970
for (auto _ : st) {
7071
// Do nothing...
7172
}
72-
st.counters["memory"] =
73-
(double)VecSimIndex_StatsInfo(INDICES[VecSimAlgo_HNSWLIB + index_offset]).memory;
73+
st.counters["memory"] = benchmark::Counter(
74+
(double)VecSimIndex_StatsInfo(INDICES[VecSimAlgo_HNSWLIB + index_offset]).memory,
75+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
7476
}
7577
template <typename index_type_t>
7678
void BM_VecSimCommon<index_type_t>::Memory_Tiered(benchmark::State &st,
@@ -79,8 +81,9 @@ void BM_VecSimCommon<index_type_t>::Memory_Tiered(benchmark::State &st,
7981
for (auto _ : st) {
8082
// Do nothing...
8183
}
82-
st.counters["memory"] =
83-
(double)VecSimIndex_StatsInfo(INDICES[VecSimAlgo_TIERED + index_offset]).memory;
84+
st.counters["memory"] = benchmark::Counter(
85+
(double)VecSimIndex_StatsInfo(INDICES[VecSimAlgo_TIERED + index_offset]).memory,
86+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
8487
}
8588

8689
// TopK search BM

‎tests/benchmark/bm_vecsim_basics.h‎

Lines changed: 89 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,14 @@ class BM_VecSimBasics : public BM_VecSimCommon<index_type_t> {
3131
static void Range_BF(benchmark::State &st);
3232
static void Range_HNSW(benchmark::State &st);
3333

34+
// Reproduces allocation/deallocation oscillation issue at block size boundaries.
35+
// Sets up index at blockSize+1 capacity, then repeatedly deletes and re-adds the same vector,
36+
// triggering constant grow-shrink cycles.
37+
// This behavior was fixed by PR #753 with a conservative resize strategy that only
38+
// shrinks containers when there are 2+ free blocks, preventing oscillation cycles.
39+
// Expected: High allocation overhead before fix, stable performance after fix.
40+
static void UpdateAtBlockSize(benchmark::State &st);
41+
3442
private:
3543
// Vectors of vector to store deleted labels' data.
3644
using LabelData = std::vector<std::vector<data_t>>;
@@ -64,7 +72,9 @@ void BM_VecSimBasics<index_type_t>::AddLabel(benchmark::State &st) {
6472
// For tiered index, wait for all threads to finish indexing
6573
BM_VecSimGeneral::mock_thread_pool.thread_pool_wait();
6674

67-
st.counters["memory_per_vector"] = (double)memory_delta / (double)added_vec_count;
75+
st.counters["memory_per_vector"] =
76+
benchmark::Counter((double)memory_delta / (double)added_vec_count,
77+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
6878
st.counters["vectors_per_label"] = vec_per_label;
6979

7080
assert(VecSimIndex_IndexSize(index) == N_VECTORS + added_vec_count);
@@ -110,7 +120,9 @@ void BM_VecSimBasics<index_type_t>::AddLabel_AsyncIngest(benchmark::State &st) {
110120
}
111121

112122
size_t memory_delta = (INDICES[st.range(0)])->getAllocationSize() - memory_before;
113-
st.counters["memory_per_vector"] = (double)memory_delta / (double)added_vec_count;
123+
st.counters["memory_per_vector"] =
124+
benchmark::Counter((double)memory_delta / (double)added_vec_count,
125+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
114126
st.counters["vectors_per_label"] = vec_per_label;
115127
st.counters["num_threads"] = BM_VecSimGeneral::mock_thread_pool.thread_pool_size;
116128

@@ -159,7 +171,9 @@ void BM_VecSimBasics<index_type_t>::DeleteLabel(algo_t *index, benchmark::State
159171
if (VecSimIndex_BasicInfo(index).algo == VecSimAlgo_TIERED) {
160172
reinterpret_cast<TieredHNSWIndex<data_t, data_t> *>(index)->executeReadySwapJobs();
161173
}
162-
st.counters["memory_per_vector"] = memory_delta / (double)removed_vectors_count;
174+
st.counters["memory_per_vector"] =
175+
benchmark::Counter((double)memory_delta / (double)removed_vectors_count,
176+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
163177

164178
// Restore index state.
165179
// For each label in removed_labels_data
@@ -207,7 +221,10 @@ void BM_VecSimBasics<index_type_t>::DeleteLabel_AsyncRepair(benchmark::State &st
207221
// Avg. memory delta per vector equals the total memory delta divided by the number
208222
// of deleted vectors.
209223
int memory_delta = tiered_index->getAllocationSize() - memory_before;
210-
st.counters["memory_per_vector"] = memory_delta / (double)removed_vectors_count;
224+
225+
st.counters["memory_per_vector"] =
226+
benchmark::Counter((double)memory_delta / (double)removed_vectors_count,
227+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
211228
st.counters["num_threads"] = (double)BM_VecSimGeneral::mock_thread_pool.thread_pool_size;
212229
st.counters["num_zombies"] = tiered_index->idToSwapJob.size();
213230

@@ -279,6 +296,69 @@ void BM_VecSimBasics<index_type_t>::Range_HNSW(benchmark::State &st) {
279296
st.counters["Recall"] = (float)total_res / total_res_bf;
280297
}
281298

299+
template <typename index_type_t>
300+
void BM_VecSimBasics<index_type_t>::UpdateAtBlockSize(benchmark::State &st) {
301+
auto index = INDICES[st.range(0)];
302+
size_t initial_index_size = VecSimIndex_IndexSize(index);
303+
// Calculate vectors needed to reach next block boundary
304+
size_t vecs_to_blocksize =
305+
BM_VecSimGeneral::block_size - (initial_index_size % BM_VecSimGeneral::block_size);
306+
assert(vecs_to_blocksize < BM_VecSimGeneral::block_size);
307+
labelType initial_label_count = index->indexLabelCount();
308+
labelType curr_label = initial_label_count;
309+
310+
// Set up index at blockSize+1 to trigger oscillation issue
311+
// Make sure we have enough queries to add a new label.
312+
assert(N_QUERIES > BM_VecSimGeneral::block_size);
313+
size_t overhead = 1;
314+
size_t added_vec_count = vecs_to_blocksize + overhead;
315+
for (size_t i = 0; i < added_vec_count; ++i) {
316+
VecSimIndex_AddVector(index, QUERIES[added_vec_count % N_QUERIES].data(), curr_label++);
317+
}
318+
// For tiered index, wait for all threads to finish indexing
319+
BM_VecSimGeneral::mock_thread_pool.thread_pool_wait();
320+
assert(VecSimIndex_IndexSize(index) % BM_VecSimGeneral::block_size == overhead);
321+
assert(VecSimIndex_IndexSize(index) == N_VECTORS + added_vec_count);
322+
323+
std::cout << "Added " << added_vec_count << " vectors to reach block size boundary."
324+
<< std::endl;
325+
std::cout << "Index size is now " << VecSimIndex_IndexSize(index) << std::endl;
326+
std::cout << "Last label is " << curr_label - 1 << std::endl;
327+
328+
// Benchmark loop: repeatedly delete/add same vector to trigger grow-shrink cycles
329+
labelType label_to_update = curr_label - 1;
330+
size_t index_cap = index->indexCapacity();
331+
for (auto _ : st) {
332+
// Remove the vector directly from hnsw
333+
size_t ret = VecSimIndex_DeleteVector(
334+
INDICES[st.range(0) == VecSimAlgo_TIERED ? VecSimAlgo_HNSWLIB : st.range(0)],
335+
label_to_update);
336+
assert(ret == 1);
337+
assert(index->indexCapacity() == index_cap - BM_VecSimGeneral::block_size);
338+
// Capacity should shrink by one block after deletion
339+
ret = VecSimIndex_AddVector(index, QUERIES[(added_vec_count - 1) % N_QUERIES].data(),
340+
label_to_update);
341+
assert(ret == 1);
342+
BM_VecSimGeneral::mock_thread_pool.thread_pool_wait();
343+
assert(VecSimIndex_IndexSize(
344+
INDICES[st.range(0) == VecSimAlgo_TIERED ? VecSimAlgo_HNSWLIB : st.range(0)]) ==
345+
N_VECTORS + added_vec_count);
346+
// Capacity should grow back to original size after addition
347+
assert(index->indexCapacity() == index_cap);
348+
}
349+
assert(VecSimIndex_IndexSize(index) == N_VECTORS + added_vec_count);
350+
351+
// Clean-up all the new vectors to restore the index size to its original value.
352+
353+
size_t new_label_count = index->indexLabelCount();
354+
for (size_t label = initial_label_count; label < new_label_count; label++) {
355+
// If index is tiered HNSW, remove directly from the underline HNSW.
356+
VecSimIndex_DeleteVector(
357+
INDICES[st.range(0) == VecSimAlgo_TIERED ? VecSimAlgo_HNSWLIB : st.range(0)], label);
358+
}
359+
assert(VecSimIndex_IndexSize(index) == N_VECTORS);
360+
}
361+
282362
#define UNIT_AND_ITERATIONS Unit(benchmark::kMillisecond)->Iterations(BM_VecSimGeneral::block_size)
283363

284364
// The actual radius will be the given arg divided by 100, since arg must be an integer.
@@ -324,3 +404,8 @@ void BM_VecSimBasics<index_type_t>::Range_HNSW(benchmark::State &st) {
324404
}
325405
#define REGISTER_DeleteLabel(BM_FUNC) \
326406
BENCHMARK_REGISTER_F(BM_VecSimBasics, BM_FUNC)->UNIT_AND_ITERATIONS
407+
408+
#define REGISTER_UpdateAtBlockSize(BM_FUNC, VecSimAlgo) \
409+
BENCHMARK_REGISTER_F(BM_VecSimBasics, BM_FUNC) \
410+
->UNIT_AND_ITERATIONS->Arg(VecSimAlgo) \
411+
->ArgName(#VecSimAlgo)

‎tests/benchmark/run_files/bm_basics_multi_fp32.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,10 @@ DEFINE_DELETE_LABEL(BM_FUNC_NAME(DeleteLabel, Tiered), fp32_index_t, TieredHNSWI
3333
VecSimAlgo_TIERED)
3434
#include "benchmark/bm_initialization/bm_basics_initialize_fp32.h"
3535

36+
// Test oscillations at block size boundaries.
37+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, UpdateAtBlockSize_Multi, fp32_index_t)
38+
(benchmark::State &st) { UpdateAtBlockSize(st); }
39+
REGISTER_UpdateAtBlockSize(UpdateAtBlockSize_Multi, VecSimAlgo_BF);
40+
REGISTER_UpdateAtBlockSize(UpdateAtBlockSize_Multi, VecSimAlgo_HNSWLIB);
41+
REGISTER_UpdateAtBlockSize(UpdateAtBlockSize_Multi, VecSimAlgo_TIERED);
3642
BENCHMARK_MAIN();

‎tests/benchmark/run_files/bm_basics_single_fp32.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,13 @@ DEFINE_DELETE_LABEL(BM_FUNC_NAME(DeleteLabel, HNSW), fp32_index_t, HNSWIndex_Sin
3131
VecSimAlgo_HNSWLIB)
3232
DEFINE_DELETE_LABEL(BM_FUNC_NAME(DeleteLabel, Tiered), fp32_index_t, TieredHNSWIndex, float, float,
3333
VecSimAlgo_TIERED)
34+
3435
#include "benchmark/bm_initialization/bm_basics_initialize_fp32.h"
36+
37+
// Test oscillations at block size boundaries.
38+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, UpdateAtBlockSize_Single, fp32_index_t)
39+
(benchmark::State &st) { UpdateAtBlockSize(st); }
40+
REGISTER_UpdateAtBlockSize(UpdateAtBlockSize_Single, VecSimAlgo_BF);
41+
REGISTER_UpdateAtBlockSize(UpdateAtBlockSize_Single, VecSimAlgo_HNSWLIB);
42+
REGISTER_UpdateAtBlockSize(UpdateAtBlockSize_Single, VecSimAlgo_TIERED);
3543
BENCHMARK_MAIN();

0 commit comments

Comments
 (0)