[8.2] [MOD-10926] Expose SVS Marked Deleted Vector Count via Statistics API (#797)

[MOD-10926] Expose SVS Marked Deleted Vector Count via Statistics API (#796)

SVSIndex
rename chnages_num -> num_marked_deleted to emphasize it onluy counts deletions

SVSIndexBase: move num_marked_deleted to SVSIndexBase so it can be simply fetched without complex cast
add getNumMarkedDeleted to API

add  virtual getNumMarkedDeleted to tiered index to be called by VecSimTieredIndex::statisticInfo

hnsw  tiered implmnetaion: call hnsw->VecSimIndexTombstone::getNumMarkedDeleted, (hnsw inhrits VecSimIndexTombstone)

svs imp: returns SVSIndexBase::getNumMarkedDeleted

(cherry picked from commit 1c0ec77)

Author: meiravgri

src/VecSim/algorithms/hnsw/hnsw_tiered.h

@@ -201,6 +201,9 @@ class TieredHNSWIndex : public VecSimTieredIndex<DataType, DistType> {
 
     int addVector(const void *blob, labelType label) override;
     int deleteVector(labelType label) override;
+    size_t getNumMarkedDeleted() const override {
+        return this->getHNSWIndex()->getNumMarkedDeleted();
+    }
     size_t indexSize() const override;
     size_t indexCapacity() const override;
     double getDistanceFrom_Unsafe(labelType label, const void *blob) const override;

src/VecSim/algorithms/svs/svs.h

@@ -35,7 +35,7 @@ struct SVSIndexBase
     : public SVSSerializer
 #endif
 {
-
+    SVSIndexBase() : num_marked_deleted{0} {};
     virtual ~SVSIndexBase() = default;
     virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0;
     virtual int deleteVectors(const labelType *labels, size_t n) = 0;
@@ -44,9 +44,14 @@ struct SVSIndexBase
     virtual void setNumThreads(size_t numThreads) = 0;
     virtual size_t getThreadPoolCapacity() const = 0;
     virtual bool isCompressed() const = 0;
+    size_t getNumMarkedDeleted() const { return num_marked_deleted; }
 #ifdef BUILD_TESTS
     virtual svs::logging::logger_ptr getLogger() const = 0;
 #endif
+protected:
+    // Index marked deleted vectors counter to initiate reindexing if it exceeds threshold
+    // markIndexUpdate() manages this counter
+    size_t num_marked_deleted;
 };
 
 template <typename MetricType, typename DataType, bool isMulti, size_t QuantBits,
@@ -72,10 +77,6 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
 
     bool forcePreprocessing;
 
-    // Index severe changes counter to initiate reindexing if number of changes exceed threshold
-    // markIndexUpdated() manages this counter
-    size_t changes_num;
-
     // Index build parameters
     svs::index::vamana::VamanaBuildParameters buildParams;
 
@@ -273,26 +274,27 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
         return deleted_num;
     }
 
-    // Count severe index changes (currently deletions only) and consolidate index if needed
+    // Count deletions and consolidate index if needed
     void markIndexUpdate(size_t n = 1) {
         if (!impl_)
             return;
 
         // SVS index instance should not be empty
         if (indexSize() == 0) {
             this->impl_.reset();
-            changes_num = 0;
+            num_marked_deleted = 0;
             return;
         }
 
-        changes_num += n;
+        num_marked_deleted += n;
         // consolidate index if number of changes bigger than 50% of index size
         const float consolidation_threshold = .5f;
         // indexSize() should not be 0 see above lines
         assert(indexSize() > 0);
-        if (static_cast<float>(changes_num) / indexSize() > consolidation_threshold) {
+        // Note: if this function is called after deleteVectorsImpl, indexSize is already updated
+        if (static_cast<float>(num_marked_deleted) / indexSize() > consolidation_threshold) {
             impl_->consolidate();
-            changes_num = 0;
+            num_marked_deleted = 0;
         }
     }
 
@@ -312,7 +314,7 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
     SVSIndex(const SVSParams &params, const AbstractIndexInitParams &abstractInitParams,
              const index_component_t &components, bool force_preprocessing)
         : Base{abstractInitParams, components}, forcePreprocessing{force_preprocessing},
-          changes_num{0}, buildParams{svs_details::makeVamanaBuildParameters(params)},
+          buildParams{svs_details::makeVamanaBuildParameters(params)},
           search_window_size{svs_details::getOrDefault(params.search_window_size,
                                                        SVS_VAMANA_DEFAULT_SEARCH_WINDOW_SIZE)},
           search_buffer_capacity{
@@ -373,7 +375,7 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
                           .pruneTo = this->buildParams.prune_to,
                           .useSearchHistory = this->buildParams.use_full_search_history,
                           .numThreads = this->getNumThreads(),
-                          .numberOfMarkedDeletedNodes = this->changes_num,
+                          .numberOfMarkedDeletedNodes = this->num_marked_deleted,
                           .searchWindowSize = this->search_window_size,
                           .searchBufferCapacity = this->search_buffer_capacity,
                           .leanvecDim = this->leanvec_dim,
@@ -673,7 +675,7 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
             // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.compact
             impl_->compact();
         }
-        changes_num = 0;
+        num_marked_deleted = 0;
     }
 
 #ifdef BUILD_TESTS

src/VecSim/algorithms/svs/svs_tiered.h

@@ -726,6 +726,9 @@ class TieredSVSIndex : public VecSimTieredIndex<DataType, float> {
         assert(ret <= 2 && "unexpected deleteVector result");
         return ret;
     }
+    size_t getNumMarkedDeleted() const override {
+        return this->GetSVSIndex()->getNumMarkedDeleted();
+    }
 
     size_t indexSize() const override {
         std::shared_lock<std::shared_mutex> flat_lock(this->flatIndexGuard);

src/VecSim/vec_sim_tiered_index.h

@@ -139,7 +139,7 @@ class VecSimTieredIndex : public VecSimIndexInterface {
         return this->allocator->getAllocationSize() + this->backendIndex->getAllocationSize() +
                this->frontendIndex->getAllocationSize();
     }
-
+    virtual size_t getNumMarkedDeleted() const = 0;
     size_t indexLabelCount() const override;
     VecSimIndexStatsInfo statisticInfo() const override;
     virtual VecSimIndexDebugInfo debugInfo() const override;
@@ -319,14 +319,9 @@ template <typename DataType, typename DistType>
 VecSimIndexStatsInfo VecSimTieredIndex<DataType, DistType>::statisticInfo() const {
     auto stats = VecSimIndexStatsInfo{
         .memory = this->getAllocationSize(),
-        .numberOfMarkedDeleted = 0, // Default value if cast fails
+        .numberOfMarkedDeleted = this->getNumMarkedDeleted(),
     };
 
-    // If backend implements VecSimIndexTombstone, get number of marked deleted
-    if (auto tombstone = dynamic_cast<VecSimIndexTombstone *>(this->backendIndex)) {
-        stats.numberOfMarkedDeleted = tombstone->getNumMarkedDeleted();
-    }
-
     return stats;
 }
 

tests/unit/test_svs.cpp

@@ -182,6 +182,7 @@ TYPED_TEST(SVSTest, svs_vector_update_test) {
     // Delete the last vector.
     VecSimIndex_DeleteVector(index, 1);
     EXPECT_EQ(VecSimIndex_IndexSize(index), 0);
+    ASSERT_EQ(svs_index->getNumMarkedDeleted(), 0);
 
     VecSimIndex_Free(index);
 }
@@ -261,9 +262,19 @@ TYPED_TEST(SVSTest, svs_bulk_vectors_add_delete_test) {
     runTopKSearchTest(index, query, k, verify_res, nullptr, BY_ID);
 
     // Delete almost all vectors
+    // First delete small amount of vector to prevent consolidation.
+    const size_t first_batch_deletion = 10;
+    ASSERT_EQ(svs_index->deleteVectors(ids.data(), first_batch_deletion), first_batch_deletion);
+    ASSERT_EQ(VecSimIndex_IndexSize(index), n - first_batch_deletion);
+    ASSERT_EQ(svs_index->getNumMarkedDeleted(), first_batch_deletion);
+
+    // Now delete enough vectors to trigger consolidation.
     const size_t keep_num = 1;
-    ASSERT_EQ(svs_index->deleteVectors(ids.data(), n - keep_num), n - keep_num);
+    ASSERT_EQ(svs_index->deleteVectors(ids.data() + first_batch_deletion,
+                                       n - keep_num - first_batch_deletion),
+              n - keep_num - first_batch_deletion);
     ASSERT_EQ(VecSimIndex_IndexSize(index), keep_num);
+    ASSERT_EQ(svs_index->getNumMarkedDeleted(), 0);
 
     VecSimIndex_Free(index);
 }
@@ -3012,7 +3023,7 @@ TYPED_TEST(SVSTest, logging_runtime_params) {
     for (size_t i = 0; i < 10; i++) {
         index->addVector(v[i].data(), ids[i]);
     }
-
+    ASSERT_EQ(svs_index->getNumMarkedDeleted(), 10);
     ASSERT_EQ(VecSimIndex_IndexSize(index), n);
 
     float query[] = {50, 50, 50, 50};

tests/unit/test_svs_tiered.cpp

@@ -738,6 +738,79 @@ TYPED_TEST(SVSTieredIndexTest, deleteVector) {
     }
 }
 
+TYPED_TEST(SVSTieredIndexTestBasic, markedDeleted) {
+    // Create TieredSVS index instance with a mock queue.
+    size_t dim = 4;
+    constexpr size_t n = 10;
+    constexpr size_t transfer_trigger = n;
+    SVSParams params = {.type = TypeParam::get_index_type(),
+                        .dim = dim,
+                        .metric = VecSimMetric_L2,
+                        .num_threads = 1};
+    VecSimParams svs_params = CreateParams(params);
+    auto mock_thread_pool = tieredIndexMock();
+
+    auto *tiered_index = this->CreateTieredSVSIndex(svs_params, mock_thread_pool, transfer_trigger,
+                                                    transfer_trigger);
+    ASSERT_INDEX(tiered_index);
+
+    for (size_t i = 0; i < n; i++) {
+        GenerateAndAddVector<TEST_DATA_T>(tiered_index, dim, i);
+    }
+    // Vectors are still in the flat buffer.
+    ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), n);
+    ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0);
+    // Override a vector while in the flat buffer
+
+    GenerateAndAddVector<TEST_DATA_T>(tiered_index, dim, 0);
+    ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), n);
+    ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0);
+
+    // Move vectors to the backend
+    mock_thread_pool.thread_iteration();
+    ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), n);
+    ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0);
+    ASSERT_EQ(tiered_index->indexSize(), n);
+    ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0);
+
+    // Override a vector while in the backend
+    GenerateAndAddVector<TEST_DATA_T>(tiered_index, dim, 1);
+    ASSERT_EQ(tiered_index->indexSize(), n);
+    ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 1);
+    ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 1);
+
+    ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), n - 1);
+    ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1);
+
+    // Delete the overriden vector
+    VecSimIndex_DeleteVector(tiered_index, 1);
+    ASSERT_EQ(tiered_index->indexSize(), n - 1);
+    ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 1);
+    ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 1);
+    ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), n - 1);
+    ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0);
+
+    // Delete another arbirtrary vector
+    VecSimIndex_DeleteVector(tiered_index, 0);
+    ASSERT_EQ(tiered_index->indexSize(), n - 2);
+    ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 2);
+    ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 2);
+    ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), n - 2);
+    ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0);
+
+    // Empty Index
+    for (size_t i = 2; i < n; i++) {
+        VecSimIndex_DeleteVector(tiered_index, i);
+    }
+
+    // Consolidate should be triggered and mark deleted count should be zeroed.
+    ASSERT_EQ(tiered_index->indexSize(), 0);
+    ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0);
+    ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 0);
+    ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 0);
+    ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0);
+}
+
 TYPED_TEST(SVSTieredIndexTest, manageIndexOwnership) {
 
     // Create TieredSVS index instance with a mock queue.
@@ -2211,6 +2284,8 @@ TYPED_TEST(SVSTieredIndexTest, writeInPlaceMode) {
     // Validate that the vector is removed in place.
     tiered_index->deleteVector(vec_label);
     ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1);
+    ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 0);
+    EXPECT_EQ(tiered_index->statisticInfo().numberOfMarkedDeleted, 0);
 }
 
 TYPED_TEST(SVSTieredIndexTest, switchWriteModes) {
@@ -2375,6 +2450,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, runGCAPI) {
     for (size_t i = 0; i < threshold; i++) {
         tiered_index->deleteVector(i);
     }
+    ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), threshold);
+    EXPECT_EQ(tiered_index->statisticInfo().numberOfMarkedDeleted, threshold);
+
     // Launch the BG threads loop that takes jobs from the queue and executes them.
     mock_thread_pool.init_threads();
     mock_thread_pool.thread_pool_join();
@@ -2391,6 +2469,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, runGCAPI) {
     auto size_after_gc = tiered_index->getAllocationSize();
     // Expect that the size of the index was reduced.
     ASSERT_LT(size_after_gc, size_before_gc);
+    ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 0);
+    EXPECT_EQ(tiered_index->statisticInfo().numberOfMarkedDeleted, 0);
 }
 
 TYPED_TEST(SVSTieredIndexTestBasic, switchDeleteModes) {