[0.7] Refactor Info Report - [MOD-9321] (#656)

* Refactor Info Report - [MOD-9321] (#650)

* refactor `info()` API to indicate it should be used for debug, and prepare structs and new API for statistics info

* test fixes

* implement new API

* format

* improve tests

* move tiered statisticInfo to base class

* refactor

* minor simplification

* review fixes

(cherry picked from commit 5f7dbdf)

* fix compilation for 0.7

Author: GuyAv46

src/VecSim/algorithms/brute_force/brute_force.h

@@ -47,8 +47,8 @@ class BruteForceIndex : public VecSimIndexAbstract<DistType> {
                                         VecSimQueryParams *queryParams) const override;
     virtual VecSimQueryReply *rangeQuery(const void *queryBlob, double radius,
                                          VecSimQueryParams *queryParams) const override;
-    virtual VecSimIndexInfo info() const override;
-    virtual VecSimInfoIterator *infoIterator() const override;
+    VecSimIndexDebugInfo debugInfo() const override;
+    VecSimDebugInfoIterator *debugInfoIterator() const override;
     VecSimIndexBasicInfo basicInfo() const override;
     virtual VecSimBatchIterator *newBatchIterator(const void *queryBlob,
                                                   VecSimQueryParams *queryParams) const override;
@@ -339,9 +339,9 @@ BruteForceIndex<DataType, DistType>::rangeQuery(const void *queryBlob, double ra
 }
 
 template <typename DataType, typename DistType>
-VecSimIndexInfo BruteForceIndex<DataType, DistType>::info() const {
+VecSimIndexDebugInfo BruteForceIndex<DataType, DistType>::debugInfo() const {
 
-    VecSimIndexInfo info;
+    VecSimIndexDebugInfo info;
     info.commonInfo = this->getCommonInfo();
     info.commonInfo.basicInfo.algo = VecSimAlgo_BF;
 
@@ -358,11 +358,11 @@ VecSimIndexBasicInfo BruteForceIndex<DataType, DistType>::basicInfo() const {
 }
 
 template <typename DataType, typename DistType>
-VecSimInfoIterator *BruteForceIndex<DataType, DistType>::infoIterator() const {
-    VecSimIndexInfo info = this->info();
+VecSimDebugInfoIterator *BruteForceIndex<DataType, DistType>::debugInfoIterator() const {
+    VecSimIndexDebugInfo info = this->debugInfo();
     // For readability. Update this number when needed.
     size_t numberOfInfoFields = 10;
-    VecSimInfoIterator *infoIterator = new VecSimInfoIterator(numberOfInfoFields, this->allocator);
+    auto *infoIterator = new VecSimDebugInfoIterator(numberOfInfoFields, this->allocator);
 
     infoIterator->addInfoField(
         VecSim_InfoField{.fieldName = VecSimCommonStrings::ALGORITHM_STRING,

src/VecSim/algorithms/hnsw/hnsw.h

@@ -255,9 +255,9 @@ class HNSWIndex : public VecSimIndexAbstract<DistType>,
     void unlockNodeLinks(ElementGraphData *node_data) const;
     VisitedNodesHandler *getVisitedList() const;
     void returnVisitedList(VisitedNodesHandler *visited_nodes_handler) const;
-    VecSimIndexInfo info() const override;
+    VecSimIndexDebugInfo debugInfo() const override;
     VecSimIndexBasicInfo basicInfo() const override;
-    VecSimInfoIterator *infoIterator() const override;
+    VecSimDebugInfoIterator *debugInfoIterator() const override;
     bool preferAdHocSearch(size_t subsetSize, size_t k, bool initial_check) const override;
     const char *getDataByInternalId(idType internal_id) const;
     ElementGraphData *getGraphDataByInternalId(idType internal_id) const;
@@ -2086,18 +2086,19 @@ VecSimQueryReply *HNSWIndex<DataType, DistType>::rangeQuery(const void *query_da
 }
 
 template <typename DataType, typename DistType>
-VecSimIndexInfo HNSWIndex<DataType, DistType>::info() const {
+VecSimIndexDebugInfo HNSWIndex<DataType, DistType>::debugInfo() const {
 
-    VecSimIndexInfo info;
+    VecSimIndexDebugInfo info;
     info.commonInfo = this->getCommonInfo();
+    auto [ep_id, max_level] = this->safeGetEntryPointState();
 
     info.commonInfo.basicInfo.algo = VecSimAlgo_HNSWLIB;
     info.hnswInfo.M = this->getM();
     info.hnswInfo.efConstruction = this->getEfConstruction();
     info.hnswInfo.efRuntime = this->getEf();
     info.hnswInfo.epsilon = this->epsilon;
-    info.hnswInfo.max_level = this->getMaxLevel();
-    info.hnswInfo.entrypoint = this->getEntryPointLabel();
+    info.hnswInfo.max_level = max_level;
+    info.hnswInfo.entrypoint = ep_id != INVALID_ID ? getExternalLabel(ep_id) : INVALID_LABEL;
     info.hnswInfo.visitedNodesPoolSize = this->visitedNodesHandlerPool.getPoolSize();
     info.hnswInfo.numberOfMarkedDeletedNodes = this->getNumMarkedDeleted();
     return info;
@@ -2112,11 +2113,11 @@ VecSimIndexBasicInfo HNSWIndex<DataType, DistType>::basicInfo() const {
 }
 
 template <typename DataType, typename DistType>
-VecSimInfoIterator *HNSWIndex<DataType, DistType>::infoIterator() const {
-    VecSimIndexInfo info = this->info();
+VecSimDebugInfoIterator *HNSWIndex<DataType, DistType>::debugInfoIterator() const {
+    VecSimIndexDebugInfo info = this->debugInfo();
     // For readability. Update this number when needed.
     size_t numberOfInfoFields = 17;
-    VecSimInfoIterator *infoIterator = new VecSimInfoIterator(numberOfInfoFields, this->allocator);
+    auto *infoIterator = new VecSimDebugInfoIterator(numberOfInfoFields, this->allocator);
 
     infoIterator->addInfoField(
         VecSim_InfoField{.fieldName = VecSimCommonStrings::ALGORITHM_STRING,

src/VecSim/algorithms/hnsw/hnsw_tiered.h

@@ -193,9 +193,9 @@ class TieredHNSWIndex : public VecSimTieredIndex<DataType, DistType> {
     double getDistanceFrom_Unsafe(labelType label, const void *blob) const override;
     // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself when
     // needed.
-    VecSimIndexInfo info() const override;
+    VecSimIndexDebugInfo debugInfo() const override;
     VecSimIndexBasicInfo basicInfo() const override;
-    VecSimInfoIterator *infoIterator() const override;
+    VecSimDebugInfoIterator *debugInfoIterator() const override;
     VecSimBatchIterator *newBatchIterator(const void *queryBlob,
                                           VecSimQueryParams *queryParams) const override {
         size_t blobSize = this->backendIndex->getDim() * sizeof(DataType);
@@ -1118,8 +1118,8 @@ void TieredHNSWIndex<DataType, DistType>::TieredHNSW_BatchIterator::filter_irrel
 }
 
 template <typename DataType, typename DistType>
-VecSimIndexInfo TieredHNSWIndex<DataType, DistType>::info() const {
-    auto info = VecSimTieredIndex<DataType, DistType>::info();
+VecSimIndexDebugInfo TieredHNSWIndex<DataType, DistType>::debugInfo() const {
+    auto info = VecSimTieredIndex<DataType, DistType>::debugInfo();
 
     HnswTieredInfo hnswTieredInfo = {.pendingSwapJobsThreshold = this->pendingSwapJobsThreshold};
     info.tieredInfo.specificTieredBackendInfo.hnswTieredInfo = hnswTieredInfo;
@@ -1128,10 +1128,10 @@ VecSimIndexInfo TieredHNSWIndex<DataType, DistType>::info() const {
 }
 
 template <typename DataType, typename DistType>
-VecSimInfoIterator *TieredHNSWIndex<DataType, DistType>::infoIterator() const {
-    VecSimIndexInfo info = this->info();
+VecSimDebugInfoIterator *TieredHNSWIndex<DataType, DistType>::debugInfoIterator() const {
+    VecSimIndexDebugInfo info = this->debugInfo();
     // Get the base tiered fields.
-    auto *infoIterator = VecSimTieredIndex<DataType, DistType>::infoIterator();
+    auto *infoIterator = VecSimTieredIndex<DataType, DistType>::debugInfoIterator();
 
     // Tiered HNSW specific param.
     infoIterator->addInfoField(VecSim_InfoField{

src/VecSim/info_iterator.cpp

@@ -6,22 +6,23 @@
 
 #include "info_iterator_struct.h"
 
-extern "C" size_t VecSimInfoIterator_NumberOfFields(VecSimInfoIterator *infoIterator) {
+extern "C" size_t VecSimDebugInfoIterator_NumberOfFields(VecSimDebugInfoIterator *infoIterator) {
     return infoIterator->numberOfFields();
 }
 
-extern "C" bool VecSimInfoIterator_HasNextField(VecSimInfoIterator *infoIterator) {
+extern "C" bool VecSimDebugInfoIterator_HasNextField(VecSimDebugInfoIterator *infoIterator) {
     return infoIterator->hasNext();
 }
 
-extern "C" VecSim_InfoField *VecSimInfoIterator_NextField(VecSimInfoIterator *infoIterator) {
+extern "C" VecSim_InfoField *
+VecSimDebugInfoIterator_NextField(VecSimDebugInfoIterator *infoIterator) {
     if (infoIterator->hasNext()) {
         return infoIterator->next();
     }
     return NULL;
 }
 
-extern "C" void VecSimInfoIterator_Free(VecSimInfoIterator *infoIterator) {
+extern "C" void VecSimDebugInfoIterator_Free(VecSimDebugInfoIterator *infoIterator) {
     if (infoIterator != NULL) {
         delete infoIterator;
     }

src/VecSim/info_iterator.h

@@ -6,6 +6,7 @@
 
 #pragma once
 #include <stdlib.h>
+#include <stdint.h>
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -15,7 +16,7 @@ extern "C" {
  * the type VecSim_InfoFieldType. This struct exposes an iterator-like API to iterate over the
  * information fields.
  */
-typedef struct VecSimInfoIterator VecSimInfoIterator;
+typedef struct VecSimDebugInfoIterator VecSimDebugInfoIterator;
 
 typedef enum {
     INFOFIELD_STRING,
@@ -26,11 +27,11 @@ typedef enum {
 } VecSim_InfoFieldType;
 
 typedef union {
-    double floatingPointValue;         // Floating point value. 64 bits float.
-    int64_t integerValue;              // Integer value. Signed 64 bits integer.
-    u_int64_t uintegerValue;           // Unsigned value. Unsigned 64 buts integer.
-    const char *stringValue;           // String value.
-    VecSimInfoIterator *iteratorValue; // Iterator value.
+    double floatingPointValue;              // Floating point value. 64 bits float.
+    int64_t integerValue;                   // Integer value. Signed 64 bits integer.
+    uint64_t uintegerValue;                 // Unsigned value. Unsigned 64 bits integer.
+    const char *stringValue;                // String value.
+    VecSimDebugInfoIterator *iteratorValue; // Iterator value.
 } FieldValue;
 
 /**
@@ -51,7 +52,7 @@ typedef struct {
  * @param infoIterator Given info iterator.
  * @return size_t Number of fields.
  */
-size_t VecSimInfoIterator_NumberOfFields(VecSimInfoIterator *infoIterator);
+size_t VecSimDebugInfoIterator_NumberOfFields(VecSimDebugInfoIterator *infoIterator);
 
 /**
  * @brief Returns if the fields iterator is depleted.
@@ -60,22 +61,22 @@ size_t VecSimInfoIterator_NumberOfFields(VecSimInfoIterator *infoIterator);
  * @return true Iterator is not depleted.
  * @return false Otherwise.
  */
-bool VecSimInfoIterator_HasNextField(VecSimInfoIterator *infoIterator);
+bool VecSimDebugInfoIterator_HasNextField(VecSimDebugInfoIterator *infoIterator);
 
 /**
  * @brief Returns a pointer to the next info field.
  *
  * @param infoIterator Given info iterator.
  * @return VecSim_InfoField* A pointer to the next info field.
  */
-VecSim_InfoField *VecSimInfoIterator_NextField(VecSimInfoIterator *infoIterator);
+VecSim_InfoField *VecSimDebugInfoIterator_NextField(VecSimDebugInfoIterator *infoIterator);
 
 /**
  * @brief Free an info iterator.
  *
  * @param infoIterator Given info iterator.
  */
-void VecSimInfoIterator_Free(VecSimInfoIterator *infoIterator);
+void VecSimDebugInfoIterator_Free(VecSimDebugInfoIterator *infoIterator);
 
 #ifdef __cplusplus
 }

src/VecSim/info_iterator_struct.h

@@ -9,13 +9,13 @@
 #include "info_iterator.h"
 #include "VecSim/utils/vecsim_stl.h"
 
-struct VecSimInfoIterator {
+struct VecSimDebugInfoIterator {
 private:
     vecsim_stl::vector<VecSim_InfoField> fields;
     size_t currentIndex;
 
 public:
-    VecSimInfoIterator(size_t len, const std::shared_ptr<VecSimAllocator> &alloc)
+    VecSimDebugInfoIterator(size_t len, const std::shared_ptr<VecSimAllocator> &alloc)
         : fields(alloc), currentIndex(0) {
         this->fields.reserve(len);
     }
@@ -28,7 +28,7 @@ struct VecSimInfoIterator {
 
     inline size_t numberOfFields() { return this->fields.size(); }
 
-    virtual ~VecSimInfoIterator() {
+    virtual ~VecSimDebugInfoIterator() {
         for (size_t i = 0; i < this->fields.size(); i++) {
             if (this->fields[i].fieldType == INFOFIELD_ITERATOR) {
                 delete this->fields[i].fieldValue.iteratorValue;

src/VecSim/vec_sim.cpp

@@ -221,16 +221,22 @@ extern "C" void VecSimIndex_Free(VecSimIndex *index) {
     delete index;
 }
 
-extern "C" VecSimIndexInfo VecSimIndex_Info(VecSimIndex *index) { return index->info(); }
+extern "C" VecSimIndexDebugInfo VecSimIndex_DebugInfo(VecSimIndex *index) {
+    return index->debugInfo();
+}
 
-extern "C" VecSimInfoIterator *VecSimIndex_InfoIterator(VecSimIndex *index) {
-    return index->infoIterator();
+extern "C" VecSimDebugInfoIterator *VecSimIndex_DebugInfoIterator(VecSimIndex *index) {
+    return index->debugInfoIterator();
 }
 
 extern "C" VecSimIndexBasicInfo VecSimIndex_BasicInfo(VecSimIndex *index) {
     return index->basicInfo();
 }
 
+extern "C" VecSimIndexStatsInfo VecSimIndex_StatsInfo(VecSimIndex *index) {
+    return index->statisticInfo();
+}
+
 extern "C" VecSimBatchIterator *VecSimBatchIterator_New(VecSimIndex *index, const void *queryBlob,
                                                         VecSimQueryParams *queryParams) {
     return index->newBatchIteratorWrapper(queryBlob, queryParams);

src/VecSim/vec_sim.h

@@ -152,7 +152,7 @@ VecSimQueryReply *VecSimIndex_RangeQuery(VecSimIndex *index, const void *queryBl
  * @param index the index to return its info.
  * @return Index general and specific meta-data.
  */
-VecSimIndexInfo VecSimIndex_Info(VecSimIndex *index);
+VecSimIndexDebugInfo VecSimIndex_DebugInfo(VecSimIndex *index);
 
 /**
  * @brief Return basic immutable index information.
@@ -161,13 +161,20 @@ VecSimIndexInfo VecSimIndex_Info(VecSimIndex *index);
  */
 VecSimIndexBasicInfo VecSimIndex_BasicInfo(VecSimIndex *index);
 
+/**
+ * @brief Return statistics information.
+ * @param index the index to return its info.
+ * @return Index statistic data.
+ */
+VecSimIndexStatsInfo VecSimIndex_StatsInfo(VecSimIndex *index);
+
 /**
  * @brief Returns an info iterator for generic reply purposes.
  *
  * @param index this index to return its info.
- * @return VecSimInfoIterator* An iterable containing the index general and specific meta-data.
+ * @return VecSimDebugInfoIterator* An iterable containing the index general and specific meta-data.
  */
-VecSimInfoIterator *VecSimIndex_InfoIterator(VecSimIndex *index);
+VecSimDebugInfoIterator *VecSimIndex_DebugInfoIterator(VecSimIndex *index);
 
 /**
  * @brief Create a new batch iterator for a specific index, for a specific query vector,

src/VecSim/vec_sim_common.h

@@ -200,16 +200,25 @@ typedef struct {
  * Index info that is static and immutable (cannot be changed over time)
  */
 typedef struct {
-    VecSimAlgo algo;     // Algorithm being used.
-    size_t blockSize;    // Brute force algorithm vector block (mini matrix) size
+    VecSimAlgo algo;     // Algorithm being used (if index is tiered, this is the backend index).
     VecSimMetric metric; // Index distance metric
     VecSimType type;     // Datatype the index holds.
     bool isMulti;        // Determines if the index should multi-index or not.
+    bool isTiered;       // Is the index is tiered or not.
+    size_t blockSize;    // Brute force algorithm vector block (mini matrix) size
     size_t dim;          // Vector size (dimension).
-
-    bool isTiered; // The algorithm for the tiered index (if algo is tiered).
 } VecSimIndexBasicInfo;
 
+/**
+ * Index info for statistics - a thin and efficient (no locks, no calculations) info. Can be used in
+ * production without worrying about performance
+ */
+typedef struct {
+    size_t memory;
+    size_t numberOfMarkedDeleted; // The number of vectors that are marked as deleted (HNSW/tiered
+                                  // only).
+} VecSimIndexStatsInfo;
+
 typedef struct {
     VecSimIndexBasicInfo basicInfo; // Index immutable meta-data.
     size_t indexSize;               // Current count of vectors.
@@ -257,7 +266,7 @@ typedef struct {
 } tieredInfoStruct;
 
 /**
- * @brief Index information. Mainly used for debug/testing.
+ * @brief Index information. Should only be used for debug/testing.
  *
  */
 typedef struct {
@@ -267,7 +276,7 @@ typedef struct {
         hnswInfoStruct hnswInfo;
         tieredInfoStruct tieredInfo;
     };
-} VecSimIndexInfo;
+} VecSimIndexDebugInfo;
 
 // Memory function declarations.
 typedef void *(*allocFn)(size_t n);

src/VecSim/vec_sim_index.h

@@ -106,6 +106,12 @@ struct VecSimIndexAbstract : public VecSimIndexInterface {
     inline VecSimMetric getMetric() const { return metric; }
     inline size_t getDataSize() const { return dataSize; }
     inline size_t getBlockSize() const { return blockSize; }
+    virtual inline VecSimIndexStatsInfo statisticInfo() const override {
+        return VecSimIndexStatsInfo{
+            .memory = this->getAllocationSize(),
+            .numberOfMarkedDeleted = 0,
+        };
+    }
 
     virtual VecSimQueryReply *rangeQuery(const void *queryBlob, double radius,
                                          VecSimQueryParams *queryParams) const = 0;
@@ -134,7 +140,8 @@ struct VecSimIndexAbstract : public VecSimIndexInterface {
     }
 
     // Adds all common info to the info iterator, besides the block size (currently 8 fields).
-    void addCommonInfoToIterator(VecSimInfoIterator *infoIterator, const CommonInfo &info) const {
+    void addCommonInfoToIterator(VecSimDebugInfoIterator *infoIterator,
+                                 const CommonInfo &info) const {
         infoIterator->addInfoField(VecSim_InfoField{
             .fieldName = VecSimCommonStrings::TYPE_STRING,
             .fieldType = INFOFIELD_STRING,
@@ -195,11 +202,13 @@ struct VecSimIndexAbstract : public VecSimIndexInterface {
      * @return basicInfo
      */
     VecSimIndexBasicInfo getBasicInfo() const {
-        VecSimIndexBasicInfo info{.blockSize = this->blockSize,
-                                  .metric = this->metric,
-                                  .type = this->vecType,
-                                  .isMulti = this->isMulti,
-                                  .dim = this->dim};
+        VecSimIndexBasicInfo info{
+            .metric = this->metric,
+            .type = this->vecType,
+            .isMulti = this->isMulti,
+            .blockSize = this->blockSize,
+            .dim = this->dim,
+        };
         return info;
     }