[0.6] Refactor Info Report - [MOD-9321] (#657)

* Refactor Info Report - [MOD-9321] (#650)

* refactor `info()` API to indicate it should be used for debug, and prepare structs and new API for statistics info

* test fixes

* implement new API

* format

* improve tests

* move tiered statisticInfo to base class

* refactor

* minor simplification

* review fixes

(cherry picked from commit 5f7dbdf)

* fixes for 0.6

* Add back missing debug info comparison and iterator free

Author: GuyAv46

src/VecSim/algorithms/brute_force/brute_force.h

@@ -43,8 +43,8 @@ class BruteForceIndex : public VecSimIndexAbstract<DistType> {
                                              VecSimQueryParams *queryParams) override;
     VecSimQueryResult_List rangeQuery(const void *queryBlob, double radius,
                                       VecSimQueryParams *queryParams) override;
-    virtual VecSimIndexInfo info() const override;
-    virtual VecSimInfoIterator *infoIterator() const override;
+    VecSimIndexDebugInfo debugInfo() const override;
+    VecSimDebugInfoIterator *debugInfoIterator() const override;
     virtual VecSimBatchIterator *newBatchIterator(const void *queryBlob,
                                                   VecSimQueryParams *queryParams) const override;
     bool preferAdHocSearch(size_t subsetSize, size_t k, bool initial_check) override;
@@ -329,9 +329,9 @@ BruteForceIndex<DataType, DistType>::rangeQuery(const void *queryBlob, double ra
 }
 
 template <typename DataType, typename DistType>
-VecSimIndexInfo BruteForceIndex<DataType, DistType>::info() const {
+VecSimIndexDebugInfo BruteForceIndex<DataType, DistType>::debugInfo() const {
 
-    VecSimIndexInfo info;
+    VecSimIndexDebugInfo info;
     info.algo = VecSimAlgo_BF;
     info.bfInfo.dim = this->dim;
     info.bfInfo.type = this->vecType;
@@ -346,11 +346,11 @@ VecSimIndexInfo BruteForceIndex<DataType, DistType>::info() const {
 }
 
 template <typename DataType, typename DistType>
-VecSimInfoIterator *BruteForceIndex<DataType, DistType>::infoIterator() const {
-    VecSimIndexInfo info = this->info();
+VecSimDebugInfoIterator *BruteForceIndex<DataType, DistType>::debugInfoIterator() const {
+    VecSimIndexDebugInfo info = this->debugInfo();
     // For readability. Update this number when needed.
     size_t numberOfInfoFields = 8;
-    VecSimInfoIterator *infoIterator = new VecSimInfoIterator(numberOfInfoFields);
+    auto *infoIterator = new VecSimDebugInfoIterator(numberOfInfoFields);
 
     infoIterator->addInfoField(VecSim_InfoField{
         .fieldName = VecSimCommonStrings::ALGORITHM_STRING,

src/VecSim/algorithms/hnsw/hnsw.h

@@ -192,8 +192,8 @@ class HNSWIndex : public VecSimIndexAbstract<DistType>
     inline labelType getEntryPointLabel() const;
     inline labelType getExternalLabel(idType internal_id) const;
     inline VisitedNodesHandler *getVisitedList() const;
-    VecSimIndexInfo info() const override;
-    VecSimInfoIterator *infoIterator() const override;
+    VecSimIndexDebugInfo debugInfo() const override;
+    VecSimDebugInfoIterator *debugInfoIterator() const override;
     bool preferAdHocSearch(size_t subsetSize, size_t k, bool initial_check) override;
     char *getDataByInternalId(idType internal_id) const;
     inline linklistsizeint *get_linklist_at_level(idType internal_id, size_t level) const;
@@ -1539,9 +1539,9 @@ VecSimQueryResult_List HNSWIndex<DataType, DistType>::rangeQuery(const void *que
 }
 
 template <typename DataType, typename DistType>
-VecSimIndexInfo HNSWIndex<DataType, DistType>::info() const {
+VecSimIndexDebugInfo HNSWIndex<DataType, DistType>::debugInfo() const {
 
-    VecSimIndexInfo info;
+    VecSimIndexDebugInfo info;
     info.algo = VecSimAlgo_HNSWLIB;
     info.hnswInfo.dim = this->dim;
     info.hnswInfo.type = this->vecType;
@@ -1562,11 +1562,11 @@ VecSimIndexInfo HNSWIndex<DataType, DistType>::info() const {
 }
 
 template <typename DataType, typename DistType>
-VecSimInfoIterator *HNSWIndex<DataType, DistType>::infoIterator() const {
-    VecSimIndexInfo info = this->info();
+VecSimDebugInfoIterator *HNSWIndex<DataType, DistType>::debugInfoIterator() const {
+    VecSimIndexDebugInfo info = this->debugInfo();
     // For readability. Update this number when needed.
     size_t numberOfInfoFields = 12;
-    VecSimInfoIterator *infoIterator = new VecSimInfoIterator(numberOfInfoFields);
+    auto *infoIterator = new VecSimDebugInfoIterator(numberOfInfoFields);
 
     infoIterator->addInfoField(VecSim_InfoField{
         .fieldName = VecSimCommonStrings::ALGORITHM_STRING,

src/VecSim/info_iterator.cpp

@@ -6,22 +6,23 @@
 
 #include "info_iterator_struct.h"
 
-extern "C" size_t VecSimInfoIterator_NumberOfFields(VecSimInfoIterator *infoIterator) {
+extern "C" size_t VecSimDebugInfoIterator_NumberOfFields(VecSimDebugInfoIterator *infoIterator) {
     return infoIterator->numberOfFields();
 }
 
-extern "C" bool VecSimInfoIterator_HasNextField(VecSimInfoIterator *infoIterator) {
+extern "C" bool VecSimDebugInfoIterator_HasNextField(VecSimDebugInfoIterator *infoIterator) {
     return infoIterator->hasNext();
 }
 
-extern "C" VecSim_InfoField *VecSimInfoIterator_NextField(VecSimInfoIterator *infoIterator) {
+extern "C" VecSim_InfoField *
+VecSimDebugInfoIterator_NextField(VecSimDebugInfoIterator *infoIterator) {
     if (infoIterator->hasNext()) {
         return infoIterator->next();
     }
     return NULL;
 }
 
-extern "C" void VecSimInfoIterator_Free(VecSimInfoIterator *infoIterator) {
+extern "C" void VecSimDebugInfoIterator_Free(VecSimDebugInfoIterator *infoIterator) {
     if (infoIterator != NULL) {
         delete infoIterator;
     }

src/VecSim/info_iterator.h

@@ -6,6 +6,7 @@
 
 #pragma once
 #include <stdlib.h>
+#include <stdint.h>
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -15,7 +16,7 @@ extern "C" {
  * the type VecSim_InfoFieldType. This struct exposes an iterator-like API to iterate over the
  * information fields.
  */
-typedef struct VecSimInfoIterator VecSimInfoIterator;
+typedef struct VecSimDebugInfoIterator VecSimDebugInfoIterator;
 
 typedef enum {
     INFOFIELD_STRING,
@@ -49,7 +50,7 @@ typedef struct {
  * @param infoIterator Given info iterator.
  * @return size_t Number of fields.
  */
-size_t VecSimInfoIterator_NumberOfFields(VecSimInfoIterator *infoIterator);
+size_t VecSimDebugInfoIterator_NumberOfFields(VecSimDebugInfoIterator *infoIterator);
 
 /**
  * @brief Returns if the fields iterator is depleted.
@@ -58,22 +59,22 @@ size_t VecSimInfoIterator_NumberOfFields(VecSimInfoIterator *infoIterator);
  * @return true Iterator is not depleted.
  * @return false Otherwise.
  */
-bool VecSimInfoIterator_HasNextField(VecSimInfoIterator *infoIterator);
+bool VecSimDebugInfoIterator_HasNextField(VecSimDebugInfoIterator *infoIterator);
 
 /**
  * @brief Returns a pointer to the next info field.
  *
  * @param infoIterator Given info iterator.
  * @return VecSim_InfoField* A pointer to the next info field.
  */
-VecSim_InfoField *VecSimInfoIterator_NextField(VecSimInfoIterator *infoIterator);
+VecSim_InfoField *VecSimDebugInfoIterator_NextField(VecSimDebugInfoIterator *infoIterator);
 
 /**
  * @brief Free an info iterator.
  *
  * @param infoIterator Given info iterator.
  */
-void VecSimInfoIterator_Free(VecSimInfoIterator *infoIterator);
+void VecSimDebugInfoIterator_Free(VecSimDebugInfoIterator *infoIterator);
 
 #ifdef __cplusplus
 }

src/VecSim/info_iterator_struct.h

@@ -9,13 +9,14 @@
 #include "info_iterator.h"
 #include "VecSim/utils/arr_cpp.h"
 
-struct VecSimInfoIterator {
+struct VecSimDebugInfoIterator {
 private:
     VecSim_InfoField *fields;
     size_t currentIndex;
 
 public:
-    VecSimInfoIterator(size_t len) : fields(array_new<VecSim_InfoField>(len)), currentIndex(0) {}
+    VecSimDebugInfoIterator(size_t len)
+        : fields(array_new<VecSim_InfoField>(len)), currentIndex(0) {}
 
     inline void addInfoField(VecSim_InfoField infoField) {
         this->fields = array_append(this->fields, infoField);
@@ -27,5 +28,5 @@ struct VecSimInfoIterator {
 
     inline size_t numberOfFields() { return array_len(this->fields); }
 
-    virtual ~VecSimInfoIterator() { array_free(this->fields); }
+    virtual ~VecSimDebugInfoIterator() { array_free(this->fields); }
 };

src/VecSim/vec_sim.cpp

@@ -172,7 +172,7 @@ extern "C" VecSimResolveCode VecSimIndex_ResolveParams(VecSimIndex *index, VecSi
     if (!qparams || (!rparams && (paramNum != 0))) {
         return VecSimParamResolverErr_NullParam;
     }
-    VecSimAlgo index_type = index->info().algo;
+    VecSimAlgo index_type = index->debugInfo().algo;
     bzero(qparams, sizeof(VecSimQueryParams));
     auto res = VecSimParamResolver_OK;
     for (int i = 0; i < paramNum; i++) {
@@ -256,10 +256,16 @@ extern "C" void VecSimIndex_Free(VecSimIndex *index) {
     delete index;
 }
 
-extern "C" VecSimIndexInfo VecSimIndex_Info(VecSimIndex *index) { return index->info(); }
+extern "C" VecSimIndexDebugInfo VecSimIndex_DebugInfo(VecSimIndex *index) {
+    return index->debugInfo();
+}
+
+extern "C" VecSimDebugInfoIterator *VecSimIndex_DebugInfoIterator(VecSimIndex *index) {
+    return index->debugInfoIterator();
+}
 
-extern "C" VecSimInfoIterator *VecSimIndex_InfoIterator(VecSimIndex *index) {
-    return index->infoIterator();
+extern "C" VecSimIndexStatsInfo VecSimIndex_StatsInfo(VecSimIndex *index) {
+    return index->statisticInfo();
 }
 
 extern "C" VecSimBatchIterator *VecSimBatchIterator_New(VecSimIndex *index, const void *queryBlob,

src/VecSim/vec_sim.h

@@ -146,15 +146,22 @@ VecSimQueryResult_List VecSimIndex_RangeQuery(VecSimIndex *index, const void *qu
  * @param index the index to return its info.
  * @return Index general and specific meta-data.
  */
-VecSimIndexInfo VecSimIndex_Info(VecSimIndex *index);
+VecSimIndexDebugInfo VecSimIndex_DebugInfo(VecSimIndex *index);
+
+/**
+ * @brief Return statistics information.
+ * @param index the index to return its info.
+ * @return Index statistic data.
+ */
+VecSimIndexStatsInfo VecSimIndex_StatsInfo(VecSimIndex *index);
 
 /**
  * @brief Returns an info iterator for generic reply purposes.
  *
  * @param index this index to return its info.
- * @return VecSimInfoIterator* An iterable containing the index general and specific meta-data.
+ * @return VecSimDebugInfoIterator* An iterable containing the index general and specific meta-data.
  */
-VecSimInfoIterator *VecSimIndex_InfoIterator(VecSimIndex *index);
+VecSimDebugInfoIterator *VecSimIndex_DebugInfoIterator(VecSimIndex *index);
 
 /**
  * @brief Create a new batch iterator for a specific index, for a specific query vector,

src/VecSim/vec_sim_common.h

@@ -140,7 +140,15 @@ typedef struct {
 } VecSimQueryParams;
 
 /**
- * @brief Index information. Mainly used for debug/testing.
+ * Index info for statistics - a thin and efficient (no locks, no calculations) info. Can be used in
+ * production without worrying about performance
+ */
+typedef struct {
+    size_t memory;
+} VecSimIndexStatsInfo;
+
+/**
+ * @brief Index information. Should only be used for debug/testing.
  *
  */
 typedef struct {
@@ -175,7 +183,7 @@ typedef struct {
         } bfInfo;
     };
     VecSimAlgo algo; // Algorithm being used.
-} VecSimIndexInfo;
+} VecSimIndexDebugInfo;
 
 // Memory function declarations.
 typedef void *(*allocFn)(size_t n);

src/VecSim/vec_sim_index.h

@@ -58,4 +58,9 @@ struct VecSimIndexAbstract : public VecSimIndexInterface {
     inline size_t GetDim() const { return dim; }
     inline void setLastSearchMode(VecSearchMode mode) override { this->last_mode = mode; }
     inline bool isMultiValue() const { return isMulti; }
+    virtual inline VecSimIndexStatsInfo statisticInfo() const override {
+        return VecSimIndexStatsInfo{
+            .memory = static_cast<size_t>(this->getAllocationSize()),
+        };
+    }
 };

src/VecSim/vec_sim_interface.h

@@ -110,14 +110,21 @@ struct VecSimIndexInterface : public VecsimBaseObject {
      *
      * @return Index general and specific meta-data.
      */
-    virtual VecSimIndexInfo info() const = 0;
+    virtual VecSimIndexDebugInfo debugInfo() const = 0;
+
+    /**
+     * @brief Return index statistic information.
+     *
+     * @return Index general and specific statistic data (for quick and lock-less retrieval)
+     */
+    virtual VecSimIndexStatsInfo statisticInfo() const = 0;
 
     /**
      * @brief Returns an index information in an iterable structure.
      *
-     * @return VecSimInfoIterator Index general and specific meta-data.
+     * @return VecSimDebugInfoIterator Index general and specific meta-data.
      */
-    virtual VecSimInfoIterator *infoIterator() const = 0;
+    virtual VecSimDebugInfoIterator *debugInfoIterator() const = 0;
 
     /**
      * @brief Create a new batch iterator for a specific index, for a specific query vector,