[8.2] [MOD-10236] Add serialization to SVS index (#782)

[MOD-10236] Add serialization to SVS index (#716)

* generalize

* remove serializer.cpp from cmake

* prepare merge with  rafik commit

* [SVS] Implement Save/Load + test

* seperate hnsw_serializer to h and cpp

* remove get version impl

* save impl

* add load

* change camelcase

* for mat

* generalzie saveIndexFields

* format

* compare metadata on load

* Add checkIntegrity with error

* checkIntegrity

* remove duplicate verification in compare meta data

* format

* svs serializetion version testing

* Revert "svs serializetion version testing"

This reverts commit 9ed7730.

* common serializer test

* remove changes_num from metadata

* Add location c'tor

* Add location ctor and to test

* Remove outdated comment from serializer header

* Enhance documentation for loadIndex function in SVSIndex

* Add comments

* format + remove test

* enable tests

* serializer test

* format

* reset SVS to master

* add logging to test_svs

* format

* remove duplicate NewIndexImpl

* expose loadIndex in VecSimIndex, add BUILD_TEST gurad

* remove string ctor from SVSIndex

* format

* fix BUILD_TEST in svs_factory

* document loadIndex

* move loadIndex to serializer

* remove excess declarations

* remove extra ;

* compatable -> compatible

* remove redundant params from test

* remove comments from threadpool_handle

* remove error context comments

* add checkIntegrity

* update checkIntegrity and format

* move loadIndex to SVSSerializer

* update bindings

* format

* add test

* add single

* adjust labels

* Refactor save_load test to simplify vector generation logic

* add HAVE_SVS guard

* Add missing include for <sstream> in svs_serializer.h

* free faulty index

* Free index

* Improve error message for index loading failure in NewIndex function

* format

---------

Co-authored-by: Rafik Saliev <rafik.f.saliev@intel.com>

Author: lerman25

src/VecSim/CMakeLists.txt

@@ -52,6 +52,9 @@ if (TARGET svs::svs)
 endif()
 
 if(VECSIM_BUILD_TESTS)
-    add_library(VectorSimilaritySerializer utils/serializer.cpp)
+    add_library(VectorSimilaritySerializer
+        algorithms/hnsw/hnsw_serializer.cpp
+        algorithms/svs/svs_serializer.cpp
+    )
     target_link_libraries(VectorSimilarity VectorSimilaritySerializer)
 endif()

src/VecSim/algorithms/hnsw/hnsw.h

@@ -25,6 +25,7 @@
 #ifdef BUILD_TESTS
 #include "hnsw_serialization_utils.h"
 #include "VecSim/utils/serializer.h"
+#include "hnsw_serializer.h"
 #endif
 
 #include <deque>
@@ -85,7 +86,7 @@ class HNSWIndex : public VecSimIndexAbstract<DataType, DistType>,
                   public VecSimIndexTombstone
 #ifdef BUILD_TESTS
     ,
-                  public Serializer
+                  public HNSWSerializer
 #endif
 {
 protected:
@@ -2355,5 +2356,5 @@ HNSWIndex<DataType, DistType>::getHNSWElementNeighbors(size_t label, int ***neig
 }
 
 #ifdef BUILD_TESTS
-#include "hnsw_serializer.h"
+#include "hnsw_serializer_impl.h"
 #endif

src/VecSim/algorithms/hnsw/hnsw_multi.h

@@ -64,7 +64,7 @@ class HNSWIndex_Multi : public HNSWIndex<DataType, DistType> {
     HNSWIndex_Multi(std::ifstream &input, const HNSWParams *params,
                     const AbstractIndexInitParams &abstractInitParams,
                     const IndexComponents<DataType, DistType> &components,
-                    Serializer::EncodingVersion version)
+                    HNSWSerializer::EncodingVersion version)
         : HNSWIndex<DataType, DistType>(input, params, abstractInitParams, components, version),
           labelLookup(this->maxElements, this->allocator) {}
 

src/VecSim/utils/serializer.cpp …cSim/algorithms/hnsw/hnsw_serializer.cppsrc/VecSim/utils/serializer.cpp renamed to src/VecSim/algorithms/hnsw/hnsw_serializer.cpp src/VecSim/utils/serializer.cpp renamed to src/VecSim/algorithms/hnsw/hnsw_serializer.cpp

@@ -7,38 +7,34 @@
  * GNU Affero General Public License v3 (AGPLv3).
  */
 
-#include <fstream>
-#include <string>
+#include "hnsw_serializer.h"
 
-#include "VecSim/utils/serializer.h"
-
-// Persist index into a file in the specified location.
-void Serializer::saveIndex(const std::string &location) {
-
-    // Serializing with the latest version.
-    EncodingVersion version = EncodingVersion_V4;
-
-    std::ofstream output(location, std::ios::binary);
-    writeBinaryPOD(output, version);
-    saveIndexIMP(output);
-    output.close();
-}
-
-Serializer::EncodingVersion Serializer::ReadVersion(std::ifstream &input) {
+HNSWSerializer::HNSWSerializer(EncodingVersion version) : m_version(version) {}
 
+HNSWSerializer::EncodingVersion HNSWSerializer::ReadVersion(std::ifstream &input) {
     input.seekg(0, std::ifstream::beg);
 
-    // The version number is the first field that is serialized.
-    EncodingVersion version = EncodingVersion_INVALID;
+    EncodingVersion version = EncodingVersion::INVALID;
     readBinaryPOD(input, version);
-    if (version <= EncodingVersion_DEPRECATED) {
+
+    if (version <= EncodingVersion::DEPRECATED) {
         input.close();
         throw std::runtime_error("Cannot load index: deprecated encoding version: " +
-                                 std::to_string(version));
-    } else if (version >= EncodingVersion_INVALID) {
+                                 std::to_string(static_cast<int>(version)));
+    } else if (version >= EncodingVersion::INVALID) {
         input.close();
         throw std::runtime_error("Cannot load index: bad encoding version: " +
-                                 std::to_string(version));
+                                 std::to_string(static_cast<int>(version)));
     }
     return version;
 }
+
+void HNSWSerializer::saveIndex(const std::string &location) {
+    EncodingVersion version = EncodingVersion::V4;
+    std::ofstream output(location, std::ios::binary);
+    writeBinaryPOD(output, version);
+    saveIndexIMP(output);
+    output.close();
+}
+
+HNSWSerializer::EncodingVersion HNSWSerializer::getVersion() const { return m_version; }