Rotate int8 v2

.github/workflows/05-windows-build.yml

@@ -110,46 +110,51 @@ jobs:
           python -m pytest python/tests/ --basetemp=./.pytest_tmp
         shell: powershell
 
-      - name: Run C++ Examples
-        run: |
-          cd "$env:GITHUB_WORKSPACE\examples\c++"
-          mkdir build
-          cd build
-          cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release `
-            -DCMAKE_C_COMPILER_LAUNCHER=sccache `
-            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
-          cmake --build . --config Release --parallel $env:NPROC
-
-          # Copy zvec DLLs next to the example executables so Windows can find them.
-          # CMake places DLLs in bin/ (RUNTIME output) and import libs in lib/.
-          $buildDir = "$env:GITHUB_WORKSPACE\build"
-          foreach ($dllName in @("zvec.dll", "zvec_core.dll", "zvec_ailego.dll")) {
-            $found = $false
-            foreach ($sub in @("$buildDir\bin", "$buildDir\bin\Release", "$buildDir\lib", "$buildDir\lib\Release")) {
-              $dllPath = Join-Path $sub $dllName
-              if (Test-Path $dllPath) {
-                Copy-Item $dllPath -Destination . -Force
-                Write-Host "Copied $dllName from $sub"
-                $found = $true
-                break
-              }
-            }
-            if (-not $found) {
-              Write-Host "WARNING: $dllName not found, searching recursively..."
-              $dll = Get-ChildItem -Path $buildDir -Filter $dllName -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1
-              if ($dll) {
-                Copy-Item $dll.FullName -Destination . -Force
-                Write-Host "Copied $dllName from $($dll.DirectoryName)"
-              } else {
-                Write-Error "$dllName not found anywhere under $buildDir"
-              }
-            }
-          }
-
-          .\db-example.exe
-          .\core-example.exe
-          .\ailego-example.exe
-        shell: powershell
+      # C++ examples are NOT built on Windows: they require all-in-one shared
+      # libraries (zvec.dll), but DLL builds are disabled on MSVC due to the
+      # PE/COFF 65535-section hard limit (LNK1189). The Python binding links
+      # to static libraries directly and is unaffected. C++ examples are
+      # tested on Linux/macOS in 03-macos-linux-build.yml.
+      # - name: Run C++ Examples
+      #   run: |
+      #     cd "$env:GITHUB_WORKSPACE\examples\c++"
+      #     mkdir build
+      #     cd build
+      #     cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release `
+      #       -DCMAKE_C_COMPILER_LAUNCHER=sccache `
+      #       -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+      #     cmake --build . --config Release --parallel $env:NPROC
+      #
+      #     # Copy zvec DLLs next to the example executables so Windows can find them.
+      #     # CMake places DLLs in bin/ (RUNTIME output) and import libs in lib/.
+      #     $buildDir = "$env:GITHUB_WORKSPACE\build"
+      #     foreach ($dllName in @("zvec.dll", "zvec_core.dll", "zvec_ailego.dll")) {
+      #       $found = $false
+      #       foreach ($sub in @("$buildDir\bin", "$buildDir\bin\Release", "$buildDir\lib", "$buildDir\lib\Release")) {
+      #         $dllPath = Join-Path $sub $dllName
+      #         if (Test-Path $dllPath) {
+      #           Copy-Item $dllPath -Destination . -Force
+      #           Write-Host "Copied $dllName from $sub"
+      #           $found = $true
+      #           break
+      #         }
+      #       }
+      #       if (-not $found) {
+      #         Write-Host "WARNING: $dllName not found, searching recursively..."
+      #         $dll = Get-ChildItem -Path $buildDir -Filter $dllName -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1
+      #         if ($dll) {
+      #           Copy-Item $dll.FullName -Destination . -Force
+      #           Write-Host "Copied $dllName from $($dll.DirectoryName)"
+      #         } else {
+      #           Write-Error "$dllName not found anywhere under $buildDir"
+      #         }
+      #       }
+      #     }
+      #
+      #     .\db-example.exe
+      #     .\core-example.exe
+      #     .\ailego-example.exe
+      #   shell: powershell
 
       - name: Show sccache statistics
         if: always()

.gitignore

@@ -52,3 +52,9 @@ allure-*
 !build_android.sh
 !build_ios.sh
 
+# congfig
+doc/
+config/
+examples/python/
+examples/c_api/
+logs/

examples/c/index_example.c

@@ -85,6 +85,11 @@ int main() {
   zvec_index_params_set_metric_type(hnsw_params_fast, ZVEC_METRIC_TYPE_L2);
   zvec_index_params_set_hnsw_params(hnsw_params_fast, 16, 100);
 
+  // Demonstrate INT8 quantization with random rotation preprocessing
+  // (enable_rotate rotates vectors before INT8 quantization to reduce error)
+  zvec_index_params_set_quantize_type(hnsw_params_fast, ZVEC_QUANTIZE_TYPE_INT8);
+  zvec_index_params_set_quantizer_enable_rotate(hnsw_params_fast, true);
+
   zvec_index_params_t *hnsw_params_balanced =
       zvec_index_params_create(ZVEC_INDEX_TYPE_HNSW);
   if (!hnsw_params_balanced) {

python/tests/test_params.py

@@ -36,6 +36,7 @@
     IndexType,
     MetricType,
     QuantizeType,
+    QuantizerParam,
     DataType,
     VectorSchema,
 )
@@ -452,3 +453,88 @@ def test_isinstance_compatibility(self):
             warnings.simplefilter("always")
             vq = VectorQuery(field_name="embedding", id="doc123")
         assert isinstance(vq, Query)
+
+
+# ----------------------------
+# QuantizerParam Test Case
+# ----------------------------
+
+
+class TestQuantizerParam:
+    def test_default(self):
+        qp = QuantizerParam()
+        assert qp.enable_rotate is False
+
+    def test_enable_rotate_true(self):
+        qp = QuantizerParam(enable_rotate=True)
+        assert qp.enable_rotate is True
+
+    def test_enable_rotate_false(self):
+        qp = QuantizerParam(enable_rotate=False)
+        assert qp.enable_rotate is False
+
+    def test_equality(self):
+        qp1 = QuantizerParam(enable_rotate=True)
+        qp2 = QuantizerParam(enable_rotate=True)
+        qp3 = QuantizerParam(enable_rotate=False)
+        assert qp1 == qp2
+        assert qp1 != qp3
+
+    def test_to_dict(self):
+        qp = QuantizerParam(enable_rotate=True)
+        d = qp.to_dict()
+        assert isinstance(d, dict)
+        assert d.get("enable_rotate") is True
+
+    def test_repr(self):
+        qp = QuantizerParam(enable_rotate=True)
+        r = repr(qp)
+        assert "enable_rotate" in r or "QuantizerParam" in r
+
+    def test_pickle_roundtrip(self):
+        import pickle
+
+        qp = QuantizerParam(enable_rotate=True)
+        data = pickle.dumps(qp)
+        qp2 = pickle.loads(data)
+        assert qp2.enable_rotate is True
+        assert qp == qp2
+
+
+# ----------------------------
+# HnswIndexParam with QuantizerParam
+# ----------------------------
+
+
+class TestHnswIndexParamQuantizer:
+    def test_default_quantizer_param(self):
+        param = HnswIndexParam()
+        assert param.quantizer_param is not None
+        assert param.quantizer_param.enable_rotate is False
+
+    def test_with_quantizer_param(self):
+        qp = QuantizerParam(enable_rotate=True)
+        param = HnswIndexParam(
+            metric_type=MetricType.L2,
+            quantize_type=QuantizeType.INT8,
+            quantizer_param=qp,
+        )
+        assert param.quantizer_param.enable_rotate is True
+        assert param.quantize_type == QuantizeType.INT8
+
+
+# ----------------------------
+# FlatIndexParam with QuantizerParam
+# ----------------------------
+
+
+class TestFlatIndexParamQuantizer:
+    def test_with_quantizer_param(self):
+        qp = QuantizerParam(enable_rotate=True)
+        param = FlatIndexParam(
+            metric_type=MetricType.L2,
+            quantize_type=QuantizeType.INT8,
+            quantizer_param=qp,
+        )
+        assert param.quantizer_param.enable_rotate is True
+        assert param.quantize_type == QuantizeType.INT8

python/zvec/__init__.py

@@ -108,6 +108,7 @@
     IVFIndexParam,
     IVFQueryParam,
     OptimizeOption,
+    QuantizerParam,
     VamanaIndexParam,
     VamanaQueryParam,
 )
@@ -171,6 +172,7 @@
     "HnswQueryParam",
     "HnswRabitqQueryParam",
     "IVFQueryParam",
+    "QuantizerParam",
     "VamanaIndexParam",
     "VamanaQueryParam",
     # Extensions

python/zvec/__init__.pyi

@@ -28,6 +28,7 @@ from .model.param import (
     IVFIndexParam,
     IVFQueryParam,
     OptimizeOption,
+    QuantizerParam,
     VamanaIndexParam,
     VamanaQueryParam,
 )
@@ -74,6 +75,7 @@ __all__: list = [
     "MetricType",
     "OptimizeOption",
     "QuantizeType",
+    "QuantizerParam",
     "Query",
     "ReRanker",
     "RrfReRanker",

python/zvec/model/param/__init__.py

@@ -31,6 +31,7 @@
     IVFIndexParam,
     IVFQueryParam,
     OptimizeOption,
+    QuantizerParam,
     VamanaIndexParam,
     VamanaQueryParam,
 )
@@ -53,6 +54,7 @@
     "IndexOption",
     "InvertIndexParam",
     "OptimizeOption",
+    "QuantizerParam",
     "VamanaIndexParam",
     "VamanaQueryParam",
 ]

python/zvec/model/param/__init__.pyi

@@ -24,6 +24,7 @@ __all__: list[str] = [
     "IndexParam",
     "InvertIndexParam",
     "OptimizeOption",
+    "QuantizerParam",
     "QueryParam",
     "SegmentOption",
     "VectorIndexParam",
@@ -145,6 +146,8 @@ class FlatIndexParam(VectorIndexParam):
         quantize_type (QuantizeType): Optional quantization type for vector
             compression (e.g., FP16, INT8). Use ``QuantizeType.UNDEFINED`` to
             disable quantization. Default is ``QuantizeType.UNDEFINED``.
+        quantizer_param (QuantizerParam): Quantizer configuration (e.g., enable_rotate).
+            Default is ``QuantizerParam()``.
 
     Examples:
         >>> from zvec.typing import MetricType, QuantizeType
@@ -161,6 +164,7 @@ class FlatIndexParam(VectorIndexParam):
         self,
         metric_type: _zvec.typing.MetricType = ...,
         quantize_type: _zvec.typing.QuantizeType = ...,
+        quantizer_param: QuantizerParam = ...,
     ) -> None:
         """
         Constructs a FlatIndexParam instance.
@@ -169,6 +173,8 @@ class FlatIndexParam(VectorIndexParam):
             metric_type (MetricType, optional): Distance metric. Defaults to MetricType.IP.
             quantize_type (QuantizeType, optional): Vector quantization type.
                 Defaults to QuantizeType.UNDEFINED (no quantization).
+            quantizer_param (QuantizerParam, optional): Quantizer configuration.
+                Defaults to QuantizerParam().
         """
 
     def __repr__(self) -> str: ...
@@ -224,6 +230,7 @@ class HnswIndexParam(VectorIndexParam):
         ef_construction: typing.SupportsInt = 500,
         quantize_type: _zvec.typing.QuantizeType = ...,
         use_contiguous_memory: bool = False,
+        quantizer_param: QuantizerParam = ...,
     ) -> None: ...
     def __repr__(self) -> str: ...
     def __setstate__(self, arg0: tuple) -> None: ...
@@ -496,6 +503,7 @@ class IVFIndexParam(VectorIndexParam):
         n_iters: typing.SupportsInt = 10,
         use_soar: bool = False,
         quantize_type: _zvec.typing.QuantizeType = ...,
+        quantizer_param: QuantizerParam = ...,
     ) -> None:
         """
         Constructs an IVFIndexParam instance.
@@ -509,6 +517,8 @@ class IVFIndexParam(VectorIndexParam):
             use_soar (bool, optional): Enable SOAR optimization. Defaults to False.
             quantize_type (QuantizeType, optional): Vector quantization type.
                 Defaults to QuantizeType.UNDEFINED.
+            quantizer_param (QuantizerParam, optional): Quantizer configuration.
+                Defaults to QuantizerParam().
         """
 
     def __repr__(self) -> str: ...
@@ -912,6 +922,49 @@ class SegmentOption:
         bool: Whether the segment is read-only.
         """
 
+class QuantizerParam:
+    """
+
+    Parameters for quantizer configuration.
+
+    Encapsulates quantization-related settings such as enable_rotate.
+    Designed for future extensibility.
+
+    Attributes:
+        enable_rotate (bool): Whether to apply random rotation before INT8
+            quantization to reduce quantization error.
+            Only effective with quantize_type=INT8. Defaults to False.
+
+    Examples:
+        >>> qp = QuantizerParam(enable_rotate=True)
+        >>> print(qp.enable_rotate)
+        True
+    """
+
+    def __getstate__(self) -> tuple: ...
+    def __init__(self, enable_rotate: bool = False) -> None:
+        """
+        Constructs a QuantizerParam instance.
+
+        Args:
+            enable_rotate (bool, optional): Whether to apply random rotation
+                before INT8 quantization. Defaults to False.
+        """
+
+    def __repr__(self) -> str: ...
+    def __setstate__(self, arg0: tuple) -> None: ...
+    def __eq__(self, arg0: typing.Any) -> bool: ...
+    def to_dict(self) -> dict:
+        """
+        Convert to dictionary with all fields
+        """
+
+    @property
+    def enable_rotate(self) -> bool:
+        """
+        bool: Whether random rotation is enabled before INT8 quantization.
+        """
+
 class VectorIndexParam(IndexParam):
     """
 
@@ -923,6 +976,7 @@ class VectorIndexParam(IndexParam):
         type (IndexType): The specific vector index type (e.g., HNSW, FLAT).
         metric_type (MetricType): Distance metric used for similarity search.
         quantize_type (QuantizeType): Optional vector quantization type.
+        quantizer_param (QuantizerParam): Quantizer configuration (e.g., enable_rotate).
     """
 
     def __getstate__(self) -> tuple: ...
@@ -944,6 +998,12 @@ class VectorIndexParam(IndexParam):
         QuantizeType: Vector quantization type (e.g., FP16, INT8).
         """
 
+    @property
+    def quantizer_param(self) -> QuantizerParam:
+        """
+        QuantizerParam: Quantizer configuration including enable_rotate.
+        """
+
 class _SearchQuery:
     field_name: str
     filter: str