feat/diskann dynamic load

.github/workflows/03-macos-linux-build.yml

@@ -54,6 +54,14 @@ jobs:
           sudo apt-get install -y clang libomp-dev
         shell: bash
 
+      - name: Install AIO 
+        if: runner.os == 'Linux' && runner.arch == 'X64'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            libaio-dev
+        shell: bash
+
       - name: Print CPU info
         if: runner.os == 'Linux'
         run: lscpu
@@ -89,7 +97,7 @@ jobs:
             pytest \
             scikit-build-core \
             setuptools_scm
-        shell: bash
+        shell: bash       
 
       - name: Build from source
         run: |

.github/workflows/clang_tidy.yml

@@ -29,6 +29,14 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y clang-tidy=1:18.0-59~exp2 cmake ninja-build libomp-dev
 
+      - name: Install AIO 
+        if: runner.os == 'Linux' && runner.arch == 'X64'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            libaio-dev
+        shell: bash
+
       - name: Configure CMake and export compile commands
         run: |
           cmake -S . -B build -G Ninja \

.gitmodules

@@ -43,3 +43,6 @@
 [submodule "thirdparty/RaBitQ-Library/RaBitQ-Library-0.1"]
 	path = thirdparty/RaBitQ-Library/RaBitQ-Library-0.1
 	url = https://github.com/VectorDB-NTU/RaBitQ-Library.git
+[submodule "thirdparty/aio/libaio-0.3"]
+	path = thirdparty/aio/libaio-0.3
+	url = https://github.com/yugabyte/libaio.git

CMakeLists.txt

@@ -109,6 +109,17 @@ else()
 endif()
 message(STATUS "RABITQ_ARCH_FLAG: ${RABITQ_ARCH_FLAG}")
 
+# DiskAnn support (Linux x86_64 only, requires libaio)
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386" AND NOT ANDROID AND NOT IOS)
+  set(DISKANN_SUPPORTED ON)
+  add_definitions(-DDISKANN_SUPPORTED=1)
+else()
+  set(DISKANN_SUPPORTED OFF)
+  add_definitions(-DDISKANN_SUPPORTED=0)
+  message(STATUS "DiskAnn support disabled - only supported on Linux x86_64")
+endif()
+message(STATUS "DISKANN_SUPPORTED: ${DISKANN_SUPPORTED}")
+
 option(USE_OSS_MIRROR "Use OSS mirror for faster third-party downloads" OFF)
 if(DEFINED ENV{USE_OSS_MIRROR} AND NOT "$ENV{USE_OSS_MIRROR}" STREQUAL "")
   set(USE_OSS_MIRROR "$ENV{USE_OSS_MIRROR}" CACHE BOOL "Use OSS mirror for faster third-party downloads" FORCE)
@@ -145,4 +156,16 @@ if(BUILD_PYTHON_BINDINGS)
 
     message(STATUS "Zvec install path: ${ZVEC_PY_INSTALL_DIR}")
     install(TARGETS _zvec LIBRARY DESTINATION ${ZVEC_PY_INSTALL_DIR})
+
+    # DiskAnn ships as a runtime-loaded shared module
+    # (libzvec_diskann_plugin.so) that is brought online implicitly the
+    # first time a DiskAnn index is created — users never call any load
+    # function. The Python extension resolves the module next to _zvec.so
+    # (see the $ORIGIN rpath in src/binding/python/CMakeLists.txt); the
+    # module must therefore be installed alongside _zvec.so in the wheel.
+    # The target exists only on platforms where DiskAnn is buildable
+    # (currently Linux x86_64 with libaio).
+    if(TARGET core_knn_diskann)
+        install(TARGETS core_knn_diskann LIBRARY DESTINATION ${ZVEC_PY_INSTALL_DIR})
+    endif()
 endif()

python/tests/detail/fixture_helper.py

@@ -1,5 +1,13 @@
 import pytest
 import logging
+import platform
+
+DISKANN_SUPPORTED = platform.system() == "Linux" and platform.machine() in (
+    "x86_64",
+    "AMD64",
+    "i686",
+    "i386",
+)
 
 from typing import Any, Generator
 from zvec.typing import DataType, StatusCode, MetricType, QuantizeType
@@ -97,6 +105,12 @@ def full_schema_new(request) -> CollectionSchema:
     else:
         nullable, has_index, vector_index = True, False, HnswIndexParam()
 
+    # Skip DiskAnn tests on unsupported platforms
+    from zvec.model.param import DiskAnnIndexParam
+
+    if isinstance(vector_index, DiskAnnIndexParam) and not DISKANN_SUPPORTED:
+        pytest.skip("DiskAnn only supported on Linux x86_64")
+
     scalar_index_param = None
     vector_index_param = None
     if has_index:

python/tests/detail/test_collection_recall.py

@@ -22,8 +22,10 @@
     HnswIndexParam,
     FlatIndexParam,
     IVFIndexParam,
+    DiskAnnIndexParam,
     HnswQueryParam,
     IVFQueryParam,
+    DiskAnnQueryParam,
 )
 
 from zvec.model.schema import FieldSchema, VectorSchema
@@ -179,10 +181,24 @@ def get_ground_truth_map(collection, test_docs, query_vectors_map, metric_type,
     for field_name, query_vectors in query_vectors_map.items():
         ground_truth_map[field_name] = {}
 
+        # Support per-field metric type: metric_type can be a dict mapping
+        # field_name -> MetricType, or a single MetricType applied to all fields.
+        if isinstance(metric_type, dict):
+            field_metric = metric_type.get(field_name, MetricType.IP)
+        else:
+            field_metric = metric_type
+
         for i, query_vector in enumerate(query_vectors):
             # Get the ground truth for this query
             relevant_doc_ids_scores = get_ground_truth_for_vector_query(
-                collection, query_vector, field_name, test_docs, i, metric_type, k, True
+                collection,
+                query_vector,
+                field_name,
+                test_docs,
+                i,
+                field_metric,
+                k,
+                True,
             )
             ground_truth_map[field_name][i] = relevant_doc_ids_scores
 
@@ -292,6 +308,7 @@ class TestRecall:
         [
             (True, True, HnswIndexParam()),
             (False, True, IVFIndexParam()),
+            (False, True, DiskAnnIndexParam()),
             (False, True, FlatIndexParam()),  # ——ok
             (
                 True,
@@ -371,6 +388,19 @@ class TestRecall:
                     use_soar=False,
                 ),
             ),
+            (
+                True,
+                True,
+                DiskAnnIndexParam(
+                    metric_type=MetricType.IP,
+                    max_degree=32,
+                ),
+            ),
+            (
+                True,
+                True,
+                DiskAnnIndexParam(metric_type=MetricType.L2, max_degree=32),
+            ),
         ],
         indirect=True,
     )
@@ -388,10 +418,16 @@ def test_recall_with_single_vector_valid_500(
     ):
         full_schema_params = request.getfixturevalue("full_schema_new")
 
+        # Build per-field metric type map so ground truth uses each field's
+        # actual index metric (fields may fall back to HnswIndexParam/IP).
+        field_metric_map = {}
         for vector_para in full_schema_params.vectors:
-            if vector_para.name == "vector_fp32_field":
-                metric_type = vector_para.index_param.metric_type
-                break
+            if vector_para.index_param is not None:
+                field_metric_map[vector_para.name] = vector_para.index_param.metric_type
+            else:
+                field_metric_map[vector_para.name] = MetricType.IP
+
+        metric_type = field_metric_map.get("vector_fp32_field", MetricType.IP)
 
         multiple_docs = [
             generate_doc_recall(i, full_collection_new.schema) for i in range(doc_num)
@@ -438,9 +474,13 @@ def test_recall_with_single_vector_valid_500(
                 multiple_docs[i].vectors[field_name] for i in range(query_num)
             ]
 
-        # Get ground truth mapping
+        # Get ground truth mapping (pass per-field metric map)
         ground_truth_map = get_ground_truth_map(
-            full_collection_new, multiple_docs, query_vectors_map, metric_type, top_k
+            full_collection_new,
+            multiple_docs,
+            query_vectors_map,
+            field_metric_map,
+            top_k,
         )
 
         # Validate ground truth mapping structure
@@ -479,8 +519,8 @@ def test_recall_with_single_vector_valid_500(
 
         print("(recall_at_k_stats:\n")
         print(recall_at_k_stats)
-        print("metric_type:")
-        print(metric_type)
+        print("field_metric_map:")
+        print(field_metric_map)
         # Print Recall@K statistics
         print(f"Recall@{top_k} using Ground Truth:")
         for field_name, stats in recall_at_k_stats.items():
@@ -552,7 +592,21 @@ def test_recall_with_single_vector_valid_500(
                     use_soar=True,
                 ),
             ),
-            # (True, True, IVFIndexParam(metric_type=MetricType.COSINE, n_list=150, n_iters=15, use_soar=False, )),
+            (
+                True,
+                True,
+                DiskAnnIndexParam(metric_type=MetricType.IP, max_degree=32),
+            ),
+            (
+                True,
+                True,
+                DiskAnnIndexParam(metric_type=MetricType.L2, max_degree=32),
+            ),
+            (
+                True,
+                True,
+                DiskAnnIndexParam(metric_type=MetricType.COSINE, max_degree=32),
+            ),
         ],
         indirect=True,
     )
@@ -571,10 +625,16 @@ def test_recall_with_single_vector_valid_2000(
     ):
         full_schema_params = request.getfixturevalue("full_schema_new")
 
+        # Build per-field metric type map so ground truth uses each field's
+        # actual index metric (fields may fall back to HnswIndexParam/IP).
+        field_metric_map = {}
         for vector_para in full_schema_params.vectors:
-            if vector_para.name == "vector_fp32_field":
-                metric_type = vector_para.index_param.metric_type
-                break
+            if vector_para.index_param is not None:
+                field_metric_map[vector_para.name] = vector_para.index_param.metric_type
+            else:
+                field_metric_map[vector_para.name] = MetricType.IP
+
+        metric_type = field_metric_map.get("vector_fp32_field", MetricType.IP)
 
         multiple_docs = [
             generate_doc_recall(i, full_collection_new.schema) for i in range(doc_num)
@@ -621,9 +681,13 @@ def test_recall_with_single_vector_valid_2000(
                 multiple_docs[i].vectors[field_name] for i in range(query_num)
             ]
 
-        # Get ground truth mapping
+        # Get ground truth mapping (pass per-field metric map)
         ground_truth_map = get_ground_truth_map(
-            full_collection_new, multiple_docs, query_vectors_map, metric_type, top_k
+            full_collection_new,
+            multiple_docs,
+            query_vectors_map,
+            field_metric_map,
+            top_k,
         )
 
         # Validate ground truth mapping structure
@@ -662,8 +726,8 @@ def test_recall_with_single_vector_valid_2000(
 
         print("(recall_at_k_stats:\n")
         print(recall_at_k_stats)
-        print("metric_type:")
-        print(metric_type)
+        print("field_metric_map:")
+        print(field_metric_map)
         # Print Recall@K statistics
         print(f"Recall@{top_k} using Ground Truth:")
         for field_name, stats in recall_at_k_stats.items():

python/zvec/__init__.py

@@ -60,6 +60,8 @@
     AddColumnOption,
     AlterColumnOption,
     CollectionOption,
+    DiskAnnIndexParam,
+    DiskAnnQueryParam,
     FlatIndexParam,
     HnswIndexParam,
     HnswQueryParam,
@@ -116,6 +118,8 @@
     "HnswRabitqIndexParam",
     "FlatIndexParam",
     "IVFIndexParam",
+    "DiskAnnIndexParam",
+    "DiskAnnQueryParam",
     "CollectionOption",
     "IndexOption",
     "OptimizeOption",

python/zvec/__init__.pyi

@@ -16,6 +16,8 @@ from .model.param import (
     AddColumnOption,
     AlterColumnOption,
     CollectionOption,
+    DiskAnnIndexParam,
+    DiskAnnQueryParam,
     FlatIndexParam,
     HnswIndexParam,
     HnswQueryParam,
@@ -53,6 +55,8 @@ __all__: list = [
     "DataType",
     "DenseEmbeddingFunction",
     "DenseEmbeddingFunction",
+    "DiskAnnIndexParam",
+    "DiskAnnQueryParam",
     "Doc",
     "FieldSchema",
     "FlatIndexParam",

python/zvec/model/param/__init__.py

@@ -17,6 +17,8 @@
     AddColumnOption,
     AlterColumnOption,
     CollectionOption,
+    DiskAnnIndexParam,
+    DiskAnnQueryParam,
     FlatIndexParam,
     HnswIndexParam,
     HnswQueryParam,
@@ -35,6 +37,8 @@
     "AddColumnOption",
     "AlterColumnOption",
     "CollectionOption",
+    "DiskAnnIndexParam",
+    "DiskAnnQueryParam",
     "FlatIndexParam",
     "HnswIndexParam",
     "HnswQueryParam",

src/ailego/algorithm/kmeans.h

@@ -94,6 +94,7 @@ class Kmc2CentroidsGenerator {
     auto *centroids = owner->mutable_centroids();
 
     std::mt19937 mt((std::random_device())());
+
     std::uniform_real_distribution<float> dist(0.0, 1.0);
 
     ContainerType benches(cache.dimension());
@@ -1216,4 +1217,4 @@ using NibbleInnerProductKmeans =
     LloydCluster<T, TPool, TContext, NibbleVectorArray<T>>;
 
 }  // namespace ailego
-}  // namespace zvec
+}  // namespace zvec

src/ailego/algorithm/lloyd_cluster.h

@@ -361,4 +361,4 @@ class LloydCluster {
 };
 
 }  // namespace ailego
-}  // namespace zvec
+}  // namespace zvec

src/binding/c/CMakeLists.txt

@@ -83,6 +83,11 @@ set_target_properties(zvec_c_api PROPERTIES
     VISIBILITY_INLINES_HIDDEN ON
 )
 
+# On Windows, define ZVEC_BUILD_SHARED so that c_api.h uses __declspec(dllexport)
+if(MSVC OR WIN32)
+    target_compile_definitions(zvec_c_api PRIVATE ZVEC_BUILD_SHARED)
+endif()
+
 find_package(Threads REQUIRED)
 
 # Static linking of C++ standard library is handled in platform-specific sections