refactor(hnsw): use turbo for distance calculation

python/tests/test_turbo_hnsw_int8.py

@@ -0,0 +1,98 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Packaged-extension regression tests for the HNSW Turbo INT8 path."""
+
+from __future__ import annotations
+
+import numpy as np
+import pytest
+
+import zvec
+from zvec import (
+    CollectionOption,
+    CollectionSchema,
+    Doc,
+    HnswIndexParam,
+    HnswQueryParam,
+    Query,
+    VectorSchema,
+)
+from zvec.typing import DataType, MetricType, QuantizeType
+
+DIMENSION = 32
+DOC_COUNT = 64
+
+
+def _normalized_vectors() -> list[list[float]]:
+    rng = np.random.default_rng(2026)
+    vectors = rng.standard_normal((DOC_COUNT, DIMENSION)).astype(np.float32)
+    vectors /= np.linalg.norm(vectors, axis=1, keepdims=True)
+    return vectors.tolist()
+
+
+@pytest.mark.parametrize(
+    "metric", [MetricType.COSINE, MetricType.L2], ids=["cosine", "l2"]
+)
+def test_turbo_int8_hnsw_python_registration_roundtrip(tmp_path, metric):
+    """Keep Turbo's factory registration reachable from the Python module."""
+    path = str(tmp_path / f"turbo_int8_hnsw_{metric.name.lower()}")
+    vectors = _normalized_vectors()
+    schema = CollectionSchema(
+        name="turbo_int8_hnsw",
+        vectors=[
+            VectorSchema(
+                "dense",
+                DataType.VECTOR_FP32,
+                dimension=DIMENSION,
+                index_param=HnswIndexParam(
+                    metric_type=metric,
+                    m=16,
+                    ef_construction=100,
+                    quantize_type=QuantizeType.INT8,
+                ),
+            )
+        ],
+    )
+
+    collection = zvec.create_and_open(
+        path=path,
+        schema=schema,
+        option=CollectionOption(read_only=False, enable_mmap=True),
+    )
+    try:
+        docs = [
+            Doc(id=str(i), vectors={"dense": vector})
+            for i, vector in enumerate(vectors)
+        ]
+        for status in collection.insert(docs):
+            assert status.ok()
+        collection.optimize()
+
+        query = Query(
+            field_name="dense",
+            vector=vectors[17],
+            param=HnswQueryParam(ef=128),
+        )
+        hits = collection.query(query, topk=5)
+        assert hits[0].id == "17"
+    finally:
+        collection.close()
+
+    reopened = zvec.open(path, option=CollectionOption(read_only=True))
+    try:
+        hits = reopened.query(query, topk=5)
+        assert hits[0].id == "17"
+    finally:
+        reopened.close()

src/core/algorithm/hnsw/CMakeLists.txt

@@ -10,7 +10,7 @@ cc_library(
     NAME core_knn_hnsw 
     STATIC SHARED STRICT ALWAYS_LINK
     SRCS *.cc
-    LIBS core_framework core_utility sparsehash
+    LIBS core_framework core_utility sparsehash zvec_turbo
     INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
     LDFLAGS "${CORE_KNN_HNSW_LDFLAGS}"
     VERSION "${PROXIMA_ZVEC_VERSION}"

src/core/algorithm/hnsw/hnsw_context.h

@@ -142,6 +142,18 @@ class HnswContext : public IndexContext {
     return vector_source_;
   }
 
+  //! Keep the unquantized input vector for external-vector graph
+  //! construction. The streamer receives a quantized datapoint from the
+  //! interface, but build-time node-to-node comparisons must stay in the
+  //! external source's input layout.
+  inline void set_external_build_query(const void *query) {
+    external_build_query_ = query;
+  }
+
+  inline const void *external_build_query() const {
+    return external_build_query_;
+  }
+
   inline void reset_query_raw(const void *query, const IndexMeta &meta) {
     dc_.set_dim(meta.dimension());
     dc_.reset_query(query);
@@ -289,8 +301,9 @@ class HnswContext : public IndexContext {
   //! distance computation
   inline void reset_query(const void *query, const IndexMeta &meta) {
     dc_.set_dim(meta.dimension());
-    if (auto query_preprocess_func = index_metric_->get_query_preprocess_func();
-        query_preprocess_func != nullptr) {
+    auto query_preprocess_func =
+        index_metric_ ? index_metric_->get_query_preprocess_func() : nullptr;
+    if (query_preprocess_func != nullptr) {
       size_t dim = meta.dimension();
       preprocess_buffer_.resize(dim);
       memcpy(preprocess_buffer_.data(), query, dim);
@@ -444,6 +457,7 @@ class HnswContext : public IndexContext {
     set_group_params(0, 0);
     reset_group_by();
     set_vector_source(nullptr);
+    set_external_build_query(nullptr);
     dc_.set_provider(nullptr);
   }
 
@@ -616,6 +630,7 @@ class HnswContext : public IndexContext {
   HnswDistCalculator dc_;
   IndexMetric::Pointer metric_;
   const VectorSource *vector_source_{nullptr};
+  const void *external_build_query_{nullptr};
   size_t vector_data_size_{0};
   size_t extra_values_size_{0};
 

src/core/algorithm/hnsw/hnsw_dist_calculator.h

@@ -13,6 +13,7 @@
 // limitations under the License.
 #pragma once
 
+#include <algorithm>
 #include <zvec/core/framework/index_meta.h>
 #include <zvec/core/framework/index_metric.h>
 #include <zvec/core/framework/index_provider.h>
@@ -38,8 +39,9 @@ class HnswDistCalculator {
   HnswDistCalculator(const HnswEntity *entity,
                      const IndexMetric::Pointer &metric, uint32_t dim)
       : entity_(entity),
-        distance_(metric->distance()),
-        batch_distance_(metric->batch_distance()),
+        distance_(metric ? metric->distance() : IndexMetric::MatrixDistance{}),
+        batch_distance_(metric ? metric->batch_distance()
+                               : IndexMetric::MatrixBatchDistance{}),
         query_(nullptr),
         dim_(dim),
         compare_cnt_(0) {}
@@ -49,8 +51,9 @@ class HnswDistCalculator {
                      const IndexMetric::Pointer &metric, uint32_t dim,
                      const void *query)
       : entity_(entity),
-        distance_(metric->distance()),
-        batch_distance_(metric->batch_distance()),
+        distance_(metric ? metric->distance() : IndexMetric::MatrixDistance{}),
+        batch_distance_(metric ? metric->batch_distance()
+                               : IndexMetric::MatrixBatchDistance{}),
         query_(query),
         dim_(dim),
         compare_cnt_(0) {}
@@ -59,23 +62,26 @@ class HnswDistCalculator {
   HnswDistCalculator(const HnswEntity *entity,
                      const IndexMetric::Pointer &metric)
       : entity_(entity),
-        distance_(metric->distance()),
-        batch_distance_(metric->batch_distance()),
+        distance_(metric ? metric->distance() : IndexMetric::MatrixDistance{}),
+        batch_distance_(metric ? metric->batch_distance()
+                               : IndexMetric::MatrixBatchDistance{}),
         query_(nullptr),
         dim_(0),
         compare_cnt_(0) {}
 
   void update(const HnswEntity *entity, const IndexMetric::Pointer &metric) {
     entity_ = entity;
-    distance_ = metric->distance();
-    batch_distance_ = metric->batch_distance();
+    distance_ = metric ? metric->distance() : IndexMetric::MatrixDistance{};
+    batch_distance_ =
+        metric ? metric->batch_distance() : IndexMetric::MatrixBatchDistance{};
   }
 
   void update(const HnswEntity *entity, const IndexMetric::Pointer &metric,
               uint32_t dim) {
     entity_ = entity;
-    distance_ = metric->distance();
-    batch_distance_ = metric->batch_distance();
+    distance_ = metric ? metric->distance() : IndexMetric::MatrixDistance{};
+    batch_distance_ =
+        metric ? metric->batch_distance() : IndexMetric::MatrixBatchDistance{};
     dim_ = dim;
   }
 
@@ -107,6 +113,11 @@ class HnswDistCalculator {
 
     float score{0.0f};
 
+    if (ailego_unlikely(!distance_)) {
+      LOG_ERROR("Distance function is not initialized");
+      error_ = true;
+      return 0.0f;
+    }
     distance_(vec_lhs, vec_rhs, dim_, &score);
 
     return score;
@@ -186,6 +197,12 @@ class HnswDistCalculator {
                   const void **extra_values) {
     compare_cnt_++;
 
+    if (ailego_unlikely(!batch_distance_)) {
+      LOG_ERROR("Batch distance function is not initialized");
+      error_ = true;
+      std::fill(distances, distances + num, 0.0f);
+      return;
+    }
     batch_distance_(vecs, query_, num, dim_, distances, extra_values);
   }
 
@@ -198,6 +215,11 @@ class HnswDistCalculator {
       return 0.0f;
     }
     dist_t score = 0;
+    if (ailego_unlikely(!batch_distance_)) {
+      LOG_ERROR("Batch distance function is not initialized");
+      error_ = true;
+      return 0.0f;
+    }
     if (extra_values != nullptr) {
       batch_distance_(&feat, query_, 1, dim_, &score, &extra_values);
     } else {